├── REQUIRE ├── .codecov.yml ├── Cincinnati ├── REQUIRE ├── src │ ├── types.jl │ ├── greetings.jl │ └── Cincinnati.jl ├── README.md ├── test │ └── runtests.jl ├── LICENSE.md └── appveyor.yml ├── .gitignore ├── Chocolate-Cake-20.jpg ├── README.md ├── LICENSE.md ├── .travis.yml ├── appveyor.yml ├── setup.md ├── 06. Ordinary differential equations.ipynb ├── live ├── 06. Ordinary differential equations.ipynb ├── 13. Manipulating images.ipynb ├── 05. Optimization with JuMP.ipynb ├── 17. Distributed arrays.ipynb └── 18. Pleasingly parallel simulations.ipynb ├── 12. Developing a package.ipynb ├── 13. Manipulating images.ipynb ├── 05. Optimization with JuMP.ipynb ├── 17. Distributed arrays.ipynb ├── 00. Introduction to Julia.ipynb ├── 10. Memory layout.ipynb ├── 01. Example - random walks.ipynb ├── 16. Domain-specific languages in Julia - Generating callable objects using macros.ipynb ├── 18. Pleasingly parallel simulations.ipynb └── 14. Interoperability - Python and C.ipynb /REQUIRE: -------------------------------------------------------------------------------- 1 | julia 0.6 2 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | -------------------------------------------------------------------------------- /Cincinnati/REQUIRE: -------------------------------------------------------------------------------- 1 | julia 0.6 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.jl.cov 2 | *.jl.*.cov 3 | *.jl.mem 4 | -------------------------------------------------------------------------------- /Cincinnati/src/types.jl: -------------------------------------------------------------------------------- 1 | struct Point2D 2 | x::Float64 3 | y::Float64 4 | end 5 | -------------------------------------------------------------------------------- /Cincinnati/src/greetings.jl: -------------------------------------------------------------------------------- 1 | 2 | 3 | greeting(name) = "Hello, $name" 4 | 5 | bye(name) = "Bye, $(name)!" 6 | -------------------------------------------------------------------------------- /Chocolate-Cake-20.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dpsanders/cincinnati_julia_workshop/master/Chocolate-Cake-20.jpg -------------------------------------------------------------------------------- /Cincinnati/src/Cincinnati.jl: -------------------------------------------------------------------------------- 1 | module Cincinnati 2 | 3 | export greeting, bye 4 | 5 | 6 | include("greetings.jl") 7 | include("types.jl") 8 | 9 | end # module 10 | -------------------------------------------------------------------------------- /Cincinnati/README.md: -------------------------------------------------------------------------------- 1 | # Cincinnati 2 | 3 | [![Build Status](https://travis-ci.org/dpsanders/Cincinnati.jl.svg?branch=master)](https://travis-ci.org/dpsanders/Cincinnati.jl) 4 | 5 | [![Coverage Status](https://coveralls.io/repos/dpsanders/Cincinnati.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/dpsanders/Cincinnati.jl?branch=master) 6 | 7 | [![codecov.io](http://codecov.io/github/dpsanders/Cincinnati.jl/coverage.svg?branch=master)](http://codecov.io/github/dpsanders/Cincinnati.jl?branch=master) 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Julia workshop at the University of Cincinnati 2 | 3 | Welcome to the workshop on the [Julia language](http://www.julialang.org) to be held at the University of Cincinnati on March 22 & 23, 2018! It is run by [David P. Sanders](http://sistemas.fciencias.unam.mx/~dsanders/). 4 | 5 | [Here](setup.md) is some information about setting up your environment and learning basic Julia syntax, so that we can progress more rapidly during the workshop itself. 6 | 7 | ## Questions and comments 8 | Please contact [David](dpsanders@ciencias.unam.mx) if you have any further questions and comments 9 | -------------------------------------------------------------------------------- /Cincinnati/test/runtests.jl: -------------------------------------------------------------------------------- 1 | using Cincinnati 2 | using Base.Test 3 | 4 | # write your own tests here 5 | @testset "Basic tests" begin 6 | @test Cincinnati.greeting("David") == "Hello, David" 7 | @test Cincinnati.greeting("Jeff") == "Hello, Jeff" 8 | end 9 | 10 | @testset "More tests" begin 11 | @test Cincinnati.greeting("David") == "Hello, David" 12 | @test Cincinnati.greeting("Jeff") == "Hello, Jeff" 13 | end 14 | 15 | @testset "Even more tests" begin 16 | @test Cincinnati.greeting("David") == "Hello, David" 17 | @test Cincinnati.greeting("Jeff") == "Hello, Jeff" 18 | end 19 | 20 | @testset "Goodbye tests" begin 21 | @test Cincinnati.bye("David") == "Bye, David!" 22 | end 23 | 24 | @testset "Exported names" begin 25 | @test bye("David") == "Bye, David!" 26 | end 27 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The Cincinnati.jl package is licensed under the MIT "Expat" License: 2 | 3 | > Copyright (c) 2018: David Sanders. 4 | > 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in all 13 | > copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | > SOFTWARE. 22 | > 23 | -------------------------------------------------------------------------------- /Cincinnati/LICENSE.md: -------------------------------------------------------------------------------- 1 | The Cincinnati.jl package is licensed under the MIT "Expat" License: 2 | 3 | > Copyright (c) 2018: David Sanders. 4 | > 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in all 13 | > copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | > SOFTWARE. 22 | > 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | ## Documentation: http://docs.travis-ci.com/user/languages/julia/ 2 | language: julia 3 | os: 4 | - linux 5 | - osx 6 | julia: 7 | - 0.6 8 | - nightly 9 | notifications: 10 | email: false 11 | git: 12 | depth: 99999999 13 | 14 | ## uncomment the following lines to allow failures on nightly julia 15 | ## (tests will run but not make your overall status red) 16 | #matrix: 17 | # allow_failures: 18 | # - julia: nightly 19 | 20 | ## uncomment and modify the following lines to manually install system packages 21 | #addons: 22 | # apt: # apt-get for linux 23 | # packages: 24 | # - gfortran 25 | #before_script: # homebrew for mac 26 | # - if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; fi 27 | 28 | ## uncomment the following lines to override the default test script 29 | #script: 30 | # - julia -e 'Pkg.clone(pwd()); Pkg.build("Cincinnati"); Pkg.test("Cincinnati"; coverage=true)' 31 | after_success: 32 | # push coverage results to Coveralls 33 | - julia -e 'cd(Pkg.dir("Cincinnati")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' 34 | # push coverage results to Codecov 35 | - julia -e 'cd(Pkg.dir("Cincinnati")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' 36 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" 4 | - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" 5 | - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" 6 | - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" 7 | 8 | ## uncomment the following lines to allow failures on nightly julia 9 | ## (tests will run but not make your overall status red) 10 | #matrix: 11 | # allow_failures: 12 | # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" 13 | # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" 14 | 15 | branches: 16 | only: 17 | - master 18 | - /release-.*/ 19 | 20 | notifications: 21 | - provider: Email 22 | on_build_success: false 23 | on_build_failure: false 24 | on_build_status_changed: false 25 | 26 | install: 27 | - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" 28 | # If there's a newer build queued for the same PR, cancel this one 29 | - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` 30 | https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` 31 | Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` 32 | throw "There are newer queued builds for this pull request, failing early." } 33 | # Download most recent Julia Windows binary 34 | - ps: (new-object net.webclient).DownloadFile( 35 | $env:JULIA_URL, 36 | "C:\projects\julia-binary.exe") 37 | # Run installer silently, output to C:\projects\julia 38 | - C:\projects\julia-binary.exe /S /D=C:\projects\julia 39 | 40 | build_script: 41 | # Need to convert from shallow to complete for Pkg.clone to work 42 | - IF EXIST .git\shallow (git fetch --unshallow) 43 | - C:\projects\julia\bin\julia -e "versioninfo(); 44 | Pkg.clone(pwd(), \"Cincinnati\"); Pkg.build(\"Cincinnati\")" 45 | 46 | test_script: 47 | - C:\projects\julia\bin\julia -e "Pkg.test(\"Cincinnati\")" 48 | -------------------------------------------------------------------------------- /Cincinnati/appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" 4 | - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" 5 | - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" 6 | - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" 7 | 8 | ## uncomment the following lines to allow failures on nightly julia 9 | ## (tests will run but not make your overall status red) 10 | #matrix: 11 | # allow_failures: 12 | # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" 13 | # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" 14 | 15 | branches: 16 | only: 17 | - master 18 | - /release-.*/ 19 | 20 | notifications: 21 | - provider: Email 22 | on_build_success: false 23 | on_build_failure: false 24 | on_build_status_changed: false 25 | 26 | install: 27 | - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" 28 | # If there's a newer build queued for the same PR, cancel this one 29 | - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` 30 | https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` 31 | Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` 32 | throw "There are newer queued builds for this pull request, failing early." } 33 | # Download most recent Julia Windows binary 34 | - ps: (new-object net.webclient).DownloadFile( 35 | $env:JULIA_URL, 36 | "C:\projects\julia-binary.exe") 37 | # Run installer silently, output to C:\projects\julia 38 | - C:\projects\julia-binary.exe /S /D=C:\projects\julia 39 | 40 | build_script: 41 | # Need to convert from shallow to complete for Pkg.clone to work 42 | - IF EXIST .git\shallow (git fetch --unshallow) 43 | - C:\projects\julia\bin\julia -e "versioninfo(); 44 | Pkg.clone(pwd(), \"Cincinnati\"); Pkg.build(\"Cincinnati\")" 45 | 46 | test_script: 47 | - C:\projects\julia\bin\julia -e "Pkg.test(\"Cincinnati\")" 48 | -------------------------------------------------------------------------------- /setup.md: -------------------------------------------------------------------------------- 1 | 2 | ## Running Julia online: JuliaBox 3 | The easiest way to start using Julia is via the online [JuliaBox](http://www.juliabox.com) service. 4 | This provides an online version of the [Jupyter notebook](http://www.jupyter.org), which we will be using throughout. Use `Shift-Enter` to execute a cell. 5 | 6 | An alternative is [CoCalc](http://www.cocalc.com), which allows simultaneous editing of notebooks by several people. 7 | 8 | ## Installing Julia and Jupyter locally 9 | To install Julia and Jupyter locally on your own machine, do the following. 10 | 11 | [Note that it is *not* necessary to install Anaconda separately; Julia will do this automatically for you.] 12 | 13 | 1. Download and install the stable version of Julia (0.6.2) from [here](http://www.julialang.org/downloads) for your operating system. 14 | 15 | [If possible, please also download and install a "nightly build" from [here](https://julialang.org/downloads/nightlies.html), since we will be looking at some differences between the current stable version, 0.6.2, and the soon-to-be-released 1.0 version.] 16 | 17 | 2. Run the copy of Julia that you just installed. 18 | 19 | Execute the following commands within the Julia terminal ("REPL") environment, where you will see a `julia> ` prompt. 20 | 21 | 3. If you use Linux, first type: 22 | ```jl 23 | julia> ENV["JUPYTER"] = "" 24 | ``` 25 | 26 | 4. Now install the IJulia package, which will automatically install Jupyter (using `miniconda`): 27 | ``` 28 | julia> Pkg.add("IJulia") 29 | ``` 30 | 31 | 5. Open the notebook as follows. 32 | ```jl 33 | julia> using IJulia 34 | julia> notebook() 35 | ``` 36 | By default, new notebooks will be created in your home directory. Navigate to a different directory to save them in your preferred location. 37 | 38 | 6. Install some of the packages that we will use during the course (you will need an internet connection) 39 | ```jl 40 | julia> packages = split( 41 | """Plots GR PlotlyJS Interact 42 | BenchmarkTools 43 | DataFrames Query 44 | JLD2 Distributions StatPlots 45 | Optim JuMP 46 | ProfileView 47 | TreeView StaticArrays Revise 48 | DifferentialEquations 49 | MappedArrays 50 | Documenter PkgDev 51 | ForwardDiff 52 | """) 53 | 54 | julia> for package in packages 55 | Pkg.add(package) 56 | end 57 | ``` 58 | ## Get up to speed with basic Julia syntax 59 | 60 | If you have had little exposure to Julia, please work through [this video tutorial](https://youtu.be/4igzy3bGVkQ) to get up to speed with basic Julia syntax, in particular the notebooks 1 through 8 (up to and including "Plotting"). The notebooks are available directly in JuliaBox, or [here](https://github.com/JuliaComputing/JuliaBoxTutorials/tree/master/intro-to-julia). 61 | 62 | We suggest that you bookmark, download or even print out the following two "cheat sheets" with summaries of basic Julia syntax: 63 | - a [one-page summary by Steven Johnson](https://github.com/stevengj/1806/blob/master/julia/Julia-cheatsheet.pdf) 64 | 65 | - a [more extensive summary](https://juliadocs.github.io/Julia-Cheat-Sheet) 66 | 67 | There is an ever-growing list of resources for learning Julia available on the [learning page](http://www.julialang.org/learning) of the Julia homepage; in particular, check out the [QuantEcon lectures](https://lectures.quantecon.org/jl). 68 | 69 | 70 | ## Julia IDE: Juno 71 | 72 | There are two IDEs (Integrated Development Environments) available for Julia: Juno, based on the [Atom editor](https://atom.io/), and a Julia plug-in for the Visual Studio Code editor. 73 | 74 | Please download Atom and install the `uber-juno` package; this will give you a Julia development environment. More info is available at the [Juno IDE homepage](http://junolab.org/). 75 | 76 | 77 | ## Questions and comments 78 | Please contact [David](dpsanders@ciencias.unam.mx) if you have any further questions and comments 79 | 80 | -------------------------------------------------------------------------------- /06. Ordinary differential equations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ordinary differential equations" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Julia now has perhaps the best suite of ordinary differential equation solvers of *any* platform.\n", 15 | "It also has SDEs, and other related types of equation. There are also various efforts for solving different kinds of PDEs." 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "using DifferentialEquations" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "To solve an ODE of the form" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "$$\\dot{\\mathbf{x}} = \\mathbf{f}_\\mathbf{\\mu}(\\mathbf{x}, t),$$\n", 39 | "where $\\mathbf{\\mu}$ is a vector of parameters, we do the following:" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "(::Lorenz) (generic function with 9 methods)" 51 | ] 52 | }, 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "f = @ode_def Lorenz begin\n", 60 | " dx = σ*(y-x)\n", 61 | " dy = x*(ρ-z) - y\n", 62 | " dz = x*y - β*z\n", 63 | "end σ ρ β" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "u0 = [1.0, 0.0, 0.0]\n", 73 | "tspan = (0.0, 20.0)\n", 74 | "\n", 75 | "p = [10.0, 28.0, 8/3] # classical parameter values for the Lorenz attractorplot(sol,vars=(1,2,3))\n", 76 | "\n", 77 | "\n", 78 | "prob = ODEProblem(f, u0, tspan, p)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "Having defined an object `ODEProblem` with the data of the ODE, we now solve it:" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "soln = solve(prob)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "Here, a default numerical method was chosen based on features of the ODE. Of course, we can also specify the method to use." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "using Plots; gr()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "`DifferentialEquations.jl` exploits a unique feature of `Plots.jl`, namely **plot recipes**. \n", 118 | "This is a way in which a package can define *how to plot* its own objects, using facilities provided by `Plots.jl`. In this way, a package can jack into the `Plots.jl` machinery, without the need to redefine all kinds of plotting from scratch." 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "plot(soln)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "By default, each variable is shown separately as a function of time.\n", 135 | "\n", 136 | "Instead, we can specify which variables to plot:" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "plotlyjs()" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "plot(soln, vars = (1, 2, 3))" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [] 163 | } 164 | ], 165 | "metadata": { 166 | "kernelspec": { 167 | "display_name": "Julia 0.6.2", 168 | "language": "julia", 169 | "name": "julia-0.6" 170 | }, 171 | "language_info": { 172 | "file_extension": ".jl", 173 | "mimetype": "application/julia", 174 | "name": "julia", 175 | "version": "0.6.2" 176 | } 177 | }, 178 | "nbformat": 4, 179 | "nbformat_minor": 2 180 | } 181 | -------------------------------------------------------------------------------- /live/06. Ordinary differential equations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Ordinary differential equations" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Julia now has perhaps the best suite of ordinary differential equation solvers of *any* platform.\n", 15 | "It also has SDEs, and other related types of equation. There are also various efforts for solving different kinds of PDEs." 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "using DifferentialEquations" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "To solve an ODE of the form" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "$$\\dot{\\mathbf{x}} = \\mathbf{f}_\\mathbf{\\mu}(\\mathbf{x}, t),$$\n", 39 | "where $\\mathbf{\\mu}$ is a vector of parameters, we do the following:" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "(::Lorenz) (generic function with 9 methods)" 51 | ] 52 | }, 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "f = @ode_def Lorenz begin\n", 60 | " dx = σ*(y-x)\n", 61 | " dy = x*(ρ-z) - y\n", 62 | " dz = x*y - β*z\n", 63 | "end σ ρ β" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "u0 = [1.0, 0.0, 0.0]\n", 73 | "tspan = (0.0, 20.0)\n", 74 | "\n", 75 | "p = [10.0, 28.0, 8/3] # classical parameter values for the Lorenz attractorplot(sol,vars=(1,2,3))\n", 76 | "\n", 77 | "\n", 78 | "prob = ODEProblem(f, u0, tspan, p)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "Having defined an object `ODEProblem` with the data of the ODE, we now solve it:" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "soln = solve(prob)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "Here, a default numerical method was chosen based on features of the ODE. Of course, we can also specify the method to use." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "using Plots; gr()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "`DifferentialEquations.jl` exploits a unique feature of `Plots.jl`, namely **plot recipes**. \n", 118 | "This is a way in which a package can define *how to plot* its own objects, using facilities provided by `Plots.jl`. In this way, a package can jack into the `Plots.jl` machinery, without the need to redefine all kinds of plotting from scratch." 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "plot(soln)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "By default, each variable is shown separately as a function of time.\n", 135 | "\n", 136 | "Instead, we can specify which variables to plot:" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "plotlyjs()" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "plot(soln, vars = (1, 2, 3))" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [] 163 | } 164 | ], 165 | "metadata": { 166 | "kernelspec": { 167 | "display_name": "Julia 0.6.2", 168 | "language": "julia", 169 | "name": "julia-0.6" 170 | }, 171 | "language_info": { 172 | "file_extension": ".jl", 173 | "mimetype": "application/julia", 174 | "name": "julia", 175 | "version": "0.6.2" 176 | } 177 | }, 178 | "nbformat": 4, 179 | "nbformat_minor": 2 180 | } 181 | -------------------------------------------------------------------------------- /12. Developing a package.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Developing a package " 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Once you have some experience with Julia, one of the best ways of learning more is to contribute to a pre-existing package. You can also [write tests for Julia itself](https://github.com/JuliaLang/julia/issues/9493). You may also wish to develop a new package for functionality that is not yet available in the Julia ecosystem." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "**NB: Details of package management will change significantly with what is currently called `Pkg3.jl` in version 1.0 of Julia.**" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "## Create a package " 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "Julia's package manager simplifies some of the trickier aspects of setting up packages. To create a new, empty package, do:" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "Pkg.add(\"PkgDev\")" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "Pkg.generate(\"Cincinnati\", \"MIT\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "replacing `Cincinnati` by the name of the package you would like to generate.\n", 61 | "\n", 62 | "This creates a new directory with the same name inside `~/.julia/v0.6` with the same name, with the default MIT license and the standard Julia package structure:" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "Packages in Julia are `git` repositories. Now (or yesterday) is a good time to learn `git`, e.g. using \n", 70 | "the [Software Carpentry lessons](http://swcarpentry.github.io/git-novice/)." 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "Inside the `src` subdirectory is a single Julia file with the same name as your package:" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "; ls ~/.julia/v0.6/Cincinnati/src" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "; cat ~/.julia/v0.6/Cincinnati/src/Cincinnati.jl" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "This is a Julia **module**, which can be thought of as a separate workspace with separate names. You make available only those functions that are relevant for the user of the package using `export`." 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "## Develop your package " 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "The next step is to fill up your package with code in the `src` directory.\n", 117 | "\n", 118 | "It is standard to separate the code into different files that you `include` in the module:" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "module Cincinnati\n", 128 | "\n", 129 | "include(\"my_stuff.jl\")\n", 130 | "include(\"my_other_stuff.jl\")\n", 131 | "\n", 132 | "end # module" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "## Write tests " 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "All code requires **tests**, using the `Base.Test` package:" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "using Base.Test" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "@testset \"Testing arithmetic\" begin\n", 165 | " @test 3+3 == 6\n", 166 | " \n", 167 | " x = 17\n", 168 | " \n", 169 | " @test x/3 isa Float64\n", 170 | "end" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "The testing code goes in `runtests.jl` in the `test` subdirectory. Again, you can `include` several files in `runtests.jl`." 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "You can test your nascent package with" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "Pkg.test(\"Cincinnati\")" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "## Document your package " 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "You must document your package if you would like to have >1 user. \n", 208 | "\n", 209 | "Julia has [`Documenter.jl`](https://github.com/JuliaDocs/Documenter.jl) for producing nice documentation for your package. \n", 210 | "\n", 211 | "The best thing to do is to follow the layout of a recent published package.\n", 212 | "\n", 213 | "There were previously packages that automated the generation of skeleton documentation, but they seem to have disappeared." 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "## Publishing your package " 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "To publish your package, or to make a new release, use [Attobot](https://github.com/attobot/attobot).\n", 228 | "This bot must be installed on your GitHub repository. When you make a new release in the Releases tab, it will automatically detect it and send a Pull Request to https://github.com/JuliaLang/METADATA.jl/pulls." 229 | ] 230 | } 231 | ], 232 | "metadata": { 233 | "kernelspec": { 234 | "display_name": "Julia 0.6.2", 235 | "language": "julia", 236 | "name": "julia-0.6" 237 | }, 238 | "language_info": { 239 | "file_extension": ".jl", 240 | "mimetype": "application/julia", 241 | "name": "julia", 242 | "version": "0.6.2" 243 | }, 244 | "widgets": { 245 | "state": {}, 246 | "version": "1.1.2" 247 | } 248 | }, 249 | "nbformat": 4, 250 | "nbformat_minor": 1 251 | } 252 | -------------------------------------------------------------------------------- /13. Manipulating images.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "using Images" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Let's load in an image and investigate what the resulting object looks like:" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "cake = load(\"Chocolate-Cake-20.jpg\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "cake" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "typeof(cake)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "cake[1, 1]" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "cake[500, 500]" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "typeof(cake[500, 500])" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "?RGB4" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "We see that the image is stored as a 2D array of RGB objects (3 numbers for each pixel: R, G and B between 0 and 1)." 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "We can instead think of it as a 3D array using the `channelview` function:" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "size(cake) # 966 rows x 1450 columns" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "channels = channelview(cake)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "This is a *view* of the data, without copying:" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "channels[3, 100, 100]" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "show(cake[100, 100])" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "channels[3, 100, 100] = 0.7N0f8" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "show(cake[100, 100]) # has now changed" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "We can convert these numbers to standard floats; note that this copies all the data:" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "channels2 = Float64.(channelview(cake))" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "We can reconstruct the image using `colorview`:" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": null, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "colorview(RGB, channels2[1, :, :], channels2[2, :, :], channels2[3, :, :])" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": { 192 | "collapsed": true 193 | }, 194 | "source": [ 195 | "For example, we could use just the red channel:" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "colorview(RGB, channels2[1, :, :], zeros(channels2[2, :, :]), zeros(channels2[3, :, :]))" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "And convert it to a gray scale:" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "Gray.(channels2[1, :, :])" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "Gray(channels2[1, 100, 100])" 230 | ] 231 | } 232 | ], 233 | "metadata": { 234 | "kernelspec": { 235 | "display_name": "Julia 0.6.2", 236 | "language": "julia", 237 | "name": "julia-0.6" 238 | }, 239 | "language_info": { 240 | "file_extension": ".jl", 241 | "mimetype": "application/julia", 242 | "name": "julia", 243 | "version": "0.6.2" 244 | }, 245 | "toc": { 246 | "colors": { 247 | "hover_highlight": "#DAA520", 248 | "running_highlight": "#FF0000", 249 | "selected_highlight": "#FFD700" 250 | }, 251 | "moveMenuLeft": true, 252 | "nav_menu": { 253 | "height": "12px", 254 | "width": "252px" 255 | }, 256 | "navigate_menu": true, 257 | "number_sections": true, 258 | "sideBar": true, 259 | "threshold": "2", 260 | "toc_cell": false, 261 | "toc_section_display": "block", 262 | "toc_window_display": false 263 | }, 264 | "widgets": { 265 | "state": { 266 | "08d20f1a-4b23-4bc8-93eb-b978d59c403d": { 267 | "views": [ 268 | { 269 | "cell_index": 26 270 | } 271 | ] 272 | }, 273 | "0ed965ea-e77d-4374-835f-19061d581792": { 274 | "views": [ 275 | { 276 | "cell_index": 25 277 | } 278 | ] 279 | }, 280 | "3fc4937d-bb08-4631-86a0-b7ee8d3dda81": { 281 | "views": [ 282 | { 283 | "cell_index": 25 284 | } 285 | ] 286 | }, 287 | "5502f4b2-7889-4ab7-ba24-1747fabd0f25": { 288 | "views": [ 289 | { 290 | "cell_index": 26 291 | } 292 | ] 293 | }, 294 | "8913957e-50a3-41fa-a0c3-24ce8b516722": { 295 | "views": [ 296 | { 297 | "cell_index": 22 298 | } 299 | ] 300 | }, 301 | "a524fb58-32f9-4682-a0be-918328d687c3": { 302 | "views": [ 303 | { 304 | "cell_index": 26 305 | } 306 | ] 307 | }, 308 | "af18e4b2-b8fa-439d-bc4b-323bcbf03a65": { 309 | "views": [ 310 | { 311 | "cell_index": 25 312 | } 313 | ] 314 | }, 315 | "b4728f7c-36da-428c-9266-e0a9bc7c8d5f": { 316 | "views": [ 317 | { 318 | "cell_index": 22 319 | } 320 | ] 321 | }, 322 | "be947142-5acf-4f68-a6a2-e43f3715c4af": { 323 | "views": [ 324 | { 325 | "cell_index": 25 326 | } 327 | ] 328 | } 329 | }, 330 | "version": "1.2.0" 331 | } 332 | }, 333 | "nbformat": 4, 334 | "nbformat_minor": 1 335 | } 336 | -------------------------------------------------------------------------------- /live/13. Manipulating images.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "using Images" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Let's load in an image and investigate what the resulting object looks like:" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "cake = load(\"Chocolate-Cake-20.jpg\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "cake" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "typeof(cake)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "cake[1, 1]" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "cake[500, 500]" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "typeof(cake[500, 500])" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "?RGB4" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "We see that the image is stored as a 2D array of RGB objects (3 numbers for each pixel: R, G and B between 0 and 1)." 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "We can instead think of it as a 3D array using the `channelview` function:" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "size(cake) # 966 rows x 1450 columns" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "channels = channelview(cake)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "This is a *view* of the data, without copying:" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "channels[3, 100, 100]" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "show(cake[100, 100])" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "channels[3, 100, 100] = 0.7N0f8" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "show(cake[100, 100]) # has now changed" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "We can convert these numbers to standard floats; note that this copies all the data:" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "channels2 = Float64.(channelview(cake))" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "We can reconstruct the image using `colorview`:" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": null, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "colorview(RGB, channels2[1, :, :], channels2[2, :, :], channels2[3, :, :])" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": { 192 | "collapsed": true 193 | }, 194 | "source": [ 195 | "For example, we could use just the red channel:" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "colorview(RGB, channels2[1, :, :], zeros(channels2[2, :, :]), zeros(channels2[3, :, :]))" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "And convert it to a gray scale:" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "Gray.(channels2[1, :, :])" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "Gray(channels2[1, 100, 100])" 230 | ] 231 | } 232 | ], 233 | "metadata": { 234 | "kernelspec": { 235 | "display_name": "Julia 0.6.2", 236 | "language": "julia", 237 | "name": "julia-0.6" 238 | }, 239 | "language_info": { 240 | "file_extension": ".jl", 241 | "mimetype": "application/julia", 242 | "name": "julia", 243 | "version": "0.6.2" 244 | }, 245 | "toc": { 246 | "colors": { 247 | "hover_highlight": "#DAA520", 248 | "running_highlight": "#FF0000", 249 | "selected_highlight": "#FFD700" 250 | }, 251 | "moveMenuLeft": true, 252 | "nav_menu": { 253 | "height": "12px", 254 | "width": "252px" 255 | }, 256 | "navigate_menu": true, 257 | "number_sections": true, 258 | "sideBar": true, 259 | "threshold": "2", 260 | "toc_cell": false, 261 | "toc_section_display": "block", 262 | "toc_window_display": false 263 | }, 264 | "widgets": { 265 | "state": { 266 | "08d20f1a-4b23-4bc8-93eb-b978d59c403d": { 267 | "views": [ 268 | { 269 | "cell_index": 26 270 | } 271 | ] 272 | }, 273 | "0ed965ea-e77d-4374-835f-19061d581792": { 274 | "views": [ 275 | { 276 | "cell_index": 25 277 | } 278 | ] 279 | }, 280 | "3fc4937d-bb08-4631-86a0-b7ee8d3dda81": { 281 | "views": [ 282 | { 283 | "cell_index": 25 284 | } 285 | ] 286 | }, 287 | "5502f4b2-7889-4ab7-ba24-1747fabd0f25": { 288 | "views": [ 289 | { 290 | "cell_index": 26 291 | } 292 | ] 293 | }, 294 | "8913957e-50a3-41fa-a0c3-24ce8b516722": { 295 | "views": [ 296 | { 297 | "cell_index": 22 298 | } 299 | ] 300 | }, 301 | "a524fb58-32f9-4682-a0be-918328d687c3": { 302 | "views": [ 303 | { 304 | "cell_index": 26 305 | } 306 | ] 307 | }, 308 | "af18e4b2-b8fa-439d-bc4b-323bcbf03a65": { 309 | "views": [ 310 | { 311 | "cell_index": 25 312 | } 313 | ] 314 | }, 315 | "b4728f7c-36da-428c-9266-e0a9bc7c8d5f": { 316 | "views": [ 317 | { 318 | "cell_index": 22 319 | } 320 | ] 321 | }, 322 | "be947142-5acf-4f68-a6a2-e43f3715c4af": { 323 | "views": [ 324 | { 325 | "cell_index": 25 326 | } 327 | ] 328 | } 329 | }, 330 | "version": "1.2.0" 331 | } 332 | }, 333 | "nbformat": 4, 334 | "nbformat_minor": 1 335 | } 336 | -------------------------------------------------------------------------------- /05. Optimization with JuMP.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Optimization with JuMP" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Julia has several packages for optimization, root-finding, etc. \n", 15 | "For optimization, there are pure-Julia routines in Optim.jl, and an interface to the NLOpt package in NLOpt.jl. \n", 16 | "\n", 17 | "But one of the crown jewels of Julia is JuMP.jl (which stands for Julia Mathematical Programming, i.e. mathematical optimization). This provides a \"domain-specific language\" (DSL) for specifying constrained optimization problems, which it converts into the correct syntax for a range of commercial and open-source optimization packages (\"solvers\")." 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "Let's see a couple of examples, taken from the JuMP documentation. First a linear programming example:" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "data": { 34 | "text/html": [ 35 | "sin(x::Float64) at math.jl:419" 36 | ], 37 | "text/plain": [ 38 | "sin(x::Float64) in Base.Math at math.jl:419" 39 | ] 40 | }, 41 | "execution_count": 2, 42 | "metadata": {}, 43 | "output_type": "execute_result" 44 | } 45 | ], 46 | "source": [ 47 | "@which sin(3.1)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "Pkg.add(\"JuMP\")\n", 64 | "Pkg.add(\"Clp\") # need to add a solver" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 7, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "ename": "LoadError", 74 | "evalue": "\u001b[91mMethodError: objects of type Module are not callable\u001b[39m", 75 | "output_type": "error", 76 | "traceback": [ 77 | "\u001b[91mMethodError: objects of type Module are not callable\u001b[39m", 78 | "", 79 | "Stacktrace:", 80 | " [1] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:522\u001b[22m\u001b[22m" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "using JuMP\n", 86 | "using Clp\n", 87 | "\n", 88 | "m = Model(solver = ClpSolver())\n", 89 | "@variable(m, 0 <= x <= 2 )\n", 90 | "@variable(m, 0 <= y <= 30 )\n", 91 | "\n", 92 | "@objective(m, Max, 5x + 3*y )\n", 93 | "@constraint(m, 1x + 5y <= 3.0 )\n", 94 | "\n", 95 | "print(m)\n", 96 | "\n", 97 | "status = solve(m)\n", 98 | "\n", 99 | "println(\"Objective value: \", getobjectivevalue(m))\n", 100 | "println(\"x = \", getvalue(x))\n", 101 | "println(\"y = \", getvalue(y))" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 10, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "Max 5 x² + 3 y\n", 114 | "Subject to\n", 115 | " x + 5 y ≤ 3\n", 116 | " 0 ≤ x ≤ 2\n", 117 | " 0 ≤ y ≤ 30\n", 118 | "\n", 119 | "******************************************************************************\n", 120 | "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", 121 | " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", 122 | " For more information visit http://projects.coin-or.org/Ipopt\n", 123 | "******************************************************************************\n", 124 | "\n", 125 | "Objective value: 20.600000400982744\n", 126 | "x = 2.0\n", 127 | "y = 0.2000000011886581\n" 128 | ] 129 | } 130 | ], 131 | "source": [ 132 | "using JuMP\n", 133 | "using Ipopt\n", 134 | "\n", 135 | "m = Model(solver = IpoptSolver(print_level=0))\n", 136 | "@variable(m, 0 <= x <= 2 )\n", 137 | "@variable(m, 0 <= y <= 30 )\n", 138 | "\n", 139 | "@objective(m, Max, 5x^2 + 3*y )\n", 140 | "@constraint(m, 1x + 5y <= 3.0 )\n", 141 | "\n", 142 | "print(m)\n", 143 | "\n", 144 | "status = solve(m)\n", 145 | "\n", 146 | "println(\"Objective value: \", getobjectivevalue(m))\n", 147 | "println(\"x = \", getvalue(x))\n", 148 | "println(\"y = \", getvalue(y))" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "## Maximum likelihood" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "Pkg.add(\"Ipopt\")" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "Pkg.build" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 6, 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "using Ipopt" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 11, 188 | "metadata": {}, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "μ = 0.02169217323539194\n", 195 | "mean(data) = 0.021692173235391934\n", 196 | "σ^2 = 0.9732830271428851\n", 197 | "var(data) = 0.9742572844248701\n", 198 | "MLE objective: -1405.3983541204962\n", 199 | "\n", 200 | "With constraint μ == σ^2:\n", 201 | "μ = 0.6062339614754363\n", 202 | "σ^2 = 0.6062339614754364\n", 203 | "Constrained MLE objective: -1753.235675254145\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "# Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors\n", 209 | "# This Source Code Form is subject to the terms of the Mozilla Public\n", 210 | "# License, v. 2.0. If a copy of the MPL was not distributed with this\n", 211 | "# file, You can obtain one at http://mozilla.org/MPL/2.0/.\n", 212 | "using JuMP, Ipopt\n", 213 | "\n", 214 | "# Use nonlinear optimization to compute the maximum likelihood estimate (MLE)\n", 215 | "# of the parameters of a normal distribution\n", 216 | "# aka the sample mean and variance\n", 217 | "\n", 218 | "n = 1000\n", 219 | "data = randn(n)\n", 220 | "\n", 221 | "m = Model(solver=IpoptSolver(print_level=0))\n", 222 | "\n", 223 | "@variable(m, μ, start = 0.0)\n", 224 | "@variable(m, σ >= 0.0, start = 1.0)\n", 225 | "\n", 226 | "@NLobjective(m, Max, (n/2)*log(1/(2π*σ^2))-sum((data[i]-μ)^2 for i=1:n)/(2σ^2))\n", 227 | "\n", 228 | "solve(m)\n", 229 | "\n", 230 | "println(\"μ = \", getvalue(μ))\n", 231 | "println(\"mean(data) = \", mean(data))\n", 232 | "println(\"σ^2 = \", getvalue(σ)^2)\n", 233 | "println(\"var(data) = \", var(data))\n", 234 | "println(\"MLE objective: \", getobjectivevalue(m))\n", 235 | "\n", 236 | "# constrained MLE?\n", 237 | "@NLconstraint(m, μ == σ^2)\n", 238 | "\n", 239 | "solve(m)\n", 240 | "println(\"\\nWith constraint μ == σ^2:\")\n", 241 | "println(\"μ = \", getvalue(μ))\n", 242 | "println(\"σ^2 = \", getvalue(σ)^2)\n", 243 | "\n", 244 | "println(\"Constrained MLE objective: \", getobjectivevalue(m))" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [] 253 | } 254 | ], 255 | "metadata": { 256 | "kernelspec": { 257 | "display_name": "Julia 0.6.2", 258 | "language": "julia", 259 | "name": "julia-0.6" 260 | }, 261 | "language_info": { 262 | "file_extension": ".jl", 263 | "mimetype": "application/julia", 264 | "name": "julia", 265 | "version": "0.6.2" 266 | }, 267 | "toc": { 268 | "colors": { 269 | "hover_highlight": "#DAA520", 270 | "running_highlight": "#FF0000", 271 | "selected_highlight": "#FFD700" 272 | }, 273 | "moveMenuLeft": true, 274 | "nav_menu": { 275 | "height": "137px", 276 | "width": "251px" 277 | }, 278 | "navigate_menu": true, 279 | "number_sections": true, 280 | "sideBar": true, 281 | "threshold": "2", 282 | "toc_cell": false, 283 | "toc_section_display": "block", 284 | "toc_window_display": false 285 | } 286 | }, 287 | "nbformat": 4, 288 | "nbformat_minor": 2 289 | } 290 | -------------------------------------------------------------------------------- /live/05. Optimization with JuMP.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Optimization with JuMP" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Julia has several packages for optimization, root-finding, etc. \n", 15 | "For optimization, there are pure-Julia routines in Optim.jl, and an interface to the NLOpt package in NLOpt.jl. \n", 16 | "\n", 17 | "But one of the crown jewels of Julia is JuMP.jl (which stands for Julia Mathematical Programming, i.e. mathematical optimization). This provides a \"domain-specific language\" (DSL) for specifying constrained optimization problems, which it converts into the correct syntax for a range of commercial and open-source optimization packages (\"solvers\")." 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "Let's see a couple of examples, taken from the JuMP documentation. First a linear programming example:" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "data": { 34 | "text/html": [ 35 | "sin(x::Float64) at math.jl:419" 36 | ], 37 | "text/plain": [ 38 | "sin(x::Float64) in Base.Math at math.jl:419" 39 | ] 40 | }, 41 | "execution_count": 2, 42 | "metadata": {}, 43 | "output_type": "execute_result" 44 | } 45 | ], 46 | "source": [ 47 | "@which sin(3.1)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "Pkg.add(\"JuMP\")\n", 64 | "Pkg.add(\"Clp\") # need to add a solver" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 7, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "ename": "LoadError", 74 | "evalue": "\u001b[91mMethodError: objects of type Module are not callable\u001b[39m", 75 | "output_type": "error", 76 | "traceback": [ 77 | "\u001b[91mMethodError: objects of type Module are not callable\u001b[39m", 78 | "", 79 | "Stacktrace:", 80 | " [1] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:522\u001b[22m\u001b[22m" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "using JuMP\n", 86 | "using Clp\n", 87 | "\n", 88 | "m = Model(solver = ClpSolver())\n", 89 | "@variable(m, 0 <= x <= 2 )\n", 90 | "@variable(m, 0 <= y <= 30 )\n", 91 | "\n", 92 | "@objective(m, Max, 5x + 3*y )\n", 93 | "@constraint(m, 1x + 5y <= 3.0 )\n", 94 | "\n", 95 | "print(m)\n", 96 | "\n", 97 | "status = solve(m)\n", 98 | "\n", 99 | "println(\"Objective value: \", getobjectivevalue(m))\n", 100 | "println(\"x = \", getvalue(x))\n", 101 | "println(\"y = \", getvalue(y))" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 10, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "Max 5 x² + 3 y\n", 114 | "Subject to\n", 115 | " x + 5 y ≤ 3\n", 116 | " 0 ≤ x ≤ 2\n", 117 | " 0 ≤ y ≤ 30\n", 118 | "\n", 119 | "******************************************************************************\n", 120 | "This program contains Ipopt, a library for large-scale nonlinear optimization.\n", 121 | " Ipopt is released as open source code under the Eclipse Public License (EPL).\n", 122 | " For more information visit http://projects.coin-or.org/Ipopt\n", 123 | "******************************************************************************\n", 124 | "\n", 125 | "Objective value: 20.600000400982744\n", 126 | "x = 2.0\n", 127 | "y = 0.2000000011886581\n" 128 | ] 129 | } 130 | ], 131 | "source": [ 132 | "using JuMP\n", 133 | "using Ipopt\n", 134 | "\n", 135 | "m = Model(solver = IpoptSolver(print_level=0))\n", 136 | "@variable(m, 0 <= x <= 2 )\n", 137 | "@variable(m, 0 <= y <= 30 )\n", 138 | "\n", 139 | "@objective(m, Max, 5x^2 + 3*y )\n", 140 | "@constraint(m, 1x + 5y <= 3.0 )\n", 141 | "\n", 142 | "print(m)\n", 143 | "\n", 144 | "status = solve(m)\n", 145 | "\n", 146 | "println(\"Objective value: \", getobjectivevalue(m))\n", 147 | "println(\"x = \", getvalue(x))\n", 148 | "println(\"y = \", getvalue(y))" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "## Maximum likelihood" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "Pkg.add(\"Ipopt\")" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "Pkg.build" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 6, 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "using Ipopt" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 11, 188 | "metadata": {}, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "μ = 0.02169217323539194\n", 195 | "mean(data) = 0.021692173235391934\n", 196 | "σ^2 = 0.9732830271428851\n", 197 | "var(data) = 0.9742572844248701\n", 198 | "MLE objective: -1405.3983541204962\n", 199 | "\n", 200 | "With constraint μ == σ^2:\n", 201 | "μ = 0.6062339614754363\n", 202 | "σ^2 = 0.6062339614754364\n", 203 | "Constrained MLE objective: -1753.235675254145\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "# Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors\n", 209 | "# This Source Code Form is subject to the terms of the Mozilla Public\n", 210 | "# License, v. 2.0. If a copy of the MPL was not distributed with this\n", 211 | "# file, You can obtain one at http://mozilla.org/MPL/2.0/.\n", 212 | "using JuMP, Ipopt\n", 213 | "\n", 214 | "# Use nonlinear optimization to compute the maximum likelihood estimate (MLE)\n", 215 | "# of the parameters of a normal distribution\n", 216 | "# aka the sample mean and variance\n", 217 | "\n", 218 | "n = 1000\n", 219 | "data = randn(n)\n", 220 | "\n", 221 | "m = Model(solver=IpoptSolver(print_level=0))\n", 222 | "\n", 223 | "@variable(m, μ, start = 0.0)\n", 224 | "@variable(m, σ >= 0.0, start = 1.0)\n", 225 | "\n", 226 | "@NLobjective(m, Max, (n/2)*log(1/(2π*σ^2))-sum((data[i]-μ)^2 for i=1:n)/(2σ^2))\n", 227 | "\n", 228 | "solve(m)\n", 229 | "\n", 230 | "println(\"μ = \", getvalue(μ))\n", 231 | "println(\"mean(data) = \", mean(data))\n", 232 | "println(\"σ^2 = \", getvalue(σ)^2)\n", 233 | "println(\"var(data) = \", var(data))\n", 234 | "println(\"MLE objective: \", getobjectivevalue(m))\n", 235 | "\n", 236 | "# constrained MLE?\n", 237 | "@NLconstraint(m, μ == σ^2)\n", 238 | "\n", 239 | "solve(m)\n", 240 | "println(\"\\nWith constraint μ == σ^2:\")\n", 241 | "println(\"μ = \", getvalue(μ))\n", 242 | "println(\"σ^2 = \", getvalue(σ)^2)\n", 243 | "\n", 244 | "println(\"Constrained MLE objective: \", getobjectivevalue(m))" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [] 253 | } 254 | ], 255 | "metadata": { 256 | "kernelspec": { 257 | "display_name": "Julia 0.6.2", 258 | "language": "julia", 259 | "name": "julia-0.6" 260 | }, 261 | "language_info": { 262 | "file_extension": ".jl", 263 | "mimetype": "application/julia", 264 | "name": "julia", 265 | "version": "0.6.2" 266 | }, 267 | "toc": { 268 | "colors": { 269 | "hover_highlight": "#DAA520", 270 | "running_highlight": "#FF0000", 271 | "selected_highlight": "#FFD700" 272 | }, 273 | "moveMenuLeft": true, 274 | "nav_menu": { 275 | "height": "137px", 276 | "width": "251px" 277 | }, 278 | "navigate_menu": true, 279 | "number_sections": true, 280 | "sideBar": true, 281 | "threshold": "2", 282 | "toc_cell": false, 283 | "toc_section_display": "block", 284 | "toc_window_display": false 285 | } 286 | }, 287 | "nbformat": 4, 288 | "nbformat_minor": 2 289 | } 290 | -------------------------------------------------------------------------------- /17. Distributed arrays.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "# Distributed arrays" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Parallel computing is a mess! There are many types of parallelism:\n", 17 | "\n", 18 | "- MPI\n", 19 | "- CUDA\n", 20 | "- OpenMP\n", 21 | "- Threads" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "Idea: `DistributedArrays` gives one of the easiest methods for parallel computing, when the parallelism is \"embarrassingly parallel\", i.e. we want different processes doing the same thing independently. It's an *easy* form of parallelism, when it works." 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "A **distributed array** is an array lives on several processors (or cores) -- each processor has a part of the array (a \"local part\"). The array is **partitioned** among the different processors.\n", 36 | "\n", 37 | "A `DistributedArray` will **look (to us) like a standard Julia array**. The partitioning among the processors should be **hidden**; i.e. it is an **abstraction**. (Hence \"AbstractArray\": something that *behaves like* an array.)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "To use on a cluster, use the `ClusterManagers.jl` package to control the adquisition of processors." 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "Pkg.add(\"DistributedArrays\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "using DistributedArrays # precompile on one process" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# Add processes:\n", 72 | "addprocs(4)\n", 73 | "\n", 74 | "# Load the package on each process:\n", 75 | "@everywhere using DistributedArrays " 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "procs() # one master process that controls and 4 subprocesses" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "@macroexpand @everywhere using DistributedArrays" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "rmprocs(2:5)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "workers()" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "addprocs()" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "workers()" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "rmprocs(workers())" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "workers()" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "addprocs(4)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "@everywhere using DistributedArrays" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "# Make some data:\n", 175 | "\n", 176 | "a = 1:10^3 \n", 177 | "\n", 178 | "b = a .^ 2" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "`a` and `b` are standard Julia arrays. Let's time how long it takes to sum them:" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "function bench_sum()\n", 195 | " a = rand(10^6)\n", 196 | " \n", 197 | " @time sum([t^2 for t in a])\n", 198 | " @time sum(t^2 for t in a)\n", 199 | "end" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "bench_sum()" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "## Pleasantly parallel (independent calculations)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "procs()" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "One \"master\" process and 4 \"workers\"." 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "a = rand(10^6); # standard Julia object" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "d = distribute(a)" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "The `distribute` command requires a transfer of data between procesess. Instead, we can create the DistributedArray directly on the different processes:" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "D = @DArray [i+j for i in 1:10, j in 1:10]" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "We can see which pieces of the array live on each worker:" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "D.indexes" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "We can create a distributed random matrix:" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": null, 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "drand(10, 10)" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": null, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "T = typeof(D)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "supertype(T)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "show(D) # requires data transfer" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "fieldnames(D)" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "We want to write *the same code* and have it *just work* on a DistributedArray:" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "metadata": {}, 347 | "outputs": [], 348 | "source": [ 349 | "[x^2 for x in D]" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "Note that the result is *not* a distributed array, so data transfer has happened." 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "@everywhere f(t) = t^2 # define a function on each worker\n", 366 | "\n", 367 | "f.(D)" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": null, 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [ 376 | "f(t) = t^2\n", 377 | "dD = f.(D) " 378 | ] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [ 386 | "@everywhere f(t) = t^2\n", 387 | "\n", 388 | "dD = map(f, D)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": null, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "# apply map to distributed vector (looks identical to non-distributed case)\n", 398 | "\n", 399 | "dD == map(t->t^2, D) # undistributes the array back onto the master node" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": null, 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "@fetchfrom 14 localpart(dD) # the result that worker 2 calculated" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": null, 414 | "metadata": {}, 415 | "outputs": [], 416 | "source": [ 417 | "import DistributedArrays.localpart\n", 418 | "localpart(dD::DArray, p::Integer) = @fetchfrom p localpart(dD)" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": null, 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "localpart(dD, 14)" 428 | ] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "Remember: NEVER do performance comparisons in global scope, always inside a function. " 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": null, 440 | "metadata": {}, 441 | "outputs": [], 442 | "source": [ 443 | "@everywhere begin\n", 444 | " using DistributedArrays\n", 445 | " using BenchmarkTools\n", 446 | "end" 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": null, 452 | "metadata": {}, 453 | "outputs": [], 454 | "source": [ 455 | "function compare_timings()\n", 456 | " \n", 457 | " # serial\n", 458 | " a = [rand(100, 100) for i in 1:500]\n", 459 | " display(@benchmark map(t->t^2, $a)) # put '$' inside @benchmark\n", 460 | " \n", 461 | " # parallel\n", 462 | " da = distribute(a)\n", 463 | " display(@benchmark map(t->t^2, $da))\n", 464 | "end\n", 465 | "\n", 466 | "compare_timings()" 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": null, 472 | "metadata": {}, 473 | "outputs": [], 474 | "source": [ 475 | "# Distributed vectors not restricted to numerical types\n", 476 | "\n", 477 | "map(t -> Dates.monthname((t - 1) % 12 + 1), D)" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "# A slightly more complicated example of map and reduce\n", 487 | "\n", 488 | "monthString = map(t -> Dates.monthname((t - 1) % 12 + 1) |> s -> s*\" is my favorite month.\\n\", D) |>\n", 489 | " t -> reduce(*, Array(t))\n", 490 | "println(monthString)" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": null, 496 | "metadata": {}, 497 | "outputs": [], 498 | "source": [ 499 | "# Distributed array comprehension\n", 500 | "\n", 501 | "D55 = @DArray [randn(5,5) for i = 1:32]" 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "execution_count": null, 507 | "metadata": {}, 508 | "outputs": [], 509 | "source": [ 510 | "# Compute eigenvalues of the distributed vector of matrices: \n", 511 | "\n", 512 | "Dsvd = map(eigvals, D55)" 513 | ] 514 | }, 515 | { 516 | "cell_type": "markdown", 517 | "metadata": { 518 | "collapsed": true 519 | }, 520 | "source": [ 521 | "**Exercise**: Check the performance: is it 4 times faster than on a single process?" 522 | ] 523 | } 524 | ], 525 | "metadata": { 526 | "anaconda-cloud": {}, 527 | "kernelspec": { 528 | "display_name": "Julia 0.5.2", 529 | "language": "julia", 530 | "name": "julia-0.5" 531 | }, 532 | "language_info": { 533 | "file_extension": ".jl", 534 | "mimetype": "application/julia", 535 | "name": "julia", 536 | "version": "0.5.2" 537 | }, 538 | "toc": { 539 | "colors": { 540 | "hover_highlight": "#DAA520", 541 | "running_highlight": "#FF0000", 542 | "selected_highlight": "#FFD700" 543 | }, 544 | "moveMenuLeft": true, 545 | "nav_menu": { 546 | "height": "30px", 547 | "width": "252px" 548 | }, 549 | "navigate_menu": true, 550 | "number_sections": true, 551 | "sideBar": true, 552 | "threshold": "2", 553 | "toc_cell": false, 554 | "toc_section_display": "block", 555 | "toc_window_display": false 556 | } 557 | }, 558 | "nbformat": 4, 559 | "nbformat_minor": 1 560 | } 561 | -------------------------------------------------------------------------------- /live/17. Distributed arrays.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "\n", 9 | "# Distributed arrays" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "Parallel computing is a mess! There are many types of parallelism:\n", 17 | "\n", 18 | "- MPI\n", 19 | "- CUDA\n", 20 | "- OpenMP\n", 21 | "- Threads" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "Idea: `DistributedArrays` gives one of the easiest methods for parallel computing, when the parallelism is \"embarrassingly parallel\", i.e. we want different processes doing the same thing independently. It's an *easy* form of parallelism, when it works." 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "A **distributed array** is an array lives on several processors (or cores) -- each processor has a part of the array (a \"local part\"). The array is **partitioned** among the different processors.\n", 36 | "\n", 37 | "A `DistributedArray` will **look (to us) like a standard Julia array**. The partitioning among the processors should be **hidden**; i.e. it is an **abstraction**. (Hence \"AbstractArray\": something that *behaves like* an array.)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "To use on a cluster, use the `ClusterManagers.jl` package to control the adquisition of processors." 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "Pkg.add(\"DistributedArrays\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "using DistributedArrays # precompile on one process" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "# Add processes:\n", 72 | "addprocs(4)\n", 73 | "\n", 74 | "# Load the package on each process:\n", 75 | "@everywhere using DistributedArrays " 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "procs() # one master process that controls and 4 subprocesses" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "@macroexpand @everywhere using DistributedArrays" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "rmprocs(2:5)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "workers()" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "addprocs()" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "workers()" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "rmprocs(workers())" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "workers()" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "addprocs(4)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "@everywhere using DistributedArrays" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "# Make some data:\n", 175 | "\n", 176 | "a = 1:10^3 \n", 177 | "\n", 178 | "b = a .^ 2" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "`a` and `b` are standard Julia arrays. Let's time how long it takes to sum them:" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "function bench_sum()\n", 195 | " a = rand(10^6)\n", 196 | " \n", 197 | " @time sum([t^2 for t in a])\n", 198 | " @time sum(t^2 for t in a)\n", 199 | "end" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "bench_sum()" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "## Pleasantly parallel (independent calculations)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "procs()" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "One \"master\" process and 4 \"workers\"." 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "a = rand(10^6); # standard Julia object" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "d = distribute(a)" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "The `distribute` command requires a transfer of data between procesess. Instead, we can create the DistributedArray directly on the different processes:" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "D = @DArray [i+j for i in 1:10, j in 1:10]" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "We can see which pieces of the array live on each worker:" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "D.indexes" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "We can create a distributed random matrix:" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": null, 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "drand(10, 10)" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": null, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "T = typeof(D)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "supertype(T)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "show(D) # requires data transfer" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "fieldnames(D)" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "We want to write *the same code* and have it *just work* on a DistributedArray:" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "metadata": {}, 347 | "outputs": [], 348 | "source": [ 349 | "[x^2 for x in D]" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "Note that the result is *not* a distributed array, so data transfer has happened." 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "@everywhere f(t) = t^2 # define a function on each worker\n", 366 | "\n", 367 | "f.(D)" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": null, 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [ 376 | "f(t) = t^2\n", 377 | "dD = f.(D) " 378 | ] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [ 386 | "@everywhere f(t) = t^2\n", 387 | "\n", 388 | "dD = map(f, D)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": null, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "# apply map to distributed vector (looks identical to non-distributed case)\n", 398 | "\n", 399 | "dD == map(t->t^2, D) # undistributes the array back onto the master node" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": null, 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "@fetchfrom 14 localpart(dD) # the result that worker 2 calculated" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": null, 414 | "metadata": {}, 415 | "outputs": [], 416 | "source": [ 417 | "import DistributedArrays.localpart\n", 418 | "localpart(dD::DArray, p::Integer) = @fetchfrom p localpart(dD)" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": null, 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "localpart(dD, 14)" 428 | ] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "Remember: NEVER do performance comparisons in global scope, always inside a function. " 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": null, 440 | "metadata": {}, 441 | "outputs": [], 442 | "source": [ 443 | "@everywhere begin\n", 444 | " using DistributedArrays\n", 445 | " using BenchmarkTools\n", 446 | "end" 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": null, 452 | "metadata": {}, 453 | "outputs": [], 454 | "source": [ 455 | "function compare_timings()\n", 456 | " \n", 457 | " # serial\n", 458 | " a = [rand(100, 100) for i in 1:500]\n", 459 | " display(@benchmark map(t->t^2, $a)) # put '$' inside @benchmark\n", 460 | " \n", 461 | " # parallel\n", 462 | " da = distribute(a)\n", 463 | " display(@benchmark map(t->t^2, $da))\n", 464 | "end\n", 465 | "\n", 466 | "compare_timings()" 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": null, 472 | "metadata": {}, 473 | "outputs": [], 474 | "source": [ 475 | "# Distributed vectors not restricted to numerical types\n", 476 | "\n", 477 | "map(t -> Dates.monthname((t - 1) % 12 + 1), D)" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "# A slightly more complicated example of map and reduce\n", 487 | "\n", 488 | "monthString = map(t -> Dates.monthname((t - 1) % 12 + 1) |> s -> s*\" is my favorite month.\\n\", D) |>\n", 489 | " t -> reduce(*, Array(t))\n", 490 | "println(monthString)" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": null, 496 | "metadata": {}, 497 | "outputs": [], 498 | "source": [ 499 | "# Distributed array comprehension\n", 500 | "\n", 501 | "D55 = @DArray [randn(5,5) for i = 1:32]" 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "execution_count": null, 507 | "metadata": {}, 508 | "outputs": [], 509 | "source": [ 510 | "# Compute eigenvalues of the distributed vector of matrices: \n", 511 | "\n", 512 | "Dsvd = map(eigvals, D55)" 513 | ] 514 | }, 515 | { 516 | "cell_type": "markdown", 517 | "metadata": { 518 | "collapsed": true 519 | }, 520 | "source": [ 521 | "**Exercise**: Check the performance: is it 4 times faster than on a single process?" 522 | ] 523 | } 524 | ], 525 | "metadata": { 526 | "anaconda-cloud": {}, 527 | "kernelspec": { 528 | "display_name": "Julia 0.5.2", 529 | "language": "julia", 530 | "name": "julia-0.5" 531 | }, 532 | "language_info": { 533 | "file_extension": ".jl", 534 | "mimetype": "application/julia", 535 | "name": "julia", 536 | "version": "0.5.2" 537 | }, 538 | "toc": { 539 | "colors": { 540 | "hover_highlight": "#DAA520", 541 | "running_highlight": "#FF0000", 542 | "selected_highlight": "#FFD700" 543 | }, 544 | "moveMenuLeft": true, 545 | "nav_menu": { 546 | "height": "30px", 547 | "width": "252px" 548 | }, 549 | "navigate_menu": true, 550 | "number_sections": true, 551 | "sideBar": true, 552 | "threshold": "2", 553 | "toc_cell": false, 554 | "toc_section_display": "block", 555 | "toc_window_display": false 556 | } 557 | }, 558 | "nbformat": 4, 559 | "nbformat_minor": 1 560 | } 561 | -------------------------------------------------------------------------------- /00. Introduction to Julia.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "slideshow": { 7 | "slide_type": "slide" 8 | } 9 | }, 10 | "source": [ 11 | "# Introduction to Julia" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": { 17 | "slideshow": { 18 | "slide_type": "-" 19 | } 20 | }, 21 | "source": [ 22 | "## David P. Sanders" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": { 28 | "slideshow": { 29 | "slide_type": "-" 30 | } 31 | }, 32 | "source": [ 33 | "\n", 34 | "### Department of Physics, Faculty of Sciences \n", 35 | "### Universidad Nacional Autónoma de México (UNAM)\n", 36 | "\n", 37 | "\n", 38 | "#### Lindner College of Business, University of Cincinnati, 22 & 23 March 2018" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": { 44 | "slideshow": { 45 | "slide_type": "slide" 46 | } 47 | }, 48 | "source": [ 49 | "# What is Julia?" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "slideshow": { 56 | "slide_type": "fragment" 57 | } 58 | }, 59 | "source": [ 60 | "- General-purpose programming language\n", 61 | "\n", 62 | "\n", 63 | "- Designed (from scratch) to be ideal for scientific computing\n", 64 | "\n", 65 | "\n", 66 | "- http://www.julialang.org\n" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": { 72 | "slideshow": { 73 | "slide_type": "subslide" 74 | } 75 | }, 76 | "source": [ 77 | "## Languages for scientific computing\n", 78 | "\n", 79 | "| Dynamic |                     | Static | \n", 80 | "|--- |---|-----|---|---|\n", 81 | "|Python, Matlab, R| | C, C++, Fortran |\n", 82 | "|Interactive| | |\n", 83 | "| | | Compile required |\n", 84 | "|Slow| |Fast\n" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": { 90 | "slideshow": { 91 | "slide_type": "subslide" 92 | } 93 | }, 94 | "source": [ 95 | "| Dynamic | **Type-inferred** | Static | \n", 96 | "|--- |---|-----|---|---|\n", 97 | "|Python, Matlab, R| **Julia** | C, C++, Fortran |\n", 98 | "|Interactive| **Interactive** | |\n", 99 | "| | **AoT compiled** | Compile required\n", 100 | "|Slow| **Fast** |Fast" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": { 106 | "slideshow": { 107 | "slide_type": "fragment" 108 | } 109 | }, 110 | "source": [ 111 | "AoT = Ahead of Time\n", 112 | "\n", 113 | "JIT = Just in Time\n", 114 | "\n", 115 | "[Why we created Julia](https://julialang.org/blog/2012/02/why-we-created-julia): **\"Greedy developers\"**" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "slideshow": { 122 | "slide_type": "subslide" 123 | } 124 | }, 125 | "source": [ 126 | "### Julia is unique" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": { 132 | "slideshow": { 133 | "slide_type": "-" 134 | } 135 | }, 136 | "source": [ 137 | "Julia allows us to:" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": { 143 | "slideshow": { 144 | "slide_type": "fragment" 145 | } 146 | }, 147 | "source": [ 148 | "- Write readable code interactively (or not)" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": { 154 | "slideshow": { 155 | "slide_type": "fragment" 156 | } 157 | }, 158 | "source": [ 159 | "- That is generic\n", 160 | "\n", 161 | " - works with objects of different types\n", 162 | " \n", 163 | " - *including* user-defined types" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": { 169 | "slideshow": { 170 | "slide_type": "fragment" 171 | } 172 | }, 173 | "source": [ 174 | "- But is **fast**" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": { 180 | "slideshow": { 181 | "slide_type": "skip" 182 | } 183 | }, 184 | "source": [ 185 | "For an interesting discussion of what makes Julia unique compared to the various attempts to accelerate Python etc., see\n", 186 | "https://discourse.julialang.org/t/julia-motivation-why-werent-numpy-scipy-numba-good-enough/" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": { 192 | "slideshow": { 193 | "slide_type": "subslide" 194 | } 195 | }, 196 | "source": [ 197 | "### Development of Julia" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": { 203 | "slideshow": { 204 | "slide_type": "fragment" 205 | } 206 | }, 207 | "source": [ 208 | "- Free software; MIT license\n", 209 | "\n", 210 | "- Available on all platforms" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": { 216 | "slideshow": { 217 | "slide_type": "subslide" 218 | } 219 | }, 220 | "source": [ 221 | "- Worldwide developer community\n", 222 | "\n", 223 | "\n", 224 | "- Open, online development: https://github.com/JuliaLang/julia\n", 225 | "\n", 226 | "\n", 227 | "- 680 contributors to the core language" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": { 233 | "slideshow": { 234 | "slide_type": "fragment" 235 | } 236 | }, 237 | "source": [ 238 | "- Extensible: 1750 registered packages, high **composability**" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "metadata": { 244 | "slideshow": { 245 | "slide_type": "subslide" 246 | } 247 | }, 248 | "source": [ 249 | "### Interopability\n", 250 | "\n", 251 | "- Easy to call libraries from C & Fortran (`ccall`)\n", 252 | "\n", 253 | "\n", 254 | "- Python (`PyCall.jl`), R (`RCall.jl`), ...\n", 255 | "\n", 256 | "- Interact with C++ (`Cxx.jl`)\n" 257 | ] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "metadata": { 262 | "slideshow": { 263 | "slide_type": "fragment" 264 | } 265 | }, 266 | "source": [ 267 | "- Julia is both a very high-**and** low-level language" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": { 273 | "slideshow": { 274 | "slide_type": "fragment" 275 | } 276 | }, 277 | "source": [ 278 | "#### Solves the 2-language problem" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": { 284 | "slideshow": { 285 | "slide_type": "fragment" 286 | } 287 | }, 288 | "source": [ 289 | "#### Or the N-tool problem: \n", 290 | "Python, bash, C, gnuplot, R, Matlab, Mathematica..." 291 | ] 292 | }, 293 | { 294 | "cell_type": "markdown", 295 | "metadata": { 296 | "slideshow": { 297 | "slide_type": "slide" 298 | } 299 | }, 300 | "source": [ 301 | "## Open science" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "metadata": { 307 | "slideshow": { 308 | "slide_type": "fragment" 309 | } 310 | }, 311 | "source": [ 312 | "- Science requires **free / libre open-source software** (FLOSS)\n", 313 | "\n", 314 | "\n", 315 | "\n", 316 | "- **All** scientific code must be freely available and reproducible \n", 317 | "\n", 318 | "\n", 319 | "\n", 320 | "- Choice of software license is important\n" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": { 326 | "slideshow": { 327 | "slide_type": "subslide" 328 | } 329 | }, 330 | "source": [ 331 | "## Basic language is (almost) stable\n", 332 | "\n", 333 | "- Today: **v0.6(.2)**\n", 334 | "\n", 335 | "- Tomorrow: **v0.7 = v1.0** - first long-term stable release" 336 | ] 337 | }, 338 | { 339 | "cell_type": "markdown", 340 | "metadata": { 341 | "slideshow": { 342 | "slide_type": "slide" 343 | } 344 | }, 345 | "source": [ 346 | "# How to use Julia" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": { 352 | "slideshow": { 353 | "slide_type": "fragment" 354 | } 355 | }, 356 | "source": [ 357 | "- Jupyter Notebook: literate, interactive computational documents\n", 358 | "\n", 359 | " - online at [JuliaBox.com](http://juliabox.com)\n", 360 | " \n", 361 | " - locally\n", 362 | " " 363 | ] 364 | }, 365 | { 366 | "cell_type": "markdown", 367 | "metadata": { 368 | "slideshow": { 369 | "slide_type": "fragment" 370 | } 371 | }, 372 | "source": [ 373 | "- REPL (command line): Interactive workflow" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": { 379 | "slideshow": { 380 | "slide_type": "fragment" 381 | } 382 | }, 383 | "source": [ 384 | "- [Juno](http://junolab.org/): Julia IDE in Atom editor (inline evaluation)" 385 | ] 386 | }, 387 | { 388 | "cell_type": "markdown", 389 | "metadata": { 390 | "slideshow": { 391 | "slide_type": "subslide" 392 | } 393 | }, 394 | "source": [ 395 | "- [JuliaPro](https://juliacomputing.com/products/juliapro.html): Bundled version from JuliaComputing; pro support" 396 | ] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "metadata": { 401 | "slideshow": { 402 | "slide_type": "fragment" 403 | } 404 | }, 405 | "source": [ 406 | "- [Nteract](nteract.io): Desktop app for Jupyter notebook format" 407 | ] 408 | }, 409 | { 410 | "cell_type": "markdown", 411 | "metadata": { 412 | "slideshow": { 413 | "slide_type": "subslide" 414 | } 415 | }, 416 | "source": [ 417 | "## Installation of Jupyter notebook:\n", 418 | "\n", 419 | "- From REPL:\n", 420 | "\n", 421 | "```\n", 422 | "julia> Pkg.add(\"IJulia\")\n", 423 | "\n", 424 | "julia> using IJulia\n", 425 | "julia> notebook()\n", 426 | "``` \n", 427 | "\n", 428 | "\n", 429 | "- NB: `pwd()` gives current directory\n", 430 | "\n", 431 | "\n", 432 | "- Installs own version of Anaconda\n" 433 | ] 434 | }, 435 | { 436 | "cell_type": "markdown", 437 | "metadata": { 438 | "slideshow": { 439 | "slide_type": "subslide" 440 | } 441 | }, 442 | "source": [ 443 | "## Getting help\n" 444 | ] 445 | }, 446 | { 447 | "cell_type": "markdown", 448 | "metadata": { 449 | "slideshow": { 450 | "slide_type": "fragment" 451 | } 452 | }, 453 | "source": [ 454 | "- Interactively (REPL / Juno / IJulia): `?sin`" 455 | ] 456 | }, 457 | { 458 | "cell_type": "markdown", 459 | "metadata": { 460 | "slideshow": { 461 | "slide_type": "fragment" 462 | } 463 | }, 464 | "source": [ 465 | "- Reference manual: https://docs.julialang.org/en/stable\n", 466 | "\n", 467 | "\n", 468 | "- Discourse discussion forum: https://discourse.julialang.org/\n", 469 | "\n", 470 | "\n", 471 | "- Slack: http://slackinvite.julialang.org\n", 472 | "\n", 473 | "\n", 474 | "- Learning resources: http://julialang.org/learning\n", 475 | "\n", 476 | "\n", 477 | "- Packages etc.: https://juliaobserver.com/" 478 | ] 479 | }, 480 | { 481 | "cell_type": "markdown", 482 | "metadata": { 483 | "slideshow": { 484 | "slide_type": "slide" 485 | } 486 | }, 487 | "source": [ 488 | "# Goal of the workshop" 489 | ] 490 | }, 491 | { 492 | "cell_type": "markdown", 493 | "metadata": { 494 | "slideshow": { 495 | "slide_type": "fragment" 496 | } 497 | }, 498 | "source": [ 499 | "- Why should I spend my valuable time learning Julia?" 500 | ] 501 | }, 502 | { 503 | "cell_type": "markdown", 504 | "metadata": { 505 | "slideshow": { 506 | "slide_type": "fragment" 507 | } 508 | }, 509 | "source": [ 510 | "- Accelerated start" 511 | ] 512 | }, 513 | { 514 | "cell_type": "markdown", 515 | "metadata": { 516 | "slideshow": { 517 | "slide_type": "fragment" 518 | } 519 | }, 520 | "source": [ 521 | "- Julian workflow" 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "metadata": { 527 | "slideshow": { 528 | "slide_type": "subslide" 529 | } 530 | }, 531 | "source": [ 532 | "- Aspects of modern Julia (0.6 to 1.0)" 533 | ] 534 | }, 535 | { 536 | "cell_type": "markdown", 537 | "metadata": { 538 | "slideshow": { 539 | "slide_type": "fragment" 540 | } 541 | }, 542 | "source": [ 543 | "- Guide to the ecosystem: Key packages" 544 | ] 545 | }, 546 | { 547 | "cell_type": "markdown", 548 | "metadata": { 549 | "slideshow": { 550 | "slide_type": "fragment" 551 | } 552 | }, 553 | "source": [ 554 | "- Performance" 555 | ] 556 | } 557 | ], 558 | "metadata": { 559 | "celltoolbar": "Slideshow", 560 | "kernelspec": { 561 | "display_name": "Julia 0.6.2", 562 | "language": "julia", 563 | "name": "julia-0.6" 564 | }, 565 | "language_info": { 566 | "file_extension": ".jl", 567 | "mimetype": "application/julia", 568 | "name": "julia", 569 | "version": "0.6.2" 570 | }, 571 | "toc": { 572 | "colors": { 573 | "hover_highlight": "#DAA520", 574 | "running_highlight": "#FF0000", 575 | "selected_highlight": "#FFD700" 576 | }, 577 | "moveMenuLeft": true, 578 | "nav_menu": { 579 | "height": "139px", 580 | "width": "252px" 581 | }, 582 | "navigate_menu": true, 583 | "number_sections": true, 584 | "sideBar": true, 585 | "threshold": "2", 586 | "toc_cell": false, 587 | "toc_section_display": "block", 588 | "toc_window_display": false 589 | } 590 | }, 591 | "nbformat": 4, 592 | "nbformat_minor": 2 593 | } 594 | -------------------------------------------------------------------------------- /10. Memory layout.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Use immutables for efficiency" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": { 13 | "collapsed": true 14 | }, 15 | "source": [ 16 | "Memory hierarchy crash course:\n", 17 | "\n", 18 | "1. When CPU runs an instruction it operates on things in the registers. There are very few of these in a computer, these are the %1s and %2s when you run @code_llvm. If something is not in the CPU registers, the CPU needs to fetch the data from memory; this is **slow**.\n", 19 | "\n", 20 | "2. The CPU first looks at L1 cache, then L2 cache then Main memory and then Swap space - L1, L2 caches are still small (order of megabytes), but hitting them often will give orders of magnitude performance gain as compared to hitting main memory that often\n", 21 | "\n", 22 | "3. The computer optimistically brings things from main memory into the caches when you access a chunk of data. Hence if you access data that is contiguous in memory, they all get asynchronously brought into the cache and your program will be really fast." 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Memory layout of an array of `mutable struct` objects" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "abstract type TestType end" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "mutable struct Typ <: TestType\n", 48 | " x::Int16\n", 49 | " y::Int16\n", 50 | "end" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "Base.:+{T<:TestType}(a::T, b::T) = T(a.x+b.x, a.y+b.y)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": {}, 66 | "outputs": [], 67 | "source": [ 68 | "Typ(2,2) + Typ(4,4)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "The size of `Typ` is 4 bytes:" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "sizeof(Typ(2,2))" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "@time typ_arr = [Typ(i%127,i%127) for i=1:10^6];" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "Notice the allocation. Also, this array is 2x bigger than it should be:" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "sizeof(typ_arr)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "sizeof(typ_arr) / 10^6 # bytes per object" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "This is because mutable objects are **passed by reference**; the objects are being \"boxed\"." 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "This is to make the following possible:" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "function someone_else_doing_something_else(a::Typ)\n", 142 | " a.x = 42\n", 143 | "end\n", 144 | "\n", 145 | "someone_else_doing_something_else(typ_arr[3])\n", 146 | "typ_arr[3]" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "Sum could also have been much more efficient...." 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "@time sum(typ_arr)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "## Memory layout of an array of (immutable) `struct`s" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "struct Imm <: TestType\n", 179 | " x::Int16\n", 180 | " y::Int16\n", 181 | "end" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "sizeof(Imm(2,2))" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "@time imm_arr = [Imm(i%127, i%127) for i=1:10^6];" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "sizeof(imm_arr)" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "This seems correct! Since immutables can never be changed, their value _is_ their identity, and the compiler can **pass them by value**. [But it does not necessarily.]" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "Base.:+(a::Imm, b::Imm) = Imm(a.x+b.x, a.y+b.y)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "@time sum(typ_arr)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "The allocation is the same as adding Float *values*" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "x = rand(10^6)\n", 250 | "\n", 251 | "@time sum(x)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "The compiler can do this optimization because it knows someone else won't be changing the insides of the `Imm` object:" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "function someone_else_doing_something_else(a::Imm)\n", 268 | " a.x = 42 # This is not allowed!!\n", 269 | "end\n", 270 | "\n", 271 | "someone_else_doing_something_else(imm_arr[3])" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": {}, 277 | "source": [ 278 | "If you don't know the type of the insides of an immutable type, you can add a type parameter; e.g." 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "immutable ImmParam{T} <: TestType\n", 288 | " x::T\n", 289 | " y::T\n", 290 | "end" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "sizeof(ImmParam{Int128}) # sizeof also works on the " 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": null, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "sizeof(ImmParam{Int8})" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "metadata": {}, 315 | "outputs": [], 316 | "source": [ 317 | "ImmParam{Int8} == ImmParam{Int64}" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": null, 323 | "metadata": {}, 324 | "outputs": [], 325 | "source": [ 326 | "ImmParam(1.0, 2.0) # Julia automatically infers this" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "ImmParam(1,2)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [ 344 | "ImmParam(1.0,2)" 345 | ] 346 | }, 347 | { 348 | "cell_type": "markdown", 349 | "metadata": {}, 350 | "source": [ 351 | "### And it is aligned tightly!" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "@time imm_par_array_int16 = [ImmParam{Int16}(2,3) for i = 1:10^6];" 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "metadata": {}, 367 | "outputs": [], 368 | "source": [ 369 | "sizeof(imm_par_array_int16)" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": null, 375 | "metadata": {}, 376 | "outputs": [], 377 | "source": [ 378 | "@time imm_par_array_int8 = [ImmParam{Int8}(2,3) for i = 1:10^6];" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": null, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [ 387 | "sizeof(imm_par_array_int8)" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "metadata": {}, 394 | "outputs": [], 395 | "source": [ 396 | "@time imm_par_array_cplx = [ImmParam(2+3im,3+2im) for i = 1:10^6];" 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": null, 402 | "metadata": {}, 403 | "outputs": [], 404 | "source": [ 405 | "sizeof(imm_par_array_cplx)" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": null, 411 | "metadata": {}, 412 | "outputs": [], 413 | "source": [ 414 | "Base.:+(a::ImmParam, b::ImmParam) = ImmParam(a.x+b.x, a.y+b.y)" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "@time sum(imm_par_array_cplx)" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": null, 429 | "metadata": {}, 430 | "outputs": [], 431 | "source": [ 432 | "using Interact" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": null, 438 | "metadata": {}, 439 | "outputs": [], 440 | "source": [ 441 | "type TypParam{T} <: TestType\n", 442 | " x::T\n", 443 | " y::T\n", 444 | "end" 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": null, 450 | "metadata": {}, 451 | "outputs": [], 452 | "source": [ 453 | "@manipulate for param = [Int8,Int16,Int32,Int64,Float16,Float32,Float64], complex=true\n", 454 | " T = complex ? Complex{param} : param\n", 455 | " a = zero(T)\n", 456 | " b = one(T)\n", 457 | "\n", 458 | " gc() \n", 459 | " local arr,t_create,arr_t,t_create_t,t_sum,t_sum_t\n", 460 | "\n", 461 | " alloc_create = @allocated begin\n", 462 | " t_create = @elapsed begin\n", 463 | " arr = [ImmParam(a,b) for i=1:10^6]\n", 464 | " end\n", 465 | " end\n", 466 | "\n", 467 | " alloc_create_t = @allocated begin\n", 468 | " t_create_t = @elapsed begin\n", 469 | " arr_t = [TypParam(a,b) for i=1:10^6]\n", 470 | " end\n", 471 | " end\n", 472 | "\n", 473 | " gc()\n", 474 | " \n", 475 | " alloc_sum = @allocated begin\n", 476 | " t_sum = @elapsed begin\n", 477 | " s = sum(arr)\n", 478 | " end\n", 479 | " end/10^6\n", 480 | "\n", 481 | " alloc_sum_t = @allocated begin\n", 482 | " t_sum_t = @elapsed begin\n", 483 | " s_t = sum(arr_t)\n", 484 | " end\n", 485 | " end/10^6\n", 486 | " HTML(\"\n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | "
mutableimmutable
createsumcreatesum
timememorytimememorytimememorytimememory
$t_create$alloc_create$t_sum$alloc_sum$t_create_t$alloc_create_t$t_sum_t$alloc_sum_t
\")\n", 520 | "end" 521 | ] 522 | }, 523 | { 524 | "cell_type": "markdown", 525 | "metadata": {}, 526 | "source": [ 527 | "## But be careful: vectors of heterogeneous types force boxing!" 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": null, 533 | "metadata": {}, 534 | "outputs": [], 535 | "source": [ 536 | "[\"xyzabc\", 1+2im, 1, 1.0]" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "metadata": {}, 543 | "outputs": [], 544 | "source": [ 545 | "[ImmParam(UInt8(1),UInt8(1)), ImmParam(1.0,1.0)] |> sizeof" 546 | ] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "execution_count": null, 551 | "metadata": {}, 552 | "outputs": [], 553 | "source": [ 554 | "@time heter_arr = [i%2 == 0 ? ImmParam(UInt8(1),UInt8(1)) : ImmParam(1.0,1.0) for i = 1:10^6]" 555 | ] 556 | }, 557 | { 558 | "cell_type": "code", 559 | "execution_count": null, 560 | "metadata": {}, 561 | "outputs": [], 562 | "source": [ 563 | "@time sum(heter_arr)" 564 | ] 565 | }, 566 | { 567 | "cell_type": "markdown", 568 | "metadata": {}, 569 | "source": [ 570 | "## Summary\n", 571 | "\n", 572 | "- Use immutables wherever you consider something to be a *value*. Use type when something is a *state*.\n", 573 | "- Never create a large array of mutable objects! Each one is heap-allocated, this kills performance and gives the GC a hard time.\n", 574 | "- Parameterize if you need to change types\n", 575 | "- In the wizard's own words: http://julialang.org/blog/2013/03/efficient-aggregates" 576 | ] 577 | }, 578 | { 579 | "cell_type": "markdown", 580 | "metadata": {}, 581 | "source": [ 582 | "## New packages that eliminate some of the drudgery" 583 | ] 584 | }, 585 | { 586 | "cell_type": "markdown", 587 | "metadata": {}, 588 | "source": [ 589 | "There are now packages available which allow us to get round some of these problems, e.g. https://github.com/tkoolen/TypeSortedCollections.jl\n", 590 | "\n", 591 | "This automatically splits up a heterogeneous collection into a sequence of homogeneous collections, and provides a nice interface so that we do not have to the book-keeping ourselves." 592 | ] 593 | } 594 | ], 595 | "metadata": { 596 | "anaconda-cloud": {}, 597 | "kernelspec": { 598 | "display_name": "Julia 0.6.2", 599 | "language": "julia", 600 | "name": "julia-0.6" 601 | }, 602 | "language_info": { 603 | "file_extension": ".jl", 604 | "mimetype": "application/julia", 605 | "name": "julia", 606 | "version": "0.6.2" 607 | }, 608 | "toc": { 609 | "colors": { 610 | "hover_highlight": "#DAA520", 611 | "running_highlight": "#FF0000", 612 | "selected_highlight": "#FFD700" 613 | }, 614 | "moveMenuLeft": true, 615 | "nav_menu": { 616 | "height": "121px", 617 | "width": "252px" 618 | }, 619 | "navigate_menu": true, 620 | "number_sections": true, 621 | "sideBar": true, 622 | "threshold": "2", 623 | "toc_cell": false, 624 | "toc_section_display": "block", 625 | "toc_window_display": false 626 | } 627 | }, 628 | "nbformat": 4, 629 | "nbformat_minor": 2 630 | } 631 | -------------------------------------------------------------------------------- /01. Example - random walks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Features of Julia: Random walk" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "An important model in science (biology, chemistry, physics, etc.) is Brownian motion, or random walk, a model of a molecule moving in a medium.\n", 15 | "\n", 16 | "As a first way to explore Julia and some of its features, we'll code the simplest version: a particle jumping in 1D on the integers. " 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "**Exercises**:\n", 24 | "\n", 25 | "**[1]** How does the `rand` function work? What options does it have?" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "rand" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "**[2]** (i) Write a simple 1D random walk for a particle that starts at the origin and only visits the integers, jumping with probability $\\frac{1}{2}$ one step to the left and $\\frac{1}{2}$ one step to the right, in \"global scope\" (as you would in Matlab or Python).\n", 42 | "\n", 43 | "(ii) Time it for 10^7 steps with\n", 44 | "\n", 45 | "```@time begin \n", 46 | " ... \n", 47 | " end\n", 48 | "```" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "**[3]** (i) Wrap the code in a function `walk`; it is easy now to give the function an argument which is the final time (to allow for code re-use). The function should return the final position.\n", 63 | "\n", 64 | "(ii) But this is not the only reason for using functions in Julia: now time running the function, and time it again. What do you notice?\n", 65 | "\n", 66 | "(iii) Give the function a **docstring**, i.e. a string written just *before* the function definition that describes what the function does. You can use multi-line strings using `\"\"\"`." 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "**[4]** Make a version of the function that stores the trajectory in a vector. In fact, there are (at least) two ways we could do so: pre-allocating the vector, or allowing it to grow. How do these compare in time?" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "**[5]** We could make a version that takes as second argument a flag, which tells us whether to calculate the trajectory or not. Does this affect the time?" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "## Visualization" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "We would like to plot the trajectory $x(n)$ as a function of the time step $n$. \n", 116 | "\n", 117 | "The `Plots.jl` meta-package provides a common syntax to several different plotting package \"backends\"." 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "# Pkg.add(\"Plots\") # install the Plots.jl package if you don't already have it -- do once in your installation\n", 127 | "\n", 128 | "using Plots # load the package in each session\n", 129 | "gr() # choose the GR backend" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "**[1]** Plot a single trajectory of the walker $x(n)$ as a function of $n$ using lines and squares. [The `plot` function takes a collection of $x$ coordinates and a collection of $y$ coordinates.] Add a horizontal dashed line at $x=0$." 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "**[2]** Switch to the PlotlyJS backend using `plotlyjs()` and plot it again. What advantage does this backend have?" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": {}, 163 | "source": [ 164 | "**[3]** Draw a few walks using a loop." 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "metadata": {}, 170 | "source": [ 171 | "### Interactive manipulation" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "If we want to explore the space of parameters `T` (maximum time of the walk) and `N` (number of walks), it would be nice to be able to manipulate them interactively.\n", 179 | "\n", 180 | "This may be done using `Interact.jl`." 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "# Pkg.add(\"Interact\")\n", 190 | "\n", 191 | "using Interact" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": {}, 197 | "source": [ 198 | "**[4]** (i) `Interact.jl` provides a macro `@manipulate` that is written directly before a `for` loop.\n", 199 | "Experiment with it:" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "@manipulate for i in 1:10\n", 209 | " i\n", 210 | "end" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "Use it to manipulate the number of walks.\n", 218 | "The `for` loop should return the object that will be displayed, in this case the result of the `plot`.\n", 219 | "\n", 220 | "(ii) Now use it to manipulate a double `for` loop, with the syntax\n", 221 | "\n", 222 | "```\n", 223 | "@manipulate for x in 1:X, y in 1:Y\n", 224 | " ...\n", 225 | "end\n", 226 | "```\n", 227 | "\n", 228 | "You will want to remove the plot legend/key with `leg=false`, and to fix the limits of the plot with keywords `xlim` and `ylim` or commands `xlims!` and `ylims!`." 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "## Collecting data" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "Now that we have a feel for the behaviour of the system, let's collect some data in a few different ways, in order to calculate the variance of the positions of `N` walks after time `T`." 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "**[1]** (i) Write a function `walks` that calculates the variance of the final positions of `N` walks of length `T` each.\n", 250 | "\n", 251 | "(ii) Use `@time` to time the function for small `N` and `T`. Note that it is not very useful." 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "#### BenchmarkTools.jl" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "**[2]** Instead, use the `BenchmarkTools.jl` package and the `@benchmark` or `@btime` macros." 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": { 271 | "slideshow": { 272 | "slide_type": "subslide" 273 | } 274 | }, 275 | "source": [ 276 | "### Investigating different coding styles" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "**[3]** First write a **generic** `walks` function that **takes a function as one of its arguments**." 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [] 292 | }, 293 | { 294 | "cell_type": "markdown", 295 | "metadata": {}, 296 | "source": [ 297 | "**[4]** Let's use the `walks` function to compare the efficiency of different coding styles for our `walk` function:\n", 298 | "\n", 299 | "(i) One that uses `rand(Bool)`.\n", 300 | " \n", 301 | "(ii) One that uses Matlab(ish) vectorised syntax, using a vector of random numbers.\n", 302 | "\n", 303 | "(iii) One using **array comprehensions**, i.e. expressions of the form\n", 304 | "\n", 305 | "```\n", 306 | "[x for i in 1:10]\n", 307 | "```\n", 308 | "\n", 309 | "(iv) One using **generator expressions**, i.e. expressions of the form\n", 310 | " \n", 311 | "```\n", 312 | "(x for i in 1:10)\n", 313 | "``` \n", 314 | " \n", 315 | "What is the difference?" 316 | ] 317 | }, 318 | { 319 | "cell_type": "markdown", 320 | "metadata": {}, 321 | "source": [ 322 | "**[5]** How could you generate the random $-1$ and $+1$ directly, using a range? Is it a good idea?" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "**[6]** Use the function `bitrand`. How does it compare?" 330 | ] 331 | }, 332 | { 333 | "cell_type": "markdown", 334 | "metadata": {}, 335 | "source": [ 336 | "Of course, it's (presumably) *possible* to do the above in C++, but it's certainly much *harder*. And in Python it will be much *slower*." 337 | ] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": {}, 342 | "source": [ 343 | "**[7]** **Bonus question** Compare the above timings to your (former) favourite programming language. There are C and Python versions in the `random_walk_in_other_languages` directory. I will add any implementations you send me there!" 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": { 349 | "slideshow": { 350 | "slide_type": "subslide" 351 | } 352 | }, 353 | "source": [ 354 | "## Data analysis" 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "**[1]** Write a function to collect the final positions of `N` walks after time `T` in an array.\n", 362 | "\n", 363 | "**[2]** Make an interactive visualization of a normalised histogram using the `histogram` function in `Plots.jl`. " 364 | ] 365 | }, 366 | { 367 | "cell_type": "markdown", 368 | "metadata": {}, 369 | "source": [ 370 | "[3] Let's compare this to the known analytical result: a normal distribution with variance equal to the number of steps.\n", 371 | "\n", 372 | "(i) Define the mean $\\mu$ and standard deviation $\\sigma$ using Unicode symbols, as `\\mu` and `\\sigma`.\n", 373 | "\n", 374 | "(ii) Load the `Distributions.jl` package.\n", 375 | "\n", 376 | "(iii) Compare the histogram with a transparent, filled PDF. Use the `pdf` function from `Distributions.jl`, and the `Normal` normal distribution object.\n", 377 | "\n", 378 | "(iv) Save the figure to a PDF using `savefig`." 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "## Write and read data to and from disk" 386 | ] 387 | }, 388 | { 389 | "cell_type": "markdown", 390 | "metadata": {}, 391 | "source": [ 392 | "We wish to save the data we have calculated to a file on disk. The simplest solution is as follows.\n", 393 | "\n", 394 | "**[1]** (i) Use `writedlm` (\"write delimited\") to save the data.\n", 395 | "\n", 396 | "(ii) Use `readdlm` to load it back in." 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "Note that the format has changed. Although there are some options to `readdlm`, this is not a good solution to the problem." 404 | ] 405 | }, 406 | { 407 | "cell_type": "markdown", 408 | "metadata": {}, 409 | "source": [ 410 | "The good solution is to use the `JLD2.jl` (\"JuLia Data\") package, which uses the standard HDF5 file format underneath. We will also save another variable for illustrative purposes." 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "# Pkg.add(\"JLD2\")\n", 420 | "\n", 421 | "# install \n", 422 | "using JLD2" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": null, 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [ 431 | "i = 17" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": null, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "@save \"random_walk.jld\" i" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": null, 446 | "metadata": {}, 447 | "outputs": [], 448 | "source": [ 449 | "@load \"random_walk.jld\"" 450 | ] 451 | }, 452 | { 453 | "cell_type": "code", 454 | "execution_count": null, 455 | "metadata": {}, 456 | "outputs": [], 457 | "source": [ 458 | "# save(\"random_walk.jld\", \"final_pos\", data, \"arbitrary\", i) # alternative" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": null, 464 | "metadata": {}, 465 | "outputs": [], 466 | "source": [ 467 | "# all = load(\"random_walk.jld\")" 468 | ] 469 | }, 470 | { 471 | "cell_type": "markdown", 472 | "metadata": {}, 473 | "source": [ 474 | "The data is read in as a **dictionary**, associating to each variable name its value. We can extract these as" 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": null, 480 | "metadata": {}, 481 | "outputs": [], 482 | "source": [ 483 | "j = all[\"arbitrary\"]" 484 | ] 485 | }, 486 | { 487 | "cell_type": "markdown", 488 | "metadata": {}, 489 | "source": [ 490 | "Alternatively, we can *read only part of the data file*:" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": null, 496 | "metadata": {}, 497 | "outputs": [], 498 | "source": [ 499 | "final_pos = load(\"random_walk.jld\", \"final_pos\");" 500 | ] 501 | }, 502 | { 503 | "cell_type": "markdown", 504 | "metadata": {}, 505 | "source": [ 506 | "Note that the type of the data has now been correctly preserved. This is the point of using `JLD`. " 507 | ] 508 | }, 509 | { 510 | "cell_type": "markdown", 511 | "metadata": {}, 512 | "source": [ 513 | "# Workflow: Extract code into a Julia script" 514 | ] 515 | }, 516 | { 517 | "cell_type": "markdown", 518 | "metadata": {}, 519 | "source": [ 520 | "Since we have developed a quantity of code in this notebook, it is a good moment to extract the code into a Julia script file, with termination `.jl`, that we edit, e.g. with Juno.\n", 521 | "\n", 522 | "We can then read in the file as if we typed it with" 523 | ] 524 | }, 525 | { 526 | "cell_type": "code", 527 | "execution_count": null, 528 | "metadata": {}, 529 | "outputs": [], 530 | "source": [ 531 | "include(\"walk.jl\")" 532 | ] 533 | }, 534 | { 535 | "cell_type": "markdown", 536 | "metadata": {}, 537 | "source": [ 538 | "Note that Julia provides various commands for manipulating the filesystem:" 539 | ] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": null, 544 | "metadata": {}, 545 | "outputs": [], 546 | "source": [ 547 | "pwd()" 548 | ] 549 | }, 550 | { 551 | "cell_type": "code", 552 | "execution_count": null, 553 | "metadata": {}, 554 | "outputs": [], 555 | "source": [ 556 | "readdir()" 557 | ] 558 | }, 559 | { 560 | "cell_type": "markdown", 561 | "metadata": {}, 562 | "source": [ 563 | "Later on, we would make it into a `module` (separate namespace) and then into a Julia pacakge." 564 | ] 565 | }, 566 | { 567 | "cell_type": "markdown", 568 | "metadata": { 569 | "slideshow": { 570 | "slide_type": "subslide" 571 | } 572 | }, 573 | "source": [ 574 | "## Summary:\n", 575 | "\n", 576 | "- Julia allows us to calculate **fast**\n", 577 | "\n", 578 | "\n", 579 | "- **And** visualize, interactively, our data\n", 580 | "\n", 581 | "\n", 582 | "- All in one place" 583 | ] 584 | } 585 | ], 586 | "metadata": { 587 | "kernelspec": { 588 | "display_name": "Julia 0.6.2", 589 | "language": "julia", 590 | "name": "julia-0.6" 591 | }, 592 | "language_info": { 593 | "file_extension": ".jl", 594 | "mimetype": "application/julia", 595 | "name": "julia", 596 | "version": "0.6.2" 597 | }, 598 | "toc": { 599 | "colors": { 600 | "hover_highlight": "#DAA520", 601 | "running_highlight": "#FF0000", 602 | "selected_highlight": "#FFD700" 603 | }, 604 | "moveMenuLeft": true, 605 | "nav_menu": { 606 | "height": "12px", 607 | "width": "252px" 608 | }, 609 | "navigate_menu": true, 610 | "number_sections": true, 611 | "sideBar": true, 612 | "threshold": "2", 613 | "toc_cell": false, 614 | "toc_section_display": "block", 615 | "toc_window_display": false 616 | } 617 | }, 618 | "nbformat": 4, 619 | "nbformat_minor": 2 620 | } 621 | -------------------------------------------------------------------------------- /16. Domain-specific languages in Julia - Generating callable objects using macros.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Domain-specific languages in Julia: Generating callable objects using macros" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Since the beginning of Julia, it has been tempting to use macros to write **domain-specific languages** (DSLs), i.e. to *extend* Julia syntax to provide a simpler interface to create Julia objects with complicated behaviour. The first, and still most extensive, example is [JuMP](https://github.com/JuliaOpt/JuMP.jl).\n", 15 | "\n", 16 | "In this post, we will describe a pattern (i.e., a reusable structure) for creating DSLs with Julia 0.6 and above." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Creating a `Model` object containing a function" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "This blog post arose from a question in the JuliaCon 2017 hackathon about the [Modia modelling language](https://github.com/ModiaSim/Modia.jl), where there is a `@model` macro. Here we will describe the simplest possible version of such a macro, which will create a `Model` object that contains a function, and is itself callable." 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "First we define the `Model` object. It is tempting to write it like this:" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "struct NaiveModel\n", 47 | " f::Function\n", 48 | "end" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "We can then create an instance of the `NaiveModel` type (i.e. an object of that type) using the default constructor, e.g. by passing it an anonymous function:" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "m1 = NaiveModel(x->2x)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "and we can call the function using" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "m1.f(10)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "If we wish instances like `m` to themselves behave like functions, we can overload the call syntax on the `NaiveModel` object:" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "(m::NaiveModel)(x) = m.f(x)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "so that we can now just write" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "m1(10)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## Parametrising the type" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "Since `Function` is an abstract type, for performance we should *not* have a field of this type inside our object.\n", 127 | "Rather, we parametrise the type using the type of the function:" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "struct Model{F}\n", 137 | " f::F\n", 138 | "end\n", 139 | "\n", 140 | "(m::Model)(x) = m.f(x)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "m2 = Model(x->2x)" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "m2(10)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "Let's compare the performance:" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "using BenchmarkTools" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "@btime m1(10)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "@btime m2(10)" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "Indeed we have removed some overhead in the second case." 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": {}, 205 | "source": [ 206 | "## Defining a macro to create objects" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "We now wish to define a macro that will allow us to use a simple syntax (that uses standard Julia syntax but itself is not valid Julia code) to create objects. In this case, we wish to write\n", 214 | "\n", 215 | " @model 2x\n", 216 | " \n", 217 | "to define a `Model` object with the function `x->2x`." 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "metadata": {}, 223 | "source": [ 224 | "We will build up to macros by first building the tools (standard Julia functions) to manipulate the expression `2x` in the correct way to build a `Model` object from it." 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "First, let's create a function to manipulate our expression:" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "function make_function(ex::Expr)\n", 241 | " return :(x -> $ex)\n", 242 | "end" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "ex = :(2x)\n", 252 | "make_function(ex)" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "This function assumes that `ex` is an expression containing the variable `x` and makes a new expression representing an\n", 260 | "anonymous function with the single argument `x`. (See [my JuliaCon 2017 tutorial](https://github.com/dpsanders/julia_towards_1.0/blob/master/4.%20Metaprogramming.ipynb) for an example of how to walk through the expression tree in order to extract *automatically* the variables that it contains.)" 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": {}, 266 | "source": [ 267 | "Now let's define a function `make_model` that takes a function and wraps it and passes it into a `Model` object:" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": null, 273 | "metadata": {}, 274 | "outputs": [], 275 | "source": [ 276 | "function make_model(ex::Expr)\n", 277 | " return :(Model($ex))\n", 278 | "end" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "make_model(make_function(:(2x)))" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "If we evaluate this \"by hand\", we see that it correctly creates a `Model` object:" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": null, 300 | "metadata": {}, 301 | "outputs": [], 302 | "source": [ 303 | "m3 = eval(make_model(make_function(:(2x))))" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": null, 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "m3(10)" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "## Making a macro" 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "However, this is ugly and clumsy. Instead, we now wrap everything inside a **macro**." 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "metadata": {}, 332 | "source": [ 333 | "A **macro** is a kind of \"super-function\" that manipulates code. \n", 334 | "\n", 335 | "\n", 336 | "A macro, in the simplest case, takes a single Julia `Expr` object (i.e. an unevaluated Julia expression) as argument. \n", 337 | "It manipulates this expression object to create a new expression object, and returns this new expression.\n", 338 | "\n", 339 | "The key point is that **this returned expression is *automatically* evaluated in the correct scope** when the macro returns.\n", 340 | "Indeed, the macro actually \"splices in\" the newly-generated code in place of the old code, before the Julia compiler kicks in." 341 | ] 342 | }, 343 | { 344 | "cell_type": "markdown", 345 | "metadata": {}, 346 | "source": [ 347 | "Let's start with the simplest possible macro:" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": null, 353 | "metadata": {}, 354 | "outputs": [], 355 | "source": [ 356 | "macro model(ex)\n", 357 | " @show ex\n", 358 | " @show typeof(ex)\n", 359 | " return nothing\n", 360 | "end" 361 | ] 362 | }, 363 | { 364 | "cell_type": "markdown", 365 | "metadata": {}, 366 | "source": [ 367 | "It just shows the argument that it was passed and exits, returning an empty expression." 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": null, 373 | "metadata": {}, 374 | "outputs": [], 375 | "source": [ 376 | "m4 = @model 2x" 377 | ] 378 | }, 379 | { 380 | "cell_type": "markdown", 381 | "metadata": {}, 382 | "source": [ 383 | "We see that the Julia `Expr` object has been automatically created from the explicit code that we typed." 384 | ] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "metadata": {}, 389 | "source": [ 390 | "Now we can plug in our previous functions to complete the macro's functionality:" 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": null, 396 | "metadata": {}, 397 | "outputs": [], 398 | "source": [ 399 | "macro model(ex)\n", 400 | " return make_model(make_function(ex))\n", 401 | "end" 402 | ] 403 | }, 404 | { 405 | "cell_type": "code", 406 | "execution_count": null, 407 | "metadata": {}, 408 | "outputs": [], 409 | "source": [ 410 | "m5 = @model 2x" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": null, 416 | "metadata": {}, 417 | "outputs": [], 418 | "source": [ 419 | "@macroexpand @model 2x" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "m5(10)" 429 | ] 430 | }, 431 | { 432 | "cell_type": "markdown", 433 | "metadata": {}, 434 | "source": [ 435 | "To check that the macro is doing what we think it is, we can use the `@macroexpand` command, which itself is a macro (as denoted by the initial `@`):" 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "execution_count": null, 441 | "metadata": {}, 442 | "outputs": [], 443 | "source": [ 444 | "@macroexpand @model 2x" 445 | ] 446 | }, 447 | { 448 | "cell_type": "markdown", 449 | "metadata": {}, 450 | "source": [ 451 | "## Macro \"hygiene\"" 452 | ] 453 | }, 454 | { 455 | "cell_type": "markdown", 456 | "metadata": {}, 457 | "source": [ 458 | "However, our macro has an issue, called macro \"hygiene\". This has to do with where variables are defined. Let's put everything we have so far inside a module:" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": null, 464 | "metadata": {}, 465 | "outputs": [], 466 | "source": [ 467 | "module Models\n", 468 | "\n", 469 | "export Model, @model\n", 470 | "\n", 471 | "struct Model{F}\n", 472 | " f::F\n", 473 | "end\n", 474 | "\n", 475 | "(m::Model)(x) = m.f(x)\n", 476 | "\n", 477 | "function make_function(ex::Expr)\n", 478 | " return :(x -> $ex)\n", 479 | "end\n", 480 | "\n", 481 | "function make_model(ex::Expr)\n", 482 | " return :(Model($ex))\n", 483 | "end\n", 484 | "\n", 485 | "macro model(ex)\n", 486 | " return make_model(make_function(ex))\n", 487 | "end\n", 488 | "\n", 489 | "end" 490 | ] 491 | }, 492 | { 493 | "cell_type": "markdown", 494 | "metadata": {}, 495 | "source": [ 496 | "Now we import the module and use the macro:" 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": null, 502 | "metadata": {}, 503 | "outputs": [], 504 | "source": [ 505 | "using Models\n", 506 | "\n", 507 | "m6 = @model 2x\n", 508 | "m6(10)" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": {}, 514 | "source": [ 515 | "So far so good. But now let's try to include a global variable in the expression:" 516 | ] 517 | }, 518 | { 519 | "cell_type": "code", 520 | "execution_count": null, 521 | "metadata": {}, 522 | "outputs": [], 523 | "source": [ 524 | "a = 2\n", 525 | "m7 = @model 2*a*x" 526 | ] 527 | }, 528 | { 529 | "cell_type": "code", 530 | "execution_count": null, 531 | "metadata": {}, 532 | "outputs": [], 533 | "source": [ 534 | "m7(10)" 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": {}, 540 | "source": [ 541 | "We see that it cannot find `a`. Let's see what the macro is doing:" 542 | ] 543 | }, 544 | { 545 | "cell_type": "code", 546 | "execution_count": null, 547 | "metadata": {}, 548 | "outputs": [], 549 | "source": [ 550 | "@macroexpand @model 2*a*x" 551 | ] 552 | }, 553 | { 554 | "cell_type": "markdown", 555 | "metadata": {}, 556 | "source": [ 557 | "We see that Julia is looking for `Models.a`, i.e. a variable `a` defined inside the `Models` module. \n", 558 | "\n", 559 | "To fix this problem, we must write a \"hygienic\" macro, by **escaping** the code. This is a mechanism telling the compiler to look for variable definitions in the scope from which the macro is called (here, the current module `Main`), rather than the scope where the macro is defined (here, the `Models` module):" 560 | ] 561 | }, 562 | { 563 | "cell_type": "code", 564 | "execution_count": null, 565 | "metadata": {}, 566 | "outputs": [], 567 | "source": [ 568 | "module Models2\n", 569 | "\n", 570 | "export Model, @model\n", 571 | "\n", 572 | "struct Model{F}\n", 573 | " f::F\n", 574 | "end\n", 575 | "\n", 576 | "(m::Model)(x) = m.f(x)\n", 577 | "\n", 578 | "function make_function(ex::Expr)\n", 579 | " return :(x -> $ex)\n", 580 | "end\n", 581 | "\n", 582 | "function make_model(ex::Expr)\n", 583 | " return :(Model($ex))\n", 584 | "end\n", 585 | "\n", 586 | "macro model(ex)\n", 587 | " return esc(make_model(make_function(ex)))\n", 588 | "end\n", 589 | "\n", 590 | "end" 591 | ] 592 | }, 593 | { 594 | "cell_type": "code", 595 | "execution_count": null, 596 | "metadata": {}, 597 | "outputs": [], 598 | "source": [ 599 | "using Models2\n", 600 | "\n", 601 | "a = 2\n", 602 | "m8 = @model 2*a*x" 603 | ] 604 | }, 605 | { 606 | "cell_type": "code", 607 | "execution_count": null, 608 | "metadata": {}, 609 | "outputs": [], 610 | "source": [ 611 | "m8(10)" 612 | ] 613 | }, 614 | { 615 | "cell_type": "markdown", 616 | "metadata": {}, 617 | "source": [ 618 | "This is the final, working version of the macro." 619 | ] 620 | }, 621 | { 622 | "cell_type": "markdown", 623 | "metadata": {}, 624 | "source": [ 625 | "## Conclusion" 626 | ] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "metadata": {}, 631 | "source": [ 632 | "We have successfully completed our task: we have seen how to create a macro that enables a simple syntax for creating a Julia object that we can use later. \n", 633 | "\n", 634 | "This is the recommended pattern for creating DSLs in Julia." 635 | ] 636 | }, 637 | { 638 | "cell_type": "markdown", 639 | "metadata": { 640 | "collapsed": true 641 | }, 642 | "source": [ 643 | "For a more in-depth discussion of metaprogramming techniques and macros, see my video tutorial *Invitation to intermediate Julia*, given at JuliaCon 2016:\n", 644 | "\n", 645 | "- Video: https://www.youtube.com/watch?v=rAxzR7lMGDM\n", 646 | "- Jupyter notebooks: https://github.com/dpsanders/intermediate_julia" 647 | ] 648 | }, 649 | { 650 | "cell_type": "markdown", 651 | "metadata": {}, 652 | "source": [ 653 | "**Author**: [David P. Sanders](http://sistemas.fciencias.unam.mx/~dsanders/), Associate Professor, Department of Physics, Faculty of Sciences, National University of Mexico (UNAM)." 654 | ] 655 | } 656 | ], 657 | "metadata": { 658 | "kernelspec": { 659 | "display_name": "Julia 0.6.2", 660 | "language": "julia", 661 | "name": "julia-0.6" 662 | }, 663 | "language_info": { 664 | "file_extension": ".jl", 665 | "mimetype": "application/julia", 666 | "name": "julia", 667 | "version": "0.6.2" 668 | }, 669 | "toc": { 670 | "colors": { 671 | "hover_highlight": "#DAA520", 672 | "running_highlight": "#FF0000", 673 | "selected_highlight": "#FFD700" 674 | }, 675 | "moveMenuLeft": true, 676 | "nav_menu": { 677 | "height": "12px", 678 | "width": "252px" 679 | }, 680 | "navigate_menu": true, 681 | "number_sections": true, 682 | "sideBar": true, 683 | "threshold": "1", 684 | "toc_cell": false, 685 | "toc_section_display": "block", 686 | "toc_window_display": false 687 | } 688 | }, 689 | "nbformat": 4, 690 | "nbformat_minor": 2 691 | } 692 | -------------------------------------------------------------------------------- /18. Pleasingly parallel simulations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Pleasingly parallel simulations" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "**Pleasingly parallel** simulations (also called \"embarrassingly parallel\") consist of simulations in which each process / worker is executing the same code independently, with no communication required between workers. It is the simplest type of parallelism, and also probably the most common." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "data": { 24 | "text/plain": [ 25 | "2-element Array{Int64,1}:\n", 26 | " 2\n", 27 | " 3" 28 | ] 29 | }, 30 | "execution_count": 1, 31 | "metadata": {}, 32 | "output_type": "execute_result" 33 | } 34 | ], 35 | "source": [ 36 | "addprocs(2)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 2, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "@everywhere using DistributedArrays" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "@everywhere function walk(numsteps)\n", 55 | " pos = 0\n", 56 | "\n", 57 | " for j in 1:numsteps\n", 58 | " \n", 59 | " if rand(Bool) # NB\n", 60 | " step = -1\n", 61 | " else\n", 62 | " step = +1\n", 63 | " end\n", 64 | " \n", 65 | " pos += step # ifelse(rand() < 0.5, -1, +1)\n", 66 | " end\n", 67 | " \n", 68 | " return pos\n", 69 | "end" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "In serial:" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 4, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/plain": [ 87 | "1000" 88 | ] 89 | }, 90 | "execution_count": 4, 91 | "metadata": {}, 92 | "output_type": "execute_result" 93 | } 94 | ], 95 | "source": [ 96 | "numwalkers = 1000\n", 97 | "numsteps = 1000" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 5, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "1000-element Array{Int64,1}:\n", 109 | " -4\n", 110 | " -32\n", 111 | " -30\n", 112 | " 2\n", 113 | " -2\n", 114 | " -10\n", 115 | " -20\n", 116 | " 4\n", 117 | " 70\n", 118 | " 30\n", 119 | " 4\n", 120 | " 30\n", 121 | " 24\n", 122 | " ⋮\n", 123 | " 6\n", 124 | " -38\n", 125 | " -12\n", 126 | " 22\n", 127 | " -38\n", 128 | " -66\n", 129 | " 18\n", 130 | " 12\n", 131 | " -32\n", 132 | " -16\n", 133 | " 0\n", 134 | " -20" 135 | ] 136 | }, 137 | "execution_count": 5, 138 | "metadata": {}, 139 | "output_type": "execute_result" 140 | } 141 | ], 142 | "source": [ 143 | "data = [walk(numsteps) for i in 1:numwalkers] " 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "Let's make a distributed array that will divide up the walkers' indices between the available workers:" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 6, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "1000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 162 | " 1\n", 163 | " 2\n", 164 | " 3\n", 165 | " 4\n", 166 | " 5\n", 167 | " 6\n", 168 | " 7\n", 169 | " 8\n", 170 | " 9\n", 171 | " 10\n", 172 | " 11\n", 173 | " 12\n", 174 | " 13\n", 175 | " ⋮\n", 176 | " 989\n", 177 | " 990\n", 178 | " 991\n", 179 | " 992\n", 180 | " 993\n", 181 | " 994\n", 182 | " 995\n", 183 | " 996\n", 184 | " 997\n", 185 | " 998\n", 186 | " 999\n", 187 | " 1000" 188 | ] 189 | }, 190 | "execution_count": 6, 191 | "metadata": {}, 192 | "output_type": "execute_result" 193 | } 194 | ], 195 | "source": [ 196 | "walkers = @DArray [i for i in 1:numwalkers]" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 7, 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "data": { 206 | "text/plain": [ 207 | "2-element Array{Tuple{UnitRange{Int64}},1}:\n", 208 | " (1:500,) \n", 209 | " (501:1000,)" 210 | ] 211 | }, 212 | "execution_count": 7, 213 | "metadata": {}, 214 | "output_type": "execute_result" 215 | } 216 | ], 217 | "source": [ 218 | "walkers.indexes" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "Let's send out the information about how many steps and walkers to do:" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 14, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "@everywhere begin\n", 235 | " numsteps = 10000\n", 236 | " numwalkers = 100000 \n", 237 | "end" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "We will now use a \"trick\": we wish to run the same function (`walk`) for each walker.\n", 245 | "But the function does not really need the information about *which* walker (index) it is. So we can use a function that takes no arguments:" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "A function that takes zero arguments and returns a constant:\n" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 14, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "data": { 262 | "text/plain": [ 263 | "f (generic function with 1 method)" 264 | ] 265 | }, 266 | "execution_count": 14, 267 | "metadata": {}, 268 | "output_type": "execute_result" 269 | } 270 | ], 271 | "source": [ 272 | "f() = 3 " 273 | ] 274 | }, 275 | { 276 | "cell_type": "markdown", 277 | "metadata": {}, 278 | "source": [ 279 | "An anonymous function that takes one argument, that it ignores, and returns a constant:" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": 15, 285 | "metadata": {}, 286 | "outputs": [ 287 | { 288 | "data": { 289 | "text/plain": [ 290 | "(::#19) (generic function with 1 method)" 291 | ] 292 | }, 293 | "execution_count": 15, 294 | "metadata": {}, 295 | "output_type": "execute_result" 296 | } 297 | ], 298 | "source": [ 299 | "_ -> 3" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "An anonymous function that returns something random:" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 16, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "data": { 316 | "text/plain": [ 317 | "(::#21) (generic function with 1 method)" 318 | ] 319 | }, 320 | "execution_count": 16, 321 | "metadata": {}, 322 | "output_type": "execute_result" 323 | } 324 | ], 325 | "source": [ 326 | "g = _ -> walk(numsteps) # _ means that the argument is ignored" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 9, 332 | "metadata": {}, 333 | "outputs": [ 334 | { 335 | "data": { 336 | "text/plain": [ 337 | "46" 338 | ] 339 | }, 340 | "execution_count": 9, 341 | "metadata": {}, 342 | "output_type": "execute_result" 343 | } 344 | ], 345 | "source": [ 346 | "g(1)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": 11, 352 | "metadata": {}, 353 | "outputs": [ 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "-18" 358 | ] 359 | }, 360 | "execution_count": 11, 361 | "metadata": {}, 362 | "output_type": "execute_result" 363 | } 364 | ], 365 | "source": [ 366 | "g(17)" 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 12, 372 | "metadata": { 373 | "scrolled": true 374 | }, 375 | "outputs": [ 376 | { 377 | "data": { 378 | "text/plain": [ 379 | "-44" 380 | ] 381 | }, 382 | "execution_count": 12, 383 | "metadata": {}, 384 | "output_type": "execute_result" 385 | } 386 | ], 387 | "source": [ 388 | "g(\"string\")" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "Using an anonymous function to generate a vector of random numbers:" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 13, 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "data": { 405 | "text/plain": [ 406 | "5-element Array{Float64,1}:\n", 407 | " 0.608136\n", 408 | " 0.300646\n", 409 | " 0.919476\n", 410 | " 0.670608\n", 411 | " 0.356738" 412 | ] 413 | }, 414 | "execution_count": 13, 415 | "metadata": {}, 416 | "output_type": "execute_result" 417 | } 418 | ], 419 | "source": [ 420 | "h = _ -> rand()\n", 421 | "map(h, 1:5)" 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": 17, 427 | "metadata": {}, 428 | "outputs": [ 429 | { 430 | "name": "stdout", 431 | "output_type": "stream", 432 | "text": [ 433 | " 0.121649 seconds (84.77 k allocations: 4.563 MiB, 6.54% gc time)\n" 434 | ] 435 | }, 436 | { 437 | "data": { 438 | "text/plain": [ 439 | "1000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 440 | " -18\n", 441 | " 58\n", 442 | " 10\n", 443 | " -34\n", 444 | " -6\n", 445 | " 34\n", 446 | " 32\n", 447 | " 36\n", 448 | " -26\n", 449 | " 20\n", 450 | " -18\n", 451 | " 4\n", 452 | " 6\n", 453 | " ⋮\n", 454 | " -26\n", 455 | " -46\n", 456 | " 4\n", 457 | " -20\n", 458 | " 4\n", 459 | " -20\n", 460 | " -76\n", 461 | " 16\n", 462 | " 24\n", 463 | " -20\n", 464 | " -22\n", 465 | " -8" 466 | ] 467 | }, 468 | "execution_count": 17, 469 | "metadata": {}, 470 | "output_type": "execute_result" 471 | } 472 | ], 473 | "source": [ 474 | "@time positions = map( _ -> walk(numsteps), walkers)" 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": 18, 480 | "metadata": {}, 481 | "outputs": [], 482 | "source": [ 483 | "@everywhere begin\n", 484 | " numsteps = 10000\n", 485 | " numwalkers = 100000 \n", 486 | "end" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 21, 492 | "metadata": {}, 493 | "outputs": [ 494 | { 495 | "data": { 496 | "text/plain": [ 497 | "run_distributed (generic function with 1 method)" 498 | ] 499 | }, 500 | "execution_count": 21, 501 | "metadata": {}, 502 | "output_type": "execute_result" 503 | } 504 | ], 505 | "source": [ 506 | "function run_serial(numwalkers, numsteps)\n", 507 | " walkers = 1:numwalkers\n", 508 | " data = map(_ -> walk(numsteps), 1:numwalkers)\n", 509 | "end\n", 510 | "\n", 511 | "function run_distributed(numwalkers, numsteps)\n", 512 | " walkers = distribute([1:numwalkers;])\n", 513 | " data = map(_ -> walk(numsteps), walkers)\n", 514 | "end" 515 | ] 516 | }, 517 | { 518 | "cell_type": "code", 519 | "execution_count": 23, 520 | "metadata": {}, 521 | "outputs": [ 522 | { 523 | "data": { 524 | "text/plain": [ 525 | "1-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 526 | " 1" 527 | ] 528 | }, 529 | "execution_count": 23, 530 | "metadata": {}, 531 | "output_type": "execute_result" 532 | } 533 | ], 534 | "source": [ 535 | "run_serial(1, 1)\n", 536 | "run_distributed(1, 1)" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": 24, 542 | "metadata": {}, 543 | "outputs": [ 544 | { 545 | "name": "stdout", 546 | "output_type": "stream", 547 | "text": [ 548 | " 2.231796 seconds (6 allocations: 781.484 KiB)\n", 549 | " 1.095464 seconds (2.93 k allocations: 1.699 MiB)\n" 550 | ] 551 | }, 552 | { 553 | "data": { 554 | "text/plain": [ 555 | "100000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 556 | " 34\n", 557 | " -124\n", 558 | " 40\n", 559 | " 48\n", 560 | " 46\n", 561 | " 160\n", 562 | " 6\n", 563 | " -96\n", 564 | " -70\n", 565 | " 112\n", 566 | " -84\n", 567 | " 40\n", 568 | " -104\n", 569 | " ⋮\n", 570 | " 2\n", 571 | " -124\n", 572 | " -150\n", 573 | " -28\n", 574 | " 124\n", 575 | " -60\n", 576 | " -36\n", 577 | " 48\n", 578 | " 60\n", 579 | " 144\n", 580 | " 14\n", 581 | " 42" 582 | ] 583 | }, 584 | "execution_count": 24, 585 | "metadata": {}, 586 | "output_type": "execute_result" 587 | } 588 | ], 589 | "source": [ 590 | "@time run_serial(numwalkers, numsteps)\n", 591 | "@time run_distributed(numwalkers, numsteps)" 592 | ] 593 | }, 594 | { 595 | "cell_type": "markdown", 596 | "metadata": {}, 597 | "source": [ 598 | "We see a 2x speedup with 2 processes!" 599 | ] 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "metadata": {}, 604 | "source": [ 605 | "# Another example: random matrices" 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "execution_count": null, 611 | "metadata": {}, 612 | "outputs": [], 613 | "source": [ 614 | "workers()" 615 | ] 616 | }, 617 | { 618 | "cell_type": "code", 619 | "execution_count": null, 620 | "metadata": {}, 621 | "outputs": [], 622 | "source": [ 623 | "# addprocs(4)" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "metadata": {}, 630 | "outputs": [], 631 | "source": [ 632 | "@everywhere begin\n", 633 | " using DistributedArrays\n", 634 | " using StatsBase\n", 635 | " using Plots\n", 636 | "end" 637 | ] 638 | }, 639 | { 640 | "cell_type": "code", 641 | "execution_count": null, 642 | "metadata": {}, 643 | "outputs": [], 644 | "source": [ 645 | "@everywhere function stochastic(β = 2, n = 200)\n", 646 | " h = n ^ -(1/3)\n", 647 | " x = 0:h:10\n", 648 | " N = length(x)\n", 649 | " d = (-2 / h^2 .- x) + 2*sqrt(h*β) * randn(N) # diagonal\n", 650 | " e = ones(N - 1) / h^2 # subdiagonal\n", 651 | " \n", 652 | " eigvals(SymTridiagonal(d, e))[N] # smallest negative eigenvalue\n", 653 | "end" 654 | ] 655 | }, 656 | { 657 | "cell_type": "markdown", 658 | "metadata": {}, 659 | "source": [ 660 | "Serial version:" 661 | ] 662 | }, 663 | { 664 | "cell_type": "code", 665 | "execution_count": null, 666 | "metadata": {}, 667 | "outputs": [], 668 | "source": [ 669 | "println(\"Serial version\")\n", 670 | "\n", 671 | "t = 10000\n", 672 | "p = plot()\n", 673 | "for β = [1,2,4,10,20]\n", 674 | " \n", 675 | " z = fit(Histogram, [stochastic(β) for i = 1:t], -4:0.01:1).weights\n", 676 | " plot!(midpoints(-4:0.01:1), z / sum(z) / 0.01)\n", 677 | "end\n", 678 | "p" 679 | ] 680 | }, 681 | { 682 | "cell_type": "markdown", 683 | "metadata": {}, 684 | "source": [ 685 | "A related parallel construct: `@parallel`. This does a \"reduce\" operation." 686 | ] 687 | }, 688 | { 689 | "cell_type": "code", 690 | "execution_count": null, 691 | "metadata": {}, 692 | "outputs": [], 693 | "source": [ 694 | "println(\"@parallel version\")\n", 695 | "\n", 696 | "@everywhere t = 10000\n", 697 | "\n", 698 | "p = plot()\n", 699 | "\n", 700 | "for β = [1,2,4,10,20]\n", 701 | " \n", 702 | " z = @parallel (+) for p = 1:nprocs()\n", 703 | " fit(Histogram, [stochastic(β) for i = 1:t], -4:0.01:1).weights\n", 704 | " end\n", 705 | " \n", 706 | " plot!(midpoints(-4:0.01:1), z / sum(z) / 0.01)\n", 707 | "end\n", 708 | "\n", 709 | "p" 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": null, 715 | "metadata": {}, 716 | "outputs": [], 717 | "source": [ 718 | "function dhist(x; closed=:left, nbins=10)\n", 719 | " \n", 720 | " hist_parts = DArray(p->fit(Histogram, localpart(x), closed=closed, nbins=nbins).weights, (nbins*length(x.pids),))\n", 721 | " \n", 722 | " reduce(+, map(pid -> @fetchfrom(pid, localpart(hist_parts)), hist_parts.pids))\n", 723 | " \n", 724 | "end" 725 | ] 726 | }, 727 | { 728 | "cell_type": "code", 729 | "execution_count": null, 730 | "metadata": {}, 731 | "outputs": [], 732 | "source": [ 733 | "a = randn(10000)\n", 734 | "d = distribute(a)\n", 735 | "\n", 736 | "dhist(d)" 737 | ] 738 | }, 739 | { 740 | "cell_type": "markdown", 741 | "metadata": {}, 742 | "source": [ 743 | "## SharedArrays and threads" 744 | ] 745 | }, 746 | { 747 | "cell_type": "markdown", 748 | "metadata": {}, 749 | "source": [ 750 | "Alternative techniques are SharedArrays and threads.\n", 751 | "\n", 752 | "https://docs.julialang.org/en/stable/manual/parallel-computing" 753 | ] 754 | } 755 | ], 756 | "metadata": { 757 | "anaconda-cloud": {}, 758 | "kernelspec": { 759 | "display_name": "Julia 0.6.2", 760 | "language": "julia", 761 | "name": "julia-0.6" 762 | }, 763 | "language_info": { 764 | "file_extension": ".jl", 765 | "mimetype": "application/julia", 766 | "name": "julia", 767 | "version": "0.6.2" 768 | }, 769 | "toc": { 770 | "colors": { 771 | "hover_highlight": "#DAA520", 772 | "running_highlight": "#FF0000", 773 | "selected_highlight": "#FFD700" 774 | }, 775 | "moveMenuLeft": true, 776 | "nav_menu": { 777 | "height": "30px", 778 | "width": "252px" 779 | }, 780 | "navigate_menu": true, 781 | "number_sections": true, 782 | "sideBar": true, 783 | "threshold": "2", 784 | "toc_cell": false, 785 | "toc_section_display": "block", 786 | "toc_window_display": false 787 | }, 788 | "widgets": { 789 | "state": { 790 | "04489acf-0ee9-4f51-9a17-dc735256cded": { 791 | "views": [ 792 | { 793 | "cell_index": 8 794 | } 795 | ] 796 | }, 797 | "5afac284-85c1-489b-8d82-036277cd1cf4": { 798 | "views": [ 799 | { 800 | "cell_index": 22 801 | } 802 | ] 803 | } 804 | }, 805 | "version": "1.2.0" 806 | } 807 | }, 808 | "nbformat": 4, 809 | "nbformat_minor": 2 810 | } 811 | -------------------------------------------------------------------------------- /live/18. Pleasingly parallel simulations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Pleasingly parallel simulations" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "**Pleasingly parallel** simulations (also called \"embarrassingly parallel\") consist of simulations in which each process / worker is executing the same code independently, with no communication required between workers. It is the simplest type of parallelism, and also probably the most common." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "data": { 24 | "text/plain": [ 25 | "2-element Array{Int64,1}:\n", 26 | " 2\n", 27 | " 3" 28 | ] 29 | }, 30 | "execution_count": 1, 31 | "metadata": {}, 32 | "output_type": "execute_result" 33 | } 34 | ], 35 | "source": [ 36 | "addprocs(2)" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 2, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "@everywhere using DistributedArrays" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "@everywhere function walk(numsteps)\n", 55 | " pos = 0\n", 56 | "\n", 57 | " for j in 1:numsteps\n", 58 | " \n", 59 | " if rand(Bool) # NB\n", 60 | " step = -1\n", 61 | " else\n", 62 | " step = +1\n", 63 | " end\n", 64 | " \n", 65 | " pos += step # ifelse(rand() < 0.5, -1, +1)\n", 66 | " end\n", 67 | " \n", 68 | " return pos\n", 69 | "end" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "In serial:" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 4, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/plain": [ 87 | "1000" 88 | ] 89 | }, 90 | "execution_count": 4, 91 | "metadata": {}, 92 | "output_type": "execute_result" 93 | } 94 | ], 95 | "source": [ 96 | "numwalkers = 1000\n", 97 | "numsteps = 1000" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 5, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "1000-element Array{Int64,1}:\n", 109 | " -4\n", 110 | " -32\n", 111 | " -30\n", 112 | " 2\n", 113 | " -2\n", 114 | " -10\n", 115 | " -20\n", 116 | " 4\n", 117 | " 70\n", 118 | " 30\n", 119 | " 4\n", 120 | " 30\n", 121 | " 24\n", 122 | " ⋮\n", 123 | " 6\n", 124 | " -38\n", 125 | " -12\n", 126 | " 22\n", 127 | " -38\n", 128 | " -66\n", 129 | " 18\n", 130 | " 12\n", 131 | " -32\n", 132 | " -16\n", 133 | " 0\n", 134 | " -20" 135 | ] 136 | }, 137 | "execution_count": 5, 138 | "metadata": {}, 139 | "output_type": "execute_result" 140 | } 141 | ], 142 | "source": [ 143 | "data = [walk(numsteps) for i in 1:numwalkers] " 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "Let's make a distributed array that will divide up the walkers' indices between the available workers:" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 6, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "1000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 162 | " 1\n", 163 | " 2\n", 164 | " 3\n", 165 | " 4\n", 166 | " 5\n", 167 | " 6\n", 168 | " 7\n", 169 | " 8\n", 170 | " 9\n", 171 | " 10\n", 172 | " 11\n", 173 | " 12\n", 174 | " 13\n", 175 | " ⋮\n", 176 | " 989\n", 177 | " 990\n", 178 | " 991\n", 179 | " 992\n", 180 | " 993\n", 181 | " 994\n", 182 | " 995\n", 183 | " 996\n", 184 | " 997\n", 185 | " 998\n", 186 | " 999\n", 187 | " 1000" 188 | ] 189 | }, 190 | "execution_count": 6, 191 | "metadata": {}, 192 | "output_type": "execute_result" 193 | } 194 | ], 195 | "source": [ 196 | "walkers = @DArray [i for i in 1:numwalkers]" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 7, 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "data": { 206 | "text/plain": [ 207 | "2-element Array{Tuple{UnitRange{Int64}},1}:\n", 208 | " (1:500,) \n", 209 | " (501:1000,)" 210 | ] 211 | }, 212 | "execution_count": 7, 213 | "metadata": {}, 214 | "output_type": "execute_result" 215 | } 216 | ], 217 | "source": [ 218 | "walkers.indexes" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "Let's send out the information about how many steps and walkers to do:" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 14, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "@everywhere begin\n", 235 | " numsteps = 10000\n", 236 | " numwalkers = 100000 \n", 237 | "end" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "We will now use a \"trick\": we wish to run the same function (`walk`) for each walker.\n", 245 | "But the function does not really need the information about *which* walker (index) it is. So we can use a function that takes no arguments:" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "A function that takes zero arguments and returns a constant:\n" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 14, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "data": { 262 | "text/plain": [ 263 | "f (generic function with 1 method)" 264 | ] 265 | }, 266 | "execution_count": 14, 267 | "metadata": {}, 268 | "output_type": "execute_result" 269 | } 270 | ], 271 | "source": [ 272 | "f() = 3 " 273 | ] 274 | }, 275 | { 276 | "cell_type": "markdown", 277 | "metadata": {}, 278 | "source": [ 279 | "An anonymous function that takes one argument, that it ignores, and returns a constant:" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": 15, 285 | "metadata": {}, 286 | "outputs": [ 287 | { 288 | "data": { 289 | "text/plain": [ 290 | "(::#19) (generic function with 1 method)" 291 | ] 292 | }, 293 | "execution_count": 15, 294 | "metadata": {}, 295 | "output_type": "execute_result" 296 | } 297 | ], 298 | "source": [ 299 | "_ -> 3" 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "An anonymous function that returns something random:" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 16, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "data": { 316 | "text/plain": [ 317 | "(::#21) (generic function with 1 method)" 318 | ] 319 | }, 320 | "execution_count": 16, 321 | "metadata": {}, 322 | "output_type": "execute_result" 323 | } 324 | ], 325 | "source": [ 326 | "g = _ -> walk(numsteps) # _ means that the argument is ignored" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 9, 332 | "metadata": {}, 333 | "outputs": [ 334 | { 335 | "data": { 336 | "text/plain": [ 337 | "46" 338 | ] 339 | }, 340 | "execution_count": 9, 341 | "metadata": {}, 342 | "output_type": "execute_result" 343 | } 344 | ], 345 | "source": [ 346 | "g(1)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": 11, 352 | "metadata": {}, 353 | "outputs": [ 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "-18" 358 | ] 359 | }, 360 | "execution_count": 11, 361 | "metadata": {}, 362 | "output_type": "execute_result" 363 | } 364 | ], 365 | "source": [ 366 | "g(17)" 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 12, 372 | "metadata": { 373 | "scrolled": true 374 | }, 375 | "outputs": [ 376 | { 377 | "data": { 378 | "text/plain": [ 379 | "-44" 380 | ] 381 | }, 382 | "execution_count": 12, 383 | "metadata": {}, 384 | "output_type": "execute_result" 385 | } 386 | ], 387 | "source": [ 388 | "g(\"string\")" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "Using an anonymous function to generate a vector of random numbers:" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 13, 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "data": { 405 | "text/plain": [ 406 | "5-element Array{Float64,1}:\n", 407 | " 0.608136\n", 408 | " 0.300646\n", 409 | " 0.919476\n", 410 | " 0.670608\n", 411 | " 0.356738" 412 | ] 413 | }, 414 | "execution_count": 13, 415 | "metadata": {}, 416 | "output_type": "execute_result" 417 | } 418 | ], 419 | "source": [ 420 | "h = _ -> rand()\n", 421 | "map(h, 1:5)" 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": 17, 427 | "metadata": {}, 428 | "outputs": [ 429 | { 430 | "name": "stdout", 431 | "output_type": "stream", 432 | "text": [ 433 | " 0.121649 seconds (84.77 k allocations: 4.563 MiB, 6.54% gc time)\n" 434 | ] 435 | }, 436 | { 437 | "data": { 438 | "text/plain": [ 439 | "1000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 440 | " -18\n", 441 | " 58\n", 442 | " 10\n", 443 | " -34\n", 444 | " -6\n", 445 | " 34\n", 446 | " 32\n", 447 | " 36\n", 448 | " -26\n", 449 | " 20\n", 450 | " -18\n", 451 | " 4\n", 452 | " 6\n", 453 | " ⋮\n", 454 | " -26\n", 455 | " -46\n", 456 | " 4\n", 457 | " -20\n", 458 | " 4\n", 459 | " -20\n", 460 | " -76\n", 461 | " 16\n", 462 | " 24\n", 463 | " -20\n", 464 | " -22\n", 465 | " -8" 466 | ] 467 | }, 468 | "execution_count": 17, 469 | "metadata": {}, 470 | "output_type": "execute_result" 471 | } 472 | ], 473 | "source": [ 474 | "@time positions = map( _ -> walk(numsteps), walkers)" 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": 18, 480 | "metadata": {}, 481 | "outputs": [], 482 | "source": [ 483 | "@everywhere begin\n", 484 | " numsteps = 10000\n", 485 | " numwalkers = 100000 \n", 486 | "end" 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 21, 492 | "metadata": {}, 493 | "outputs": [ 494 | { 495 | "data": { 496 | "text/plain": [ 497 | "run_distributed (generic function with 1 method)" 498 | ] 499 | }, 500 | "execution_count": 21, 501 | "metadata": {}, 502 | "output_type": "execute_result" 503 | } 504 | ], 505 | "source": [ 506 | "function run_serial(numwalkers, numsteps)\n", 507 | " walkers = 1:numwalkers\n", 508 | " data = map(_ -> walk(numsteps), 1:numwalkers)\n", 509 | "end\n", 510 | "\n", 511 | "function run_distributed(numwalkers, numsteps)\n", 512 | " walkers = distribute([1:numwalkers;])\n", 513 | " data = map(_ -> walk(numsteps), walkers)\n", 514 | "end" 515 | ] 516 | }, 517 | { 518 | "cell_type": "code", 519 | "execution_count": 23, 520 | "metadata": {}, 521 | "outputs": [ 522 | { 523 | "data": { 524 | "text/plain": [ 525 | "1-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 526 | " 1" 527 | ] 528 | }, 529 | "execution_count": 23, 530 | "metadata": {}, 531 | "output_type": "execute_result" 532 | } 533 | ], 534 | "source": [ 535 | "run_serial(1, 1)\n", 536 | "run_distributed(1, 1)" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": 24, 542 | "metadata": {}, 543 | "outputs": [ 544 | { 545 | "name": "stdout", 546 | "output_type": "stream", 547 | "text": [ 548 | " 2.231796 seconds (6 allocations: 781.484 KiB)\n", 549 | " 1.095464 seconds (2.93 k allocations: 1.699 MiB)\n" 550 | ] 551 | }, 552 | { 553 | "data": { 554 | "text/plain": [ 555 | "100000-element DistributedArrays.DArray{Int64,1,Array{Int64,1}}:\n", 556 | " 34\n", 557 | " -124\n", 558 | " 40\n", 559 | " 48\n", 560 | " 46\n", 561 | " 160\n", 562 | " 6\n", 563 | " -96\n", 564 | " -70\n", 565 | " 112\n", 566 | " -84\n", 567 | " 40\n", 568 | " -104\n", 569 | " ⋮\n", 570 | " 2\n", 571 | " -124\n", 572 | " -150\n", 573 | " -28\n", 574 | " 124\n", 575 | " -60\n", 576 | " -36\n", 577 | " 48\n", 578 | " 60\n", 579 | " 144\n", 580 | " 14\n", 581 | " 42" 582 | ] 583 | }, 584 | "execution_count": 24, 585 | "metadata": {}, 586 | "output_type": "execute_result" 587 | } 588 | ], 589 | "source": [ 590 | "@time run_serial(numwalkers, numsteps)\n", 591 | "@time run_distributed(numwalkers, numsteps)" 592 | ] 593 | }, 594 | { 595 | "cell_type": "markdown", 596 | "metadata": {}, 597 | "source": [ 598 | "We see a 2x speedup with 2 processes!" 599 | ] 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "metadata": {}, 604 | "source": [ 605 | "# Another example: random matrices" 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "execution_count": null, 611 | "metadata": {}, 612 | "outputs": [], 613 | "source": [ 614 | "workers()" 615 | ] 616 | }, 617 | { 618 | "cell_type": "code", 619 | "execution_count": null, 620 | "metadata": {}, 621 | "outputs": [], 622 | "source": [ 623 | "# addprocs(4)" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": null, 629 | "metadata": {}, 630 | "outputs": [], 631 | "source": [ 632 | "@everywhere begin\n", 633 | " using DistributedArrays\n", 634 | " using StatsBase\n", 635 | " using Plots\n", 636 | "end" 637 | ] 638 | }, 639 | { 640 | "cell_type": "code", 641 | "execution_count": null, 642 | "metadata": {}, 643 | "outputs": [], 644 | "source": [ 645 | "@everywhere function stochastic(β = 2, n = 200)\n", 646 | " h = n ^ -(1/3)\n", 647 | " x = 0:h:10\n", 648 | " N = length(x)\n", 649 | " d = (-2 / h^2 .- x) + 2*sqrt(h*β) * randn(N) # diagonal\n", 650 | " e = ones(N - 1) / h^2 # subdiagonal\n", 651 | " \n", 652 | " eigvals(SymTridiagonal(d, e))[N] # smallest negative eigenvalue\n", 653 | "end" 654 | ] 655 | }, 656 | { 657 | "cell_type": "markdown", 658 | "metadata": {}, 659 | "source": [ 660 | "Serial version:" 661 | ] 662 | }, 663 | { 664 | "cell_type": "code", 665 | "execution_count": null, 666 | "metadata": {}, 667 | "outputs": [], 668 | "source": [ 669 | "println(\"Serial version\")\n", 670 | "\n", 671 | "t = 10000\n", 672 | "p = plot()\n", 673 | "for β = [1,2,4,10,20]\n", 674 | " \n", 675 | " z = fit(Histogram, [stochastic(β) for i = 1:t], -4:0.01:1).weights\n", 676 | " plot!(midpoints(-4:0.01:1), z / sum(z) / 0.01)\n", 677 | "end\n", 678 | "p" 679 | ] 680 | }, 681 | { 682 | "cell_type": "markdown", 683 | "metadata": {}, 684 | "source": [ 685 | "A related parallel construct: `@parallel`. This does a \"reduce\" operation." 686 | ] 687 | }, 688 | { 689 | "cell_type": "code", 690 | "execution_count": null, 691 | "metadata": {}, 692 | "outputs": [], 693 | "source": [ 694 | "println(\"@parallel version\")\n", 695 | "\n", 696 | "@everywhere t = 10000\n", 697 | "\n", 698 | "p = plot()\n", 699 | "\n", 700 | "for β = [1,2,4,10,20]\n", 701 | " \n", 702 | " z = @parallel (+) for p = 1:nprocs()\n", 703 | " fit(Histogram, [stochastic(β) for i = 1:t], -4:0.01:1).weights\n", 704 | " end\n", 705 | " \n", 706 | " plot!(midpoints(-4:0.01:1), z / sum(z) / 0.01)\n", 707 | "end\n", 708 | "\n", 709 | "p" 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": null, 715 | "metadata": {}, 716 | "outputs": [], 717 | "source": [ 718 | "function dhist(x; closed=:left, nbins=10)\n", 719 | " \n", 720 | " hist_parts = DArray(p->fit(Histogram, localpart(x), closed=closed, nbins=nbins).weights, (nbins*length(x.pids),))\n", 721 | " \n", 722 | " reduce(+, map(pid -> @fetchfrom(pid, localpart(hist_parts)), hist_parts.pids))\n", 723 | " \n", 724 | "end" 725 | ] 726 | }, 727 | { 728 | "cell_type": "code", 729 | "execution_count": null, 730 | "metadata": {}, 731 | "outputs": [], 732 | "source": [ 733 | "a = randn(10000)\n", 734 | "d = distribute(a)\n", 735 | "\n", 736 | "dhist(d)" 737 | ] 738 | }, 739 | { 740 | "cell_type": "markdown", 741 | "metadata": {}, 742 | "source": [ 743 | "## SharedArrays and threads" 744 | ] 745 | }, 746 | { 747 | "cell_type": "markdown", 748 | "metadata": {}, 749 | "source": [ 750 | "Alternative techniques are SharedArrays and threads.\n", 751 | "\n", 752 | "https://docs.julialang.org/en/stable/manual/parallel-computing" 753 | ] 754 | } 755 | ], 756 | "metadata": { 757 | "anaconda-cloud": {}, 758 | "kernelspec": { 759 | "display_name": "Julia 0.6.2", 760 | "language": "julia", 761 | "name": "julia-0.6" 762 | }, 763 | "language_info": { 764 | "file_extension": ".jl", 765 | "mimetype": "application/julia", 766 | "name": "julia", 767 | "version": "0.6.2" 768 | }, 769 | "toc": { 770 | "colors": { 771 | "hover_highlight": "#DAA520", 772 | "running_highlight": "#FF0000", 773 | "selected_highlight": "#FFD700" 774 | }, 775 | "moveMenuLeft": true, 776 | "nav_menu": { 777 | "height": "30px", 778 | "width": "252px" 779 | }, 780 | "navigate_menu": true, 781 | "number_sections": true, 782 | "sideBar": true, 783 | "threshold": "2", 784 | "toc_cell": false, 785 | "toc_section_display": "block", 786 | "toc_window_display": false 787 | }, 788 | "widgets": { 789 | "state": { 790 | "04489acf-0ee9-4f51-9a17-dc735256cded": { 791 | "views": [ 792 | { 793 | "cell_index": 8 794 | } 795 | ] 796 | }, 797 | "5afac284-85c1-489b-8d82-036277cd1cf4": { 798 | "views": [ 799 | { 800 | "cell_index": 22 801 | } 802 | ] 803 | } 804 | }, 805 | "version": "1.2.0" 806 | } 807 | }, 808 | "nbformat": 4, 809 | "nbformat_minor": 2 810 | } 811 | -------------------------------------------------------------------------------- /14. Interoperability - Python and C.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Interoperability with Python: the `PyCall` package" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Calling Python functions is easy, thanks to the [`PyCall.jl` package](https://github.com/stevengj/PyCall.jl):" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "using PyCall" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "`PyCall` has a high-level interface that is designed so that the \"transport\" between Julia and Python is transparent from the user's point of view. For example, to import the Python `math` module, we do" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 2, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "@pyimport math" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "We can see what this actually does:" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 4, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "quote # /Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl, line 466:\n", 58 | " if !((PyCall.isdefined)(:math)) # /Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl, line 467:\n", 59 | " const math = (PyCall.pywrap)((PyCall.pyimport)(\"math\"))\n", 60 | " else # /Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl, line 468:\n", 61 | " if !(math isa PyCall.Module) # /Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl, line 469:\n", 62 | " (PyCall.error)(\"@pyimport: \", :math, \" already defined\")\n", 63 | " end\n", 64 | " end # /Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl, line 471:\n", 65 | " PyCall.nothing\n", 66 | "end" 67 | ] 68 | }, 69 | "execution_count": 4, 70 | "metadata": {}, 71 | "output_type": "execute_result" 72 | } 73 | ], 74 | "source": [ 75 | "@macroexpand @pyimport math" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "We can now mix and match Python calls, labelled by the `math.` qualifier, and Julia calls:" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 6, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "text/plain": [ 93 | "-1.1102230246251565e-16" 94 | ] 95 | }, 96 | "execution_count": 6, 97 | "metadata": {}, 98 | "output_type": "execute_result" 99 | } 100 | ], 101 | "source": [ 102 | "math.sin(0.3*math.pi) - sin(0.3*pi) " 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "Array objects are automatically converted:" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 3, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "3×4 Array{Float64,2}:\n", 121 | " 0.184095 0.332699 0.594909 0.671562\n", 122 | " 0.465193 0.316497 0.0114373 0.324822\n", 123 | " 0.91203 0.696114 0.410632 0.708648" 124 | ] 125 | }, 126 | "execution_count": 3, 127 | "metadata": {}, 128 | "output_type": "execute_result" 129 | } 130 | ], 131 | "source": [ 132 | "@pyimport numpy.random as nprandom\n", 133 | "nprandom.rand(3,4)" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "Let's define a Julia function:" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 4, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/plain": [ 151 | "objective (generic function with 1 method)" 152 | ] 153 | }, 154 | "execution_count": 4, 155 | "metadata": {}, 156 | "output_type": "execute_result" 157 | } 158 | ], 159 | "source": [ 160 | "objective(x) = cos(x) - x" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 5, 166 | "metadata": {}, 167 | "outputs": [ 168 | { 169 | "data": { 170 | "text/plain": [ 171 | "-3.989992496600445" 172 | ] 173 | }, 174 | "execution_count": 5, 175 | "metadata": {}, 176 | "output_type": "execute_result" 177 | } 178 | ], 179 | "source": [ 180 | "objective(3)" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "We can pass this Julia function to a Python module:" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 7, 193 | "metadata": {}, 194 | "outputs": [ 195 | { 196 | "name": "stdout", 197 | "output_type": "stream", 198 | "text": [ 199 | "Solving environment: ...working... done\n" 200 | ] 201 | }, 202 | { 203 | "name": "stderr", 204 | "output_type": "stream", 205 | "text": [ 206 | "\n", 207 | "\n", 208 | "==> WARNING: A newer version of conda exists. <==\n", 209 | " current version: 4.4.10\n", 210 | " latest version: 4.5.0\n", 211 | "\n", 212 | "Please update conda by running\n", 213 | "\n", 214 | " $ conda update -n base conda\n", 215 | "\n", 216 | "\n", 217 | "scipy 1.0.0: #########9 | 99% " 218 | ] 219 | }, 220 | { 221 | "name": "stdout", 222 | "output_type": "stream", 223 | "text": [ 224 | "\n", 225 | "## Package Plan ##\n", 226 | "\n", 227 | " environment location: /Users/dpsanders/.julia/v0.6/Conda/deps/usr\n", 228 | "\n", 229 | " added / updated specs: \n", 230 | " - scipy\n", 231 | "\n", 232 | "\n", 233 | "The following packages will be downloaded:\n", 234 | "\n", 235 | " package | build\n", 236 | " ---------------------------|-----------------\n", 237 | " scipy-1.0.0 |py27_blas_openblas_201 15.7 MB conda-forge\n", 238 | "\n", 239 | "The following NEW packages will be INSTALLED:\n", 240 | "\n", 241 | " scipy: 1.0.0-py27_blas_openblas_201 conda-forge [blas_openblas]\n", 242 | "\n", 243 | "\n", 244 | "Downloading and Extracting Packages\n", 245 | "Preparing transaction: ...working... " 246 | ] 247 | }, 248 | { 249 | "name": "stderr", 250 | "output_type": "stream", 251 | "text": [ 252 | "\r", 253 | "scipy 1.0.0: ########## | 100% \n" 254 | ] 255 | }, 256 | { 257 | "name": "stdout", 258 | "output_type": "stream", 259 | "text": [ 260 | "done\n", 261 | "Verifying transaction: ...working... done\n", 262 | "Executing transaction: ...working... done\n" 263 | ] 264 | } 265 | ], 266 | "source": [ 267 | "using Conda\n", 268 | "Conda.add(\"scipy\") # use Julia's Conda package to install a Python package" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": 6, 274 | "metadata": {}, 275 | "outputs": [ 276 | { 277 | "ename": "LoadError", 278 | "evalue": "\u001b[91mPyError (ccall(@pysym(:PyImport_ImportModule), PyPtr, (Cstring,), name)\n\nThe Python package scipy.optimize could not be found by pyimport. Usually this means\nthat you did not install scipy.optimize in the Python version being used by PyCall.\n\nPyCall is currently configured to use the Julia-specific Python distribution\ninstalled by the Conda.jl package. To install the scipy.optimize module, you can\nuse `pyimport_conda(\"scipy.optimize\", PKG)`, where PKG is the Anaconda\npackage the contains the module scipy.optimize, or alternatively you can use the\nConda package directly (via `using Conda` followed by `Conda.add` etcetera).\n\nAlternatively, if you want to use a different Python distribution on your\nsystem, such as a system-wide Python (as opposed to the Julia-specific Python),\nyou can re-configure PyCall with that Python. As explained in the PyCall\ndocumentation, set ENV[\"PYTHON\"] to the path/name of the python executable\nyou want to use, run Pkg.build(\"PyCall\"), and re-launch Julia.\n\n) \nImportError('No module named scipy.optimize',)\n\u001b[39m", 279 | "output_type": "error", 280 | "traceback": [ 281 | "\u001b[91mPyError (ccall(@pysym(:PyImport_ImportModule), PyPtr, (Cstring,), name)\n\nThe Python package scipy.optimize could not be found by pyimport. Usually this means\nthat you did not install scipy.optimize in the Python version being used by PyCall.\n\nPyCall is currently configured to use the Julia-specific Python distribution\ninstalled by the Conda.jl package. To install the scipy.optimize module, you can\nuse `pyimport_conda(\"scipy.optimize\", PKG)`, where PKG is the Anaconda\npackage the contains the module scipy.optimize, or alternatively you can use the\nConda package directly (via `using Conda` followed by `Conda.add` etcetera).\n\nAlternatively, if you want to use a different Python distribution on your\nsystem, such as a system-wide Python (as opposed to the Julia-specific Python),\nyou can re-configure PyCall with that Python. As explained in the PyCall\ndocumentation, set ENV[\"PYTHON\"] to the path/name of the python executable\nyou want to use, run Pkg.build(\"PyCall\"), and re-launch Julia.\n\n) \nImportError('No module named scipy.optimize',)\n\u001b[39m", 282 | "", 283 | "Stacktrace:", 284 | " [1] \u001b[1mpyerr_check\u001b[22m\u001b[22m at \u001b[1m/Users/dpsanders/.julia/v0.6/PyCall/src/exception.jl:56\u001b[22m\u001b[22m [inlined]", 285 | " [2] \u001b[1mpyerr_check\u001b[22m\u001b[22m at \u001b[1m/Users/dpsanders/.julia/v0.6/PyCall/src/exception.jl:61\u001b[22m\u001b[22m [inlined]", 286 | " [3] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m/Users/dpsanders/.julia/v0.6/PyCall/src/exception.jl:81\u001b[22m\u001b[22m [inlined]", 287 | " [4] \u001b[1mpyimport\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/Users/dpsanders/.julia/v0.6/PyCall/src/PyCall.jl:374\u001b[22m\u001b[22m", 288 | " [5] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:522\u001b[22m\u001b[22m" 289 | ] 290 | } 291 | ], 292 | "source": [ 293 | "@pyimport scipy.optimize as so\n", 294 | "so.newton(objective, 1)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "## Accessing fields and methods of Python objects" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "metadata": {}, 307 | "source": [ 308 | "Accessing fields (properties) and methods of Python objects uses the `obj.a` and `obj.b()` syntax, where `obj` is a Python object.\n", 309 | "\n", 310 | "In Julia 0.6 the `obj.b` syntax in Julia is restricted to accessing fields of Julia composite types, so to access fields and methods of Python objects via `PyCall.jl`, it is necessary to use the syntax\n", 311 | "\n", 312 | "`obj[:a]` for fields, and\n", 313 | "\n", 314 | "`obj[:a]()` for methods\n", 315 | "\n", 316 | "Here, the Julia syntax `:a` refers to the Julia symbol `a`.\n", 317 | "\n", 318 | "In 0.7, it will be possible to use the `obj.b` syntax." 319 | ] 320 | }, 321 | { 322 | "cell_type": "markdown", 323 | "metadata": {}, 324 | "source": [ 325 | "# Interoperability with C" 326 | ] 327 | }, 328 | { 329 | "cell_type": "markdown", 330 | "metadata": {}, 331 | "source": [ 332 | "Julia has a simple way to call C and Fortran functions in shared libraries, via the `ccall` function." 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": 9, 338 | "metadata": {}, 339 | "outputs": [ 340 | { 341 | "name": "stdout", 342 | "output_type": "stream", 343 | "text": [ 344 | "search: \u001b[1mc\u001b[22m\u001b[1mc\u001b[22m\u001b[1ma\u001b[22m\u001b[1ml\u001b[22m\u001b[1ml\u001b[22m Abstra\u001b[1mc\u001b[22mt\u001b[1mC\u001b[22mh\u001b[1ma\u001b[22mnne\u001b[1ml\u001b[22m\n", 345 | "\n" 346 | ] 347 | }, 348 | { 349 | "data": { 350 | "text/markdown": [ 351 | "```\n", 352 | "ccall((symbol, library) or function_pointer, ReturnType, (ArgumentType1, ...), ArgumentValue1, ...)\n", 353 | "```\n", 354 | "\n", 355 | "Call function in C-exported shared library, specified by `(function name, library)` tuple, where each component is a string or symbol.\n", 356 | "\n", 357 | "Note that the argument type tuple must be a literal tuple, and not a tuple-valued variable or expression. Alternatively, `ccall` may also be used to call a function pointer, such as one returned by `dlsym`.\n", 358 | "\n", 359 | "Each `ArgumentValue` to the `ccall` will be converted to the corresponding `ArgumentType`, by automatic insertion of calls to `unsafe_convert(ArgumentType, cconvert(ArgumentType, ArgumentValue))`. (See also the documentation for each of these functions for further details.) In most cases, this simply results in a call to `convert(ArgumentType, ArgumentValue)`.\n" 360 | ], 361 | "text/plain": [ 362 | "```\n", 363 | "ccall((symbol, library) or function_pointer, ReturnType, (ArgumentType1, ...), ArgumentValue1, ...)\n", 364 | "```\n", 365 | "\n", 366 | "Call function in C-exported shared library, specified by `(function name, library)` tuple, where each component is a string or symbol.\n", 367 | "\n", 368 | "Note that the argument type tuple must be a literal tuple, and not a tuple-valued variable or expression. Alternatively, `ccall` may also be used to call a function pointer, such as one returned by `dlsym`.\n", 369 | "\n", 370 | "Each `ArgumentValue` to the `ccall` will be converted to the corresponding `ArgumentType`, by automatic insertion of calls to `unsafe_convert(ArgumentType, cconvert(ArgumentType, ArgumentValue))`. (See also the documentation for each of these functions for further details.) In most cases, this simply results in a call to `convert(ArgumentType, ArgumentValue)`.\n" 371 | ] 372 | }, 373 | "execution_count": 9, 374 | "metadata": {}, 375 | "output_type": "execute_result" 376 | } 377 | ], 378 | "source": [ 379 | "?ccall" 380 | ] 381 | }, 382 | { 383 | "cell_type": "markdown", 384 | "metadata": {}, 385 | "source": [ 386 | "We see that we must specify:\n", 387 | "\n", 388 | "- the name of the function, as a Julia symbol or as a string\n", 389 | "- the name of the shared library where it lives; these are given as an ordered pair (tuple)\n", 390 | "\n", 391 | "- the return type of the function\n", 392 | "\n", 393 | "- the argument types that the function accepts, as a tuple\n", 394 | "\n", 395 | "- the arguments themselves" 396 | ] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "metadata": {}, 401 | "source": [ 402 | "A simple example is to call the `sin` function from the `libm` (math) library:" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 14, 408 | "metadata": {}, 409 | "outputs": [ 410 | { 411 | "data": { 412 | "text/plain": [ 413 | "0.09983341664682815" 414 | ] 415 | }, 416 | "execution_count": 14, 417 | "metadata": {}, 418 | "output_type": "execute_result" 419 | } 420 | ], 421 | "source": [ 422 | "x = ccall( (:sin, \"libm\"), Float64, (Float64,), 0.1)" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 15, 428 | "metadata": {}, 429 | "outputs": [ 430 | { 431 | "data": { 432 | "text/plain": [ 433 | "0.09983341664682815" 434 | ] 435 | }, 436 | "execution_count": 15, 437 | "metadata": {}, 438 | "output_type": "execute_result" 439 | } 440 | ], 441 | "source": [ 442 | "sin(0.1)" 443 | ] 444 | }, 445 | { 446 | "cell_type": "markdown", 447 | "metadata": {}, 448 | "source": [ 449 | "Note that a tuple with a single element is written with a single trailing `,`." 450 | ] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": {}, 455 | "source": [ 456 | "## Wrapping your own C code" 457 | ] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": {}, 462 | "source": [ 463 | "The following example of wrapping your own C code is taken from a short course by Steven Johnson that is highly recommended: https://github.com/stevengj/18S096 (lecture 1):" 464 | ] 465 | }, 466 | { 467 | "cell_type": "code", 468 | "execution_count": 19, 469 | "metadata": {}, 470 | "outputs": [ 471 | { 472 | "data": { 473 | "text/plain": [ 474 | "c_sum (generic function with 1 method)" 475 | ] 476 | }, 477 | "execution_count": 19, 478 | "metadata": {}, 479 | "output_type": "execute_result" 480 | } 481 | ], 482 | "source": [ 483 | "C_code = \"\"\"\n", 484 | "#include \n", 485 | "double c_sum(size_t n, double *X) {\n", 486 | " double s = 0.0;\n", 487 | " for (size_t i = 0; i < n; ++i) {\n", 488 | " s += X[i];\n", 489 | " }\n", 490 | " return s;\n", 491 | "}\n", 492 | "\"\"\"\n", 493 | "\n", 494 | "# compile to a shared library by piping C_code to gcc\n", 495 | "# (only works if you have gcc installed):\n", 496 | "\n", 497 | "const Clib = tempname() # generate a temporary file\n", 498 | "\n", 499 | "open(`gcc -fPIC -O3 -msse3 -xc -shared -o $(Clib * \".\" * Libdl.dlext) -`, \"w\") do f\n", 500 | " print(f, C_code)\n", 501 | "end\n", 502 | "\n", 503 | "c_sum(X::Array{Float64}) = ccall((\"c_sum\", Clib), Float64, (Csize_t, Ptr{Float64}), length(X), X)" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": 17, 509 | "metadata": {}, 510 | "outputs": [ 511 | { 512 | "data": { 513 | "text/plain": [ 514 | "\"dylib\"" 515 | ] 516 | }, 517 | "execution_count": 17, 518 | "metadata": {}, 519 | "output_type": "execute_result" 520 | } 521 | ], 522 | "source": [ 523 | "Libdl.dlext # specifies the correct termination for a shared library on the current system" 524 | ] 525 | }, 526 | { 527 | "cell_type": "code", 528 | "execution_count": 20, 529 | "metadata": {}, 530 | "outputs": [ 531 | { 532 | "data": { 533 | "text/plain": [ 534 | "\"/var/folders/h0/68t1xc991mq20mrkn6yy6n5h0000gn/T/juliadpiXdk\"" 535 | ] 536 | }, 537 | "execution_count": 20, 538 | "metadata": {}, 539 | "output_type": "execute_result" 540 | } 541 | ], 542 | "source": [ 543 | "Clib" 544 | ] 545 | }, 546 | { 547 | "cell_type": "markdown", 548 | "metadata": {}, 549 | "source": [ 550 | "The (strange, in my opinion) [`do` block syntax](https://docs.julialang.org/en/stable/manual/functions/#Do-Block-Syntax-for-Function-Arguments-1) creates an anonymous function from the body of the `do` block, and passes the result of the `open` call in as the argument `f`.\n", 551 | "\n", 552 | "The code between backticks ( \\` ) creates a command object. The `$(...)` interpolates the result of running that piece of Julia code into the command object." 553 | ] 554 | }, 555 | { 556 | "cell_type": "markdown", 557 | "metadata": {}, 558 | "source": [ 559 | "## Wrapping C code in a package" 560 | ] 561 | }, 562 | { 563 | "cell_type": "markdown", 564 | "metadata": {}, 565 | "source": [ 566 | "Usually we wrap C code that will not change, so it is more common to compile the C code once and for all to a shared library and wrap it all in a package." 567 | ] 568 | }, 569 | { 570 | "cell_type": "markdown", 571 | "metadata": {}, 572 | "source": [ 573 | "For an example of how to do this in a package, see the author's [`CRlibm.jl` package](https://github.com/dpsanders/CRlibm.jl). \n", 574 | "\n", 575 | "There the C code is compiled when the package is installed, via the `deps/build.jl` script, which creates the `libcrlibm` shared library in a known location inside the package." 576 | ] 577 | } 578 | ], 579 | "metadata": { 580 | "kernelspec": { 581 | "display_name": "Julia 0.6.2", 582 | "language": "julia", 583 | "name": "julia-0.6" 584 | }, 585 | "language_info": { 586 | "file_extension": ".jl", 587 | "mimetype": "application/julia", 588 | "name": "julia", 589 | "version": "0.6.2" 590 | } 591 | }, 592 | "nbformat": 4, 593 | "nbformat_minor": 1 594 | } 595 | --------------------------------------------------------------------------------