├── README.md ├── .gitignore ├── advanced-python ├── Python Tricks.ipynb ├── 08-Lambda.ipynb ├── 20-Copying.ipynb ├── 15-Threading vs Multiprocessing.ipynb ├── 07-Itertools.ipynb ├── 12-RandomNumbers.ipynb ├── 14-Generators.ipynb ├── 06-Collections.ipynb ├── 19-The Asterisk(*).ipynb ├── 03-Dictionary.ipynb ├── 21-Context manager.ipynb ├── 05-Strings.ipynb ├── 02-Tuple.ipynb ├── 01-Lists.ipynb ├── 04-Sets.ipynb ├── 09-Exceptions.ipynb ├── 13-Decoratos.ipynb ├── 18-Functions arguments.ipynb ├── 16-Threading in Python.ipynb └── 11-JSON.ipynb └── general └── Anaconda Tutorial.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # Collection of Jupyter Notebooks for the website -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | advanced-python/.DS_Store 3 | advanced-python/.ipynb_checkpoints -------------------------------------------------------------------------------- /advanced-python/Python Tricks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Python Tricks" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Value swapping" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "5 10\n", 27 | "10 5\n", 28 | "Initial list : [1, 2, 3, 4, 5]\n", 29 | "Modified list: [2, 1, 3, 4, 5]\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "a, b = 5, 10\n", 35 | "print(a, b)\n", 36 | "a, b = b, a\n", 37 | "print(a, b)\n", 38 | "# also possible in a list\n", 39 | "myList = [1, 2, 3, 4, 5]\n", 40 | "print(\"Initial list :\", myList)\n", 41 | "myList[0], myList[1] = myList[1], myList[0]\n", 42 | "print(\"Modified list:\", myList)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "#### Create a single string from list" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 2, 55 | "metadata": {}, 56 | "outputs": [ 57 | { 58 | "name": "stdout", 59 | "output_type": "stream", 60 | "text": [ 61 | "I am awesome \n", 62 | "I am awesome\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "my_list = [\"I\", \"am\", \"awesome\"]\n", 68 | "\n", 69 | "# bad\n", 70 | "a = \"\"\n", 71 | "for i in my_list:\n", 72 | " a += i + \" \"\n", 73 | "print(a)\n", 74 | "\n", 75 | "# good\n", 76 | "a = \" \".join(my_list)\n", 77 | "print(a)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "0.4564969649999999\n", 90 | "0.0229722050000003\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "# join method is much faster\n", 96 | "from timeit import default_timer as timer\n", 97 | "my_list = [\"a\"] * 1000000\n", 98 | "\n", 99 | "# bad\n", 100 | "start = timer()\n", 101 | "a = \"\"\n", 102 | "for i in my_list:\n", 103 | " a += i\n", 104 | "end = timer()\n", 105 | "print(end - start)\n", 106 | "#print(a)\n", 107 | "\n", 108 | "# good\n", 109 | "start = timer()\n", 110 | "a = \" \".join(my_list)\n", 111 | "end = timer()\n", 112 | "print(end - start)\n", 113 | "#print(a)" 114 | ] 115 | } 116 | ], 117 | "metadata": { 118 | "kernelspec": { 119 | "display_name": "Python [conda env:pythonengineer_env]", 120 | "language": "python", 121 | "name": "conda-env-pythonengineer_env-py" 122 | }, 123 | "language_info": { 124 | "codemirror_mode": { 125 | "name": "ipython", 126 | "version": 3 127 | }, 128 | "file_extension": ".py", 129 | "mimetype": "text/x-python", 130 | "name": "python", 131 | "nbconvert_exporter": "python", 132 | "pygments_lexer": "ipython3", 133 | "version": "3.7.3" 134 | } 135 | }, 136 | "nbformat": 4, 137 | "nbformat_minor": 2 138 | } 139 | -------------------------------------------------------------------------------- /general/Anaconda Tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Anaconda Tutorial\n", 8 | "Anaconda is a free Python distribution specifically designed for scientific computing (data science, machine learning). Anaconda let's you easily manage multiple Python environments and simplifies package management.\n", 9 | "\n", 10 | "### Why multiple environments\n", 11 | "- Always manage the correct versions and dependencies for your project\n", 12 | "- Don't spoil your system with too many site packages\n", 13 | "\n", 14 | "## Install\n", 15 | "Go to https://www.anaconda.com/distribution/ and download the latest installer for your machine. Follow the setup instructions.\n", 16 | "\n", 17 | "## Get started\n", 18 | "\n", 19 | "\n", 20 | "### General\n", 21 | "Check version \n", 22 | "`conda --version` \n", 23 | "Check more information \n", 24 | "`conda info` \n", 25 | "Update conda \n", 26 | "`conda update conda`\n", 27 | "\n", 28 | "\n", 29 | "### Manage environments\n", 30 | "Create a virtual environment \n", 31 | "`conda create -n myenv` \n", 32 | "Specify a specific Python version \n", 33 | "`conda create -n myenv Python=3.7` \n", 34 | "Specify specific packages that are installed \n", 35 | "`conda create -n myenv Python=3.7 numpy matplotlib` \n", 36 | "\n", 37 | "Activate it (Depending on your machine): \n", 38 | "`conda activat myenv` \n", 39 | "`source activate myenv` \n", 40 | "Your terminal will feature the current activated environment. \n", 41 | "\n", 42 | "Deactivate it (Depending on your machine): \n", 43 | "`conda deactivate` \n", 44 | "`source deactivate`\n", 45 | "\n", 46 | "List all environments: \n", 47 | "`conda env list`\n", 48 | "\n", 49 | "Remove a specific environment \n", 50 | "`conda env remove -n myenv`\n", 51 | "\n", 52 | "### Manage packages\n", 53 | "Install specific packages \n", 54 | "`conda install numpy` \n", 55 | "Install multiple packages with one command \n", 56 | "`conda install seaborn matplotlib pandas`\n", 57 | "\n", 58 | "Deinstall packages \n", 59 | "`conda remove numpy` \n", 60 | "\n", 61 | "List all packages in an environment \n", 62 | "`conda list` \n", 63 | "\n", 64 | "Update a package \n", 65 | "`conda update numpy`\n", 66 | "\n", 67 | "Search for packages \n", 68 | "`conda search numpy`\n", 69 | "\n", 70 | "Installing with pip is also possible: \n", 71 | "`pip install numpy` \n", 72 | "It is recommended to install pip in your environment and then use your local pip version. Otherwise it will try to fall back to other ones on the machine \n", 73 | "`conda install pip` \n", 74 | "`pip install numpy` \n", 75 | "\n", 76 | "\n", 77 | "## Conda and Visual Studio Code\n", 78 | "Conda integrates nicely into VS Code. It can automatically detect your available conda environments and let's you specify which one you want to use." 79 | ] 80 | } 81 | ], 82 | "metadata": { 83 | "kernelspec": { 84 | "display_name": "Python (pyeng_env)", 85 | "language": "python", 86 | "name": "pyeng_env" 87 | }, 88 | "language_info": { 89 | "codemirror_mode": { 90 | "name": "ipython", 91 | "version": 3 92 | }, 93 | "file_extension": ".py", 94 | "mimetype": "text/x-python", 95 | "name": "python", 96 | "nbconvert_exporter": "python", 97 | "pygments_lexer": "ipython3", 98 | "version": "3.7.6" 99 | } 100 | }, 101 | "nbformat": 4, 102 | "nbformat_minor": 2 103 | } 104 | -------------------------------------------------------------------------------- /advanced-python/08-Lambda.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lambda functions\n", 8 | "A lambda function is a small (one line) anonymous function that is defined without a name. A lambda function can take any number of arguments, but can only have one expression. While normal functions are defined using the def keyword, in Python anonymous functions are defined using the lambda keyword.\n", 9 | "\n", 10 | "`lambda arguments: expression` \n", 11 | "\n", 12 | "Lambda functions are used when a simple function is used only once or for a short period in your code. It's most common use is as an argument to higher-order functions (functions that takes in other functions as arguments). They are also used along with built-in functions like `map()`, `filter()`, `reduce()`." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 4, 18 | "metadata": {}, 19 | "outputs": [ 20 | { 21 | "name": "stdout", 22 | "output_type": "stream", 23 | "text": [ 24 | "15 110\n", 25 | "20 35\n" 26 | ] 27 | } 28 | ], 29 | "source": [ 30 | "# a lambda function that adds 10 to the input argument\n", 31 | "f = lambda x: x+10\n", 32 | "val1 = f(5)\n", 33 | "val2 = f(100)\n", 34 | "print(val1, val2)\n", 35 | "\n", 36 | "# a lambda function that multiplies two input arguments and returns the result\n", 37 | "f = lambda x,y: x*y\n", 38 | "val3 = f(2,10)\n", 39 | "val4 = f(7,5)\n", 40 | "print(val3, val4)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "## Usage example: Lamdba inside another function\n", 48 | "Return a customized lambda function from another function and create different function variations depending on your needs." 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 3, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "12\n", 61 | "18\n" 62 | ] 63 | } 64 | ], 65 | "source": [ 66 | "def myfunc(n):\n", 67 | " return lambda x: x * n\n", 68 | "\n", 69 | "doubler = myfunc(2)\n", 70 | "print(doubler(6))\n", 71 | "\n", 72 | "tripler = myfunc(3)\n", 73 | "print(tripler(6))" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "## Custom sorting using a lambda function as key parameter\n", 81 | "The key function transforms each element before sorting." 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 9, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "[(5, -3), (4, 1), (10, 2), (1, 9)]\n", 94 | "[-1, 1, -2, 2, -3, 3, -4, 4]\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "points2D = [(1, 9), (4, 1), (5, -3), (10, 2)]\n", 100 | "sorted_by_y = sorted(points2D, key= lambda x: x[1])\n", 101 | "print(sorted_by_y)\n", 102 | "\n", 103 | "mylist = [- 1, -4, -2, -3, 1, 2, 3, 4]\n", 104 | "sorted_by_abs = sorted(mylist, key= lambda x: abs(x))\n", 105 | "print(sorted_by_abs)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "## Use lambda for map function\n", 113 | "`map(func, seq)`, transforms each element with the function." 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 10, 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "name": "stdout", 123 | "output_type": "stream", 124 | "text": [ 125 | "[2, 4, 6, 8, 10, 12]\n", 126 | "[2, 4, 6, 8, 10, 12]\n" 127 | ] 128 | } 129 | ], 130 | "source": [ 131 | "a = [1, 2, 3, 4, 5, 6]\n", 132 | "b = list(map(lambda x: x * 2 , a))\n", 133 | "\n", 134 | "# However, try to prefer list comprehension\n", 135 | "# Use map if you have an already defined function\n", 136 | "c = [x*2 for x in a]\n", 137 | "print(b)\n", 138 | "print(c)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "## Use lambda for filter function\n", 146 | "`filter(func, seq)`, returns all elements for which func evaluates to True." 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 11, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "[2, 4, 6, 8]\n", 159 | "[2, 4, 6, 8]\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "a = [1, 2, 3, 4, 5, 6, 7, 8]\n", 165 | "b = list(filter(lambda x: (x%2 == 0) , a))\n", 166 | "\n", 167 | "# However, the same can be achieved with list comprehension\n", 168 | "c = [x for x in a if x%2 == 0]\n", 169 | "print(b)\n", 170 | "print(c)" 171 | ] 172 | }, 173 | { 174 | "cell_type": "markdown", 175 | "metadata": {}, 176 | "source": [ 177 | "# reduce\n", 178 | "`reduce(func, seq)`, repeatedly applies the func to the elements and returns a single value. \n", 179 | "`func` takes 2 arguments." 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 12, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "name": "stdout", 189 | "output_type": "stream", 190 | "text": [ 191 | "24\n", 192 | "10\n" 193 | ] 194 | } 195 | ], 196 | "source": [ 197 | "from functools import reduce\n", 198 | "a = [1, 2, 3, 4]\n", 199 | "product_a = reduce(lambda x, y: x*y, a)\n", 200 | "print(product_a)\n", 201 | "sum_a = reduce(lambda x, y: x+y, a)\n", 202 | "print(sum_a)" 203 | ] 204 | } 205 | ], 206 | "metadata": { 207 | "kernelspec": { 208 | "display_name": "Python [conda env:pythonengineer_env]", 209 | "language": "python", 210 | "name": "conda-env-pythonengineer_env-py" 211 | }, 212 | "language_info": { 213 | "codemirror_mode": { 214 | "name": "ipython", 215 | "version": 3 216 | }, 217 | "file_extension": ".py", 218 | "mimetype": "text/x-python", 219 | "name": "python", 220 | "nbconvert_exporter": "python", 221 | "pygments_lexer": "ipython3", 222 | "version": "3.7.3" 223 | } 224 | }, 225 | "nbformat": 4, 226 | "nbformat_minor": 2 227 | } 228 | -------------------------------------------------------------------------------- /advanced-python/20-Copying.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Shallow vs deep copying\n", 8 | "In Python, assignment statements (`obj_b = obj_a`) do not create real copies. It only creates a new variable with the same reference. So when you want to make actual copies of mutable objects (lists, dicts) and want to modify the copy without affecting the original, you have to be careful. \n", 9 | "For 'real' copies we can use the `copy` module. However, for compound/nested objects (e.g. nested lists or dicts) and custom objects there is an important difference between **shallow** and **deep** copying:\n", 10 | "- shallow copies: Only *one level deep*. It creates a new collection object and populates it with references to the nested objects. This means modyfing a nested object in the copy deeper than one level affects the original.\n", 11 | "- deep copies: A *full independent clone*. It creates a new collection object and then recursively populates it with copies of the nested objects found in the original." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Assignment operation\n", 19 | "This will only create a new variable with the same reference. Modifying one will affect the other." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 16, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "name": "stdout", 29 | "output_type": "stream", 30 | "text": [ 31 | "[-10, 2, 3, 4, 5]\n", 32 | "[-10, 2, 3, 4, 5]\n" 33 | ] 34 | } 35 | ], 36 | "source": [ 37 | "list_a = [1, 2, 3, 4, 5]\n", 38 | "list_b = list_a\n", 39 | "\n", 40 | "list_a[0] = -10\n", 41 | "print(list_a)\n", 42 | "print(list_b)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Shallow copy\n", 50 | "One level deep. Modifying on level 1 does not affect the other list. Use `copy.copy()`, or object-specific copy functions/copy constructors." 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 17, 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "[1, 2, 3, 4, 5]\n", 63 | "[-10, 2, 3, 4, 5]\n" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "import copy\n", 69 | "list_a = [1, 2, 3, 4, 5]\n", 70 | "list_b = copy.copy(list_a)\n", 71 | "\n", 72 | "# not affects the other list\n", 73 | "list_b[0] = -10\n", 74 | "print(list_a)\n", 75 | "print(list_b)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "But with nested objects, modifying on level 2 or deeper does affect the other!" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 18, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "name": "stdout", 92 | "output_type": "stream", 93 | "text": [ 94 | "[[-10, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n", 95 | "[[-10, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "import copy\n", 101 | "list_a = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n", 102 | "list_b = copy.copy(list_a)\n", 103 | "\n", 104 | "# affects the other!\n", 105 | "list_a[0][0]= -10\n", 106 | "print(list_a)\n", 107 | "print(list_b)" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "Note: You can also use the following to create shallow copies" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 20, 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "# shallow copies\n", 126 | "list_b = list(list_a)\n", 127 | "list_b = list_a[:]\n", 128 | "list_b = list_a.copy()" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "## Deep copies\n", 136 | "Full independent clones. Use `copy.deepcopy()`." 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 27, 142 | "metadata": {}, 143 | "outputs": [ 144 | { 145 | "name": "stdout", 146 | "output_type": "stream", 147 | "text": [ 148 | "[[-10, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n", 149 | "[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n" 150 | ] 151 | } 152 | ], 153 | "source": [ 154 | "import copy\n", 155 | "list_a = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]\n", 156 | "list_b = copy.deepcopy(list_a)\n", 157 | "\n", 158 | "# not affects the other\n", 159 | "list_a[0][0]= -10\n", 160 | "print(list_a)\n", 161 | "print(list_b)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "## Custom objects\n", 169 | "You can use the `copy` module to get shallow or deep copies of custom objects." 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 28, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stdout", 179 | "output_type": "stream", 180 | "text": [ 181 | "28\n", 182 | "28\n" 183 | ] 184 | } 185 | ], 186 | "source": [ 187 | "class Person:\n", 188 | " def __init__(self, name, age):\n", 189 | " self.name = name\n", 190 | " self.age = age\n", 191 | " \n", 192 | "# Only copies the reference\n", 193 | "p1 = Person('Alex', 27)\n", 194 | "p2 = p1\n", 195 | "p2.age = 28\n", 196 | "print(p1.age)\n", 197 | "print(p2.age)" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 30, 203 | "metadata": {}, 204 | "outputs": [ 205 | { 206 | "name": "stdout", 207 | "output_type": "stream", 208 | "text": [ 209 | "27\n", 210 | "28\n" 211 | ] 212 | } 213 | ], 214 | "source": [ 215 | "# shallow copy\n", 216 | "import copy\n", 217 | "p1 = Person('Alex', 27)\n", 218 | "p2 = copy.copy(p1)\n", 219 | "p2.age = 28\n", 220 | "print(p1.age)\n", 221 | "print(p2.age)" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 33, 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "56\n", 234 | "56\n", 235 | "\n", 236 | "55\n", 237 | "56\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "class Company:\n", 243 | " def __init__(self, boss, employee):\n", 244 | " self. boss = boss\n", 245 | " self.employee = employee\n", 246 | "\n", 247 | "# shallow copy will affect nested objects\n", 248 | "boss = Person('Jane', 55)\n", 249 | "employee = Person('Joe', 28)\n", 250 | "company = Company(boss, employee)\n", 251 | "\n", 252 | "company_clone = copy.copy(company)\n", 253 | "company_clone.boss.age = 56\n", 254 | "print(company.boss.age)\n", 255 | "print(company_clone.boss.age)\n", 256 | "\n", 257 | "print()\n", 258 | "# deep copy will not affect nested objects\n", 259 | "boss = Person('Jane', 55)\n", 260 | "employee = Person('Joe', 28)\n", 261 | "company = Company(boss, employee)\n", 262 | "company_clone = copy.deepcopy(company)\n", 263 | "company_clone.boss.age = 56\n", 264 | "print(company.boss.age)\n", 265 | "print(company_clone.boss.age)" 266 | ] 267 | } 268 | ], 269 | "metadata": { 270 | "kernelspec": { 271 | "display_name": "Python [conda env:pythonengineer_env]", 272 | "language": "python", 273 | "name": "conda-env-pythonengineer_env-py" 274 | }, 275 | "language_info": { 276 | "codemirror_mode": { 277 | "name": "ipython", 278 | "version": 3 279 | }, 280 | "file_extension": ".py", 281 | "mimetype": "text/x-python", 282 | "name": "python", 283 | "nbconvert_exporter": "python", 284 | "pygments_lexer": "ipython3", 285 | "version": "3.7.3" 286 | } 287 | }, 288 | "nbformat": 4, 289 | "nbformat_minor": 2 290 | } 291 | -------------------------------------------------------------------------------- /advanced-python/15-Threading vs Multiprocessing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Threading vs Multiprocessing\n", 8 | "We have two common approaches to run code in parallel (achieve multitasking and speed up your program) : via threads or via multiple processes.\n", 9 | "## Process\n", 10 | "A Process is an instance of a program, e.g. a Python interpreter. They are independent from each other and do not share the same memory.\n", 11 | "\n", 12 | "Key facts:\n", 13 | "- A new process is started independently from the first process\n", 14 | "- Takes advantage of multiple CPUs and cores\n", 15 | "- Separate memory space\n", 16 | "- Memory is not shared between processes\n", 17 | "- One GIL (Global interpreter lock) for each process, i.e. avoids GIL limitation\n", 18 | "- Great for CPU-bound processing\n", 19 | "- Child processes are interruptable/killable\n", 20 | "\n", 21 | "\n", 22 | "- Starting a process is slower that starting a thread\n", 23 | "- Larger memory footprint\n", 24 | "- IPC (inter-process communication) is more complicated\n", 25 | "\n", 26 | "## Threads\n", 27 | "A thread is an entity within a process that can be scheduled for execution (Also known as \"leightweight process\"). A Process can spawn multiple threads.\n", 28 | "The main difference is that all threads within a process share the same memory.\n", 29 | "\n", 30 | "Key facts:\n", 31 | "- Multiple threads can be spawned within one process\n", 32 | "- Memory is shared between all threads\n", 33 | "- Starting a thread is faster than starting a process\n", 34 | "- Great for I/O-bound tasks\n", 35 | "- Leightweight - low memory footprint\n", 36 | " \n", 37 | "\n", 38 | "- One GIL for all threads, i.e. threads are limited by GIL\n", 39 | "- Multithreading has no effect for CPU-bound tasks due to the GIL\n", 40 | "- Not interruptible/killable -> be careful with memory leaks\n", 41 | "- increased potential for race conditions" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "## Threading in Python\n", 49 | "Use the `threading` module. \n", 50 | "\n", 51 | "Note: The following example usually won't benefit from multiple threads since it is CPU-bound. It should just show the example of how to use threads." 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 9, 57 | "metadata": { 58 | "collapsed": true 59 | }, 60 | "outputs": [], 61 | "source": [ 62 | "from threading import Thread\n", 63 | "\n", 64 | "def square_numbers():\n", 65 | " for i in range(1000):\n", 66 | " result = i * i\n", 67 | "\n", 68 | " \n", 69 | "if __name__ == \"__main__\": \n", 70 | " threads = []\n", 71 | " num_threads = 10\n", 72 | "\n", 73 | " # create threads and asign a function for each thread\n", 74 | " for i in range(num_threads):\n", 75 | " thread = Thread(target=square_numbers)\n", 76 | " threads.append(thread)\n", 77 | "\n", 78 | " # start all threads\n", 79 | " for thread in threads:\n", 80 | " thread.start()\n", 81 | "\n", 82 | " # wait for all threads to finish\n", 83 | " # block the main thread until these threads are finished\n", 84 | " for thread in threads:\n", 85 | " thread.join()" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "### When is Threading useful\n", 93 | "Despite the GIL it is useful for I/O-bound tasks when your program has to talk to slow devices, like a hard drive or a network connection. With threading the program can use the time waiting for these devices and intelligently do other tasks in the meantime. \n", 94 | "\n", 95 | "Example: Download website information from multiple sites. Use a thread for each site." 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "## Multiprocessing\n", 103 | "Use the `multiprocessing` module. The syntax is very similar to above." 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 10, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "from multiprocessing import Process\n", 115 | "import os\n", 116 | "\n", 117 | "\n", 118 | "def square_numbers():\n", 119 | " for i in range(1000):\n", 120 | " result = i * i\n", 121 | "\n", 122 | "\n", 123 | "if __name__ == \"__main__\":\n", 124 | " processes = []\n", 125 | " num_processes = os.cpu_count()\n", 126 | "\n", 127 | " # create processes and asign a function for each process\n", 128 | " for i in range(num_processes):\n", 129 | " process = Process(target=square_numbers)\n", 130 | " processes.append(process)\n", 131 | "\n", 132 | " # start all processes\n", 133 | " for process in processes:\n", 134 | " process.start()\n", 135 | "\n", 136 | " # wait for all processes to finish\n", 137 | " # block the main thread until these processes are finished\n", 138 | " for process in processes:\n", 139 | " process.join()" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "### When is Multiprocessing useful\n", 147 | "It is useful for CPU-bound tasks that have to do a lot of CPU operations for a large amount of data and require a lot of computation time. With multiprocessing you can\n", 148 | "split the data into equal parts an do parallel computing on different CPUs.\n", 149 | "\n", 150 | "Example: Calculate the square numbers for all numbers from 1 to 1000000. Divide the numbers into equal sized parts and use a process for each subset." 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "## GIL - Global interpreter lock\n", 158 | "This is a mutex (or a lock) that allows only one thread to hold control of the Python interpreter. This means that the GIL allows only one thread to execute at a time even in a multi-threaded architecture.\n", 159 | "\n", 160 | "#### Why is it needed?\n", 161 | "It is needed because CPython's (reference implementation of Python) memory management is not thread-safe. Python uses reference counting for memory management. It means that objects created in Python have a reference count variable that keeps track of the number of references that point to the object. When this count reaches zero, the memory occupied by the object is released. The problem was that this reference count variable needed protection from race conditions where two threads increase or decrease its value simultaneously. If this happens, it can cause either leaked memory that is never released or incorrectly release the memory while a reference to that object still exists.\n", 162 | "\n", 163 | "#### How to avoid the GIL\n", 164 | "The GIL is very controversial in the Python community. The main way to avoid the GIL is by using multiprocessing instead of threading. Another (however uncomfortable) solution would be to avoid the CPython implementation and use a free-threaded Python implementation like `Jython` or `IronPython`. A third option is to move parts of the application out into binary extensions modules, i.e. use Python as a wrapper for third party libraries (e.g. in C/C++). This is the path taken by `numypy` and `scipy`." 165 | ] 166 | } 167 | ], 168 | "metadata": { 169 | "kernelspec": { 170 | "display_name": "Python [conda env:pythonengineer_env]", 171 | "language": "python", 172 | "name": "conda-env-pythonengineer_env-py" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.7.3" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 2 189 | } 190 | -------------------------------------------------------------------------------- /advanced-python/07-Itertools.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Itertools\n", 8 | "\n", 9 | "The Python itertools module is a collection of tools for handling iterators. Simply put, iterators are data types that can be used in a for loop. The most common iterator in Python is the list.\n", 10 | "\n", 11 | "See https://docs.python.org/3/library/itertools.html for all possible itertools" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## product()\n", 19 | "This tool computes the cartesian product of input iterables. \n", 20 | "It is equivalent to nested for-loops. For example, *product(A, B)* returns the same as *((x,y) for x in A for y in B)*. " 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 94, 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "[(1, 3), (1, 4), (2, 3), (2, 4)]\n", 33 | "[(1, 3, 1, 3), (1, 3, 2, 3), (2, 3, 1, 3), (2, 3, 2, 3)]\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "from itertools import product\n", 39 | "\n", 40 | "prod = product([1, 2], [3, 4])\n", 41 | "print(list(prod)) # note that we convert the iterator to a list for printing\n", 42 | "\n", 43 | "# to allow the product of an iterable with itself, specify the number of repetitions \n", 44 | "prod = product([1, 2], [3], repeat=2)\n", 45 | "print(list(prod)) # note that we convert the iterator to a list for printing" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## permutations()\n", 53 | "This tool returns successive length permutations of elements in an iterable, with all possible orderings, and no repeated elements." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 95, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "[(1, 2, 3), (1, 3, 2), (2, 1, 3), (2, 3, 1), (3, 1, 2), (3, 2, 1)]\n", 66 | "[(1, 2), (1, 3), (2, 1), (2, 3), (3, 1), (3, 2)]\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "from itertools import permutations\n", 72 | "\n", 73 | "perm = permutations([1, 2, 3])\n", 74 | "print(list(perm))\n", 75 | "\n", 76 | "# optional: the length of the permutation tuples\n", 77 | "perm = permutations([1, 2, 3], 2)\n", 78 | "print(list(perm))" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "## combinations() and combinations_with_replacement()\n", 86 | "\n", 87 | "r-length tuples, in sorted order. So, if the input iterable is sorted, the combination tuples will be produced in sorted order. *combinations()* does not allow repeated elements, but *combinations_with_replacement()* does." 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 96, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "[(1, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 4)]\n", 100 | "[(1, 1), (1, 2), (1, 3), (1, 4), (2, 2), (2, 3), (2, 4), (3, 3), (3, 4), (4, 4)]\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "from itertools import combinations, combinations_with_replacement\n", 106 | "\n", 107 | "# the second argument is mandatory and specifies the length of the output tuples.\n", 108 | "comb = combinations([1, 2, 3, 4], 2)\n", 109 | "print(list(comb))\n", 110 | "\n", 111 | "comb = combinations_with_replacement([1, 2, 3, 4], 2)\n", 112 | "print(list(comb))" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## accumulate()\n", 120 | "Make an iterator that returns accumulated sums, or accumulated results of other binary functions." 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 97, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "[1, 3, 6, 10]\n", 133 | "[1, 2, 6, 24]\n", 134 | "[1, 5, 5, 6, 6, 6]\n" 135 | ] 136 | } 137 | ], 138 | "source": [ 139 | "from itertools import accumulate\n", 140 | "\n", 141 | "# return accumulated sums\n", 142 | "acc = accumulate([1,2,3,4])\n", 143 | "print(list(acc))\n", 144 | "\n", 145 | "# other possible functions are possible\n", 146 | "import operator\n", 147 | "acc = accumulate([1,2,3,4], func=operator.mul)\n", 148 | "print(list(acc))\n", 149 | "\n", 150 | "acc = accumulate([1,5,2,6,3,4], func=max)\n", 151 | "print(list(acc))" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "## groupby()\n", 159 | "Make an iterator that returns consecutive keys and groups from the iterable. The key is a function computing a key value for each element. If not specified or is None, key defaults to an identity function and returns the element unchanged. Generally, the iterable needs to already be sorted on the same key function." 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 98, 165 | "metadata": {}, 166 | "outputs": [ 167 | { 168 | "name": "stdout", 169 | "output_type": "stream", 170 | "text": [ 171 | "True [1, 2]\n", 172 | "False [3, 4]\n", 173 | "True ['hi', 'nice']\n", 174 | "False ['hello', 'cool']\n", 175 | "25 [{'name': 'Tim', 'age': 25}, {'name': 'Dan', 'age': 25}]\n", 176 | "27 [{'name': 'Lisa', 'age': 27}]\n", 177 | "28 [{'name': 'Claire', 'age': 28}]\n" 178 | ] 179 | } 180 | ], 181 | "source": [ 182 | "from itertools import groupby\n", 183 | "\n", 184 | "# use a function as key\n", 185 | "def smaller_than_3(x):\n", 186 | " return x < 3\n", 187 | "\n", 188 | "group_obj = groupby([1, 2, 3, 4], key=smaller_than_3)\n", 189 | "for key, group in group_obj:\n", 190 | " print(key, list(group))\n", 191 | " \n", 192 | "# or use a lamda expression, e.g. words with an 'i':\n", 193 | "group_obj = groupby([\"hi\", \"nice\", \"hello\", \"cool\"], key=lambda x: \"i\" in x)\n", 194 | "for key, group in group_obj:\n", 195 | " print(key, list(group))\n", 196 | " \n", 197 | "persons = [{'name': 'Tim', 'age': 25}, {'name': 'Dan', 'age': 25}, \n", 198 | " {'name': 'Lisa', 'age': 27}, {'name': 'Claire', 'age': 28}]\n", 199 | "\n", 200 | "for key, group in groupby(persons, key=lambda x: x['age']):\n", 201 | " print(key, list(group))" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "## Infinite iterators: count(), cycle(), repeat()" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 99, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "name": "stdout", 218 | "output_type": "stream", 219 | "text": [ 220 | "10\n", 221 | "11\n", 222 | "12\n", 223 | "13\n", 224 | "\n", 225 | "1\n", 226 | "2\n", 227 | "3\n", 228 | "1\n", 229 | "2\n", 230 | "3\n", 231 | "\n", 232 | "A\n", 233 | "A\n", 234 | "A\n" 235 | ] 236 | } 237 | ], 238 | "source": [ 239 | "from itertools import count, cycle, repeat\n", 240 | "# count(x): count from x: x, x+1, x+2, x+3...\n", 241 | "for i in count(10):\n", 242 | " print(i)\n", 243 | " if i >= 13:\n", 244 | " break\n", 245 | "\n", 246 | "# cycle(iterable) : cycle infinitely through an iterable\n", 247 | "print(\"\")\n", 248 | "sum = 0\n", 249 | "for i in cycle([1, 2, 3]):\n", 250 | " print(i)\n", 251 | " sum += i\n", 252 | " if sum >= 12:\n", 253 | " break\n", 254 | "\n", 255 | "# repeat(x): repeat x infinitely or n times\n", 256 | "print(\"\")\n", 257 | "for i in repeat(\"A\", 3):\n", 258 | " print(i)" 259 | ] 260 | } 261 | ], 262 | "metadata": { 263 | "kernelspec": { 264 | "display_name": "Python [conda env:pythonengineer_env]", 265 | "language": "python", 266 | "name": "conda-env-pythonengineer_env-py" 267 | }, 268 | "language_info": { 269 | "codemirror_mode": { 270 | "name": "ipython", 271 | "version": 3 272 | }, 273 | "file_extension": ".py", 274 | "mimetype": "text/x-python", 275 | "name": "python", 276 | "nbconvert_exporter": "python", 277 | "pygments_lexer": "ipython3", 278 | "version": "3.7.3" 279 | } 280 | }, 281 | "nbformat": 4, 282 | "nbformat_minor": 2 283 | } 284 | -------------------------------------------------------------------------------- /advanced-python/12-RandomNumbers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Random Numbers\n", 8 | "Python defines a set of functions that are used to generate or manipulate random numbers. This article covers:\n", 9 | "\n", 10 | "- the `random` module\n", 11 | "- reproduce numbers with random.seed()\n", 12 | "- create cryptographically strong random numbers with the `secrets` module\n", 13 | "- create random nd arrays with `numpy.random`" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## The random module\n", 21 | "This module implements pseudo-random number generators for various distributions. It uses the Mersenne Twister algorithm (https://en.wikipedia.org/wiki/Mersenne_Twister) as its core generator. It is called pseudo-random, because the numbers seem random, but are reproducable." 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 3, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "0.10426373452067317\n", 34 | "3.34983979352444\n", 35 | "3\n", 36 | "4\n", 37 | "-1.004568769635799\n", 38 | "E\n", 39 | "['G', 'C', 'B']\n", 40 | "['E', 'D', 'E']\n", 41 | "['D', 'I', 'G', 'H', 'E', 'B', 'C', 'F', 'A']\n" 42 | ] 43 | } 44 | ], 45 | "source": [ 46 | "import random\n", 47 | "\n", 48 | "# random float in [0,1)\n", 49 | "a = random.random()\n", 50 | "print(a)\n", 51 | "\n", 52 | "# random float in range [a,b]\n", 53 | "a = random.uniform(1,10)\n", 54 | "print(a)\n", 55 | "\n", 56 | "# random integer in range [a,b]. b is included\n", 57 | "a = random.randint(1,10)\n", 58 | "print(a)\n", 59 | "\n", 60 | "# random integer in range [a,b). b is excluded\n", 61 | "a = random.randrange(1,10)\n", 62 | "print(a)\n", 63 | "\n", 64 | "# random float from a normal distribution with mu and sigma\n", 65 | "a = random.normalvariate(0, 1)\n", 66 | "print(a)\n", 67 | "\n", 68 | "# choose a random element from a sequence\n", 69 | "a = random.choice(list(\"ABCDEFGHI\"))\n", 70 | "print(a)\n", 71 | "\n", 72 | "# choose k unique random elements from a sequence\n", 73 | "a = random.sample(list(\"ABCDEFGHI\"), 3)\n", 74 | "print(a)\n", 75 | "\n", 76 | "# choose k elements with replacement, and return k sized list\n", 77 | "a = random.choices(list(\"ABCDEFGHI\"),k=3)\n", 78 | "print(a)\n", 79 | "\n", 80 | "# shuffle list in place\n", 81 | "a = list(\"ABCDEFGHI\")\n", 82 | "random.shuffle(a)\n", 83 | "print(a)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "## The seed generator" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "With `random.seed()`, you can make results reproducible, and the chain of calls after `random.seed()` will produce the same trail of data. The sequence of random numbers becomes deterministic, or completely determined by the seed value." 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 52, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "name": "stdout", 107 | "output_type": "stream", 108 | "text": [ 109 | "Seeding with 1...\n", 110 | "\n", 111 | "0.13436424411240122\n", 112 | "8.626903632435095\n", 113 | "B\n", 114 | "\n", 115 | "Re-seeding with 42...\n", 116 | "\n", 117 | "0.6394267984578837\n", 118 | "1.2250967970040025\n", 119 | "E\n", 120 | "\n", 121 | "Re-seeding with 1...\n", 122 | "\n", 123 | "0.13436424411240122\n", 124 | "8.626903632435095\n", 125 | "B\n", 126 | "\n", 127 | "Re-seeding with 42...\n", 128 | "\n", 129 | "0.6394267984578837\n", 130 | "1.2250967970040025\n", 131 | "E\n" 132 | ] 133 | } 134 | ], 135 | "source": [ 136 | "print('Seeding with 1...\\n')\n", 137 | "\n", 138 | "random.seed(1)\n", 139 | "print(random.random())\n", 140 | "print(random.uniform(1,10))\n", 141 | "print(random.choice(list(\"ABCDEFGHI\")))\n", 142 | "\n", 143 | "print('\\nRe-seeding with 42...\\n')\n", 144 | "random.seed(42) # Re-seed\n", 145 | "\n", 146 | "print(random.random())\n", 147 | "print(random.uniform(1,10))\n", 148 | "print(random.choice(list(\"ABCDEFGHI\")))\n", 149 | "\n", 150 | "print('\\nRe-seeding with 1...\\n')\n", 151 | "random.seed(1) # Re-seed\n", 152 | "\n", 153 | "print(random.random())\n", 154 | "print(random.uniform(1,10))\n", 155 | "print(random.choice(list(\"ABCDEFGHI\")))\n", 156 | "\n", 157 | "print('\\nRe-seeding with 42...\\n')\n", 158 | "random.seed(42) # Re-seed\n", 159 | "\n", 160 | "print(random.random())\n", 161 | "print(random.uniform(1,10))\n", 162 | "print(random.choice(list(\"ABCDEFGHI\")))" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "## The secrets module\n", 170 | "The secrets module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. \n", 171 | "In particularly, secrets should be used in preference to the default pseudo-random number generator in the random module, which is designed for modelling and simulation, not security or cryptography." 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 53, 177 | "metadata": {}, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "6\n", 184 | "6\n", 185 | "E\n" 186 | ] 187 | } 188 | ], 189 | "source": [ 190 | "import secrets\n", 191 | "\n", 192 | "# random integer in range [0, n).\n", 193 | "a = secrets.randbelow(10)\n", 194 | "print(a)\n", 195 | "\n", 196 | "# return an integer with k random bits.\n", 197 | "a = secrets.randbits(5)\n", 198 | "print(a)\n", 199 | "\n", 200 | "# choose a random element from a sequence\n", 201 | "a = secrets.choice(list(\"ABCDEFGHI\"))\n", 202 | "print(a)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "## Random numbers with NumPy\n", 210 | "Create random numbers for nd arrays. The NumPy pseudorandom number generator is different from the Python standard library pseudorandom number generator. \n", 211 | "Importantly, seeding the Python pseudorandom number generator does not impact the NumPy pseudorandom number generator. It must be seeded and used separately." 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 2, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "[4.17022005e-01 7.20324493e-01 1.14374817e-04]\n", 224 | "[4.17022005e-01 7.20324493e-01 1.14374817e-04]\n", 225 | "[[5 0 0]\n", 226 | " [1 7 6]\n", 227 | " [9 2 4]\n", 228 | " [5 2 4]\n", 229 | " [2 4 7]]\n", 230 | "[-2.29230928 -1.41555249 0.8858294 0.63190187 0.04026035]\n", 231 | "[[4 5 6]\n", 232 | " [7 8 9]\n", 233 | " [1 2 3]]\n" 234 | ] 235 | } 236 | ], 237 | "source": [ 238 | "import numpy as np\n", 239 | "\n", 240 | "np.random.seed(1)\n", 241 | "# rand(d0,d1,…,dn)\n", 242 | "# generate nd array with random floats, arrays has size (d0,d1,…,dn)\n", 243 | "print(np.random.rand(3))\n", 244 | "# reset the seed\n", 245 | "np.random.seed(1)\n", 246 | "print(np.random.rand(3))\n", 247 | "\n", 248 | "# generate nd array with random integers in range [a,b) with size n\n", 249 | "values = np.random.randint(0, 10, (5,3))\n", 250 | "print(values)\n", 251 | "\n", 252 | "# generate nd array with Gaussian values, array has size (d0,d1,…,dn)\n", 253 | "# values from standard normal distribution with mean 0.0 and standard deviation 1.0\n", 254 | "values = np.random.randn(5)\n", 255 | "print(values)\n", 256 | "\n", 257 | "# randomly shuffle a nd array.\n", 258 | "# only shuffles the array along the first axis of a multi-dimensional array\n", 259 | "arr = np.array([[1,2,3], [4,5,6], [7,8,9]])\n", 260 | "np.random.shuffle(arr)\n", 261 | "print(arr)" 262 | ] 263 | } 264 | ], 265 | "metadata": { 266 | "kernelspec": { 267 | "display_name": "Python [conda env:pythonengineer_env]", 268 | "language": "python", 269 | "name": "conda-env-pythonengineer_env-py" 270 | }, 271 | "language_info": { 272 | "codemirror_mode": { 273 | "name": "ipython", 274 | "version": 3 275 | }, 276 | "file_extension": ".py", 277 | "mimetype": "text/x-python", 278 | "name": "python", 279 | "nbconvert_exporter": "python", 280 | "pygments_lexer": "ipython3", 281 | "version": "3.7.3" 282 | } 283 | }, 284 | "nbformat": 4, 285 | "nbformat_minor": 2 286 | } 287 | -------------------------------------------------------------------------------- /advanced-python/14-Generators.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Generators\n", 8 | "Generators are functions that can be paused and resumed on the fly, returning an object that can be iterated over. Unlike lists, they are lazy and thus produce items one at a time and only when asked. So they are much more memory efficient when dealing with large datasets. \n", 9 | "A generator is defined like a normal function but with the `yield` statement instead of `return`.\n", 10 | "```python\n", 11 | "def my_generator():\n", 12 | " yield 1\n", 13 | " yield 2\n", 14 | " yield 3\n", 15 | "```" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "## Execution of a generator function\n", 23 | "Calling the function does not execute it. Instead, the function returns a generator object which is used to control execution. Generator objects execute when `next()` is called. When calling `next()` the first time, execution begins at the start of the function and continues until the first `yield` statement where the value to the right of the statement is returned. Subsequent calls to `next()` continue from the `yield` statement (and loop around) until another `yield` is reached. If `yield` is not called because of a condition or the end is reached, a `StopIteration exception` is raised:" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 1, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "name": "stdout", 33 | "output_type": "stream", 34 | "text": [ 35 | "Starting\n", 36 | "3\n", 37 | "2\n", 38 | "1\n" 39 | ] 40 | }, 41 | { 42 | "ename": "StopIteration", 43 | "evalue": "", 44 | "output_type": "error", 45 | "traceback": [ 46 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 47 | "\u001b[0;31mStopIteration\u001b[0m Traceback (most recent call last)", 48 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;31m# this will raise a StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 49 | "\u001b[0;31mStopIteration\u001b[0m: " 50 | ] 51 | } 52 | ], 53 | "source": [ 54 | "def countdown(num):\n", 55 | " print('Starting')\n", 56 | " while num > 0:\n", 57 | " yield num\n", 58 | " num -= 1\n", 59 | "\n", 60 | "# this will not print 'Starting'\n", 61 | "cd = countdown(3)\n", 62 | "\n", 63 | "# this will print 'Starting' and the first value\n", 64 | "print(next(cd))\n", 65 | "\n", 66 | "# will print the next values\n", 67 | "print(next(cd))\n", 68 | "print(next(cd))\n", 69 | "\n", 70 | "# this will raise a StopIteration\n", 71 | "print(next(cd))" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 2, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "Starting\n", 84 | "3\n", 85 | "2\n", 86 | "1\n" 87 | ] 88 | } 89 | ], 90 | "source": [ 91 | "# you can iterate over a generator object with a for in loop\n", 92 | "cd = countdown(3)\n", 93 | "for x in cd:\n", 94 | " print(x)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 3, 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "name": "stdout", 104 | "output_type": "stream", 105 | "text": [ 106 | "Starting\n", 107 | "6\n", 108 | "Starting\n", 109 | "[1, 2, 3]\n" 110 | ] 111 | } 112 | ], 113 | "source": [ 114 | "# you can use it for functions that take iterables as input\n", 115 | "cd = countdown(3)\n", 116 | "sum_cd = sum(cd)\n", 117 | "print(sum_cd)\n", 118 | "\n", 119 | "cd = countdown(3)\n", 120 | "sorted_cd = sorted(cd)\n", 121 | "print(sorted_cd)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "## Big advantage: Generators save memory!\n", 129 | "Since the values are generated lazily, i.e. only when needed, it saves a lot of memory, especially when working with large data. Furthermore, we do not need to wait until all the elements have been generated before we start to use them." 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 4, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "499999500000\n", 142 | "8697464 bytes\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "# without a generator, the complete sequence has to be stored here in a list\n", 148 | "def firstn(n):\n", 149 | " num, nums = 0, []\n", 150 | " while num < n:\n", 151 | " nums.append(num)\n", 152 | " num += 1\n", 153 | " return nums\n", 154 | "\n", 155 | "sum_of_first_n = sum(firstn(1000000))\n", 156 | "print(sum_of_first_n)\n", 157 | "import sys\n", 158 | "print(sys.getsizeof(firstn(1000000)), \"bytes\")" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 9, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "name": "stdout", 168 | "output_type": "stream", 169 | "text": [ 170 | "499999500000\n", 171 | "120 bytes\n" 172 | ] 173 | } 174 | ], 175 | "source": [ 176 | "# with a generator, no additional sequence is needed to store the numbers\n", 177 | "def firstn(n):\n", 178 | " num = 0\n", 179 | " while num < n:\n", 180 | " yield num\n", 181 | " num += 1\n", 182 | "\n", 183 | "sum_of_first_n = sum(firstn(1000000))\n", 184 | "print(sum_of_first_n)\n", 185 | "import sys\n", 186 | "print(sys.getsizeof(firstn(1000000)), \"bytes\")" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": {}, 192 | "source": [ 193 | "## Another example: Fibonacci numbers" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 6, 199 | "metadata": {}, 200 | "outputs": [ 201 | { 202 | "name": "stdout", 203 | "output_type": "stream", 204 | "text": [ 205 | "[0, 1, 1, 2, 3, 5, 8, 13, 21]\n" 206 | ] 207 | } 208 | ], 209 | "source": [ 210 | "def fibonacci(limit):\n", 211 | " a, b = 0, 1 # first two fibonacci numbers\n", 212 | " while a < limit:\n", 213 | " yield a\n", 214 | " a, b = b, a + b\n", 215 | "\n", 216 | "fib = fibonacci(30)\n", 217 | "# generator objects can be converted to a list (only used for printing here)\n", 218 | "print(list(fib))" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "## Generator expressions\n", 226 | "Just like list comprehensions, generators can be written in the same syntax except with parenthesis instead of square brackets. Be careful not to mix them up, since generator expressions are often slower than list comprehensions because of the overhead of function calls (https://stackoverflow.com/questions/11964130/list-comprehension-vs-generator-expressions-weird-timeit-results/11964478#11964478)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 7, 232 | "metadata": {}, 233 | "outputs": [ 234 | { 235 | "name": "stdout", 236 | "output_type": "stream", 237 | "text": [ 238 | "120\n", 239 | "4272\n" 240 | ] 241 | } 242 | ], 243 | "source": [ 244 | "# generator expression\n", 245 | "mygenerator = (i for i in range(1000) if i % 2 == 0)\n", 246 | "print(sys.getsizeof(mygenerator))\n", 247 | "\n", 248 | "# list comprehension\n", 249 | "mylist = [i for i in range(1000) if i % 2 == 0]\n", 250 | "print(sys.getsizeof(mylist))" 251 | ] 252 | }, 253 | { 254 | "cell_type": "markdown", 255 | "metadata": {}, 256 | "source": [ 257 | "## Concept behind a generator\n", 258 | "This class implements our generator as an iterable object. It has to implement `__iter__` and `__next__` to make it iterable, keep track of the current state (the current number in this case), and take care of a `StopIteration`. It can be used to understand the concept behind generators. However, there is a lot of boilerplate code, and the logic is not as clear as with a simple function using the `yield` keyword. " 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": 8, 264 | "metadata": {}, 265 | "outputs": [ 266 | { 267 | "name": "stdout", 268 | "output_type": "stream", 269 | "text": [ 270 | "499999500000\n" 271 | ] 272 | } 273 | ], 274 | "source": [ 275 | "class firstn:\n", 276 | " def __init__(self, n):\n", 277 | " self.n = n\n", 278 | " self.num = 0\n", 279 | " \n", 280 | " def __iter__(self):\n", 281 | " return self\n", 282 | " \n", 283 | " def __next__(self):\n", 284 | " if self.num < self.n:\n", 285 | " cur = self.num\n", 286 | " self.num += 1\n", 287 | " return cur\n", 288 | " else:\n", 289 | " raise StopIteration()\n", 290 | " \n", 291 | "firstn_object = firstn(1000000)\n", 292 | "print(sum(firstn_object))" 293 | ] 294 | } 295 | ], 296 | "metadata": { 297 | "kernelspec": { 298 | "display_name": "Python [conda env:pythonengineer_env]", 299 | "language": "python", 300 | "name": "conda-env-pythonengineer_env-py" 301 | } 302 | }, 303 | "nbformat": 4, 304 | "nbformat_minor": 2 305 | } 306 | -------------------------------------------------------------------------------- /advanced-python/06-Collections.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Collections\n", 8 | "The collections module in Python implements specialized container datatypes providing alternatives to Python’s general purpose built-in containers, dict, list, set, and tuple. \n", 9 | "The following tools exist: \n", 10 | "- namedtuple : factory function for creating tuple subclasses with named fields\n", 11 | "- OrderedDict : dict subclass that remembers the order entries were added \n", 12 | "- Counter : dict subclass for counting hashable objects \n", 13 | "- defaultdict : dict subclass that calls a factory function to supply missing values\n", 14 | "- deque : list-like container with fast appends and pops on either end\n", 15 | "\n", 16 | "In Python 3 some more modules exist (ChainMap, UserDict, UserList, UserString). See https://docs.python.org/3/library/collections.html for further references." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Counter\n", 24 | "A counter is a container that stores elements as dictionary keys, and their counts are stored as dictionary values." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 3, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "Counter({'a': 5, 'b': 4, 'c': 3, 'd': 2, 'e': 1})\n", 37 | "dict_items([('a', 5), ('b', 4), ('c', 3), ('d', 2), ('e', 1)])\n", 38 | "dict_keys(['a', 'b', 'c', 'd', 'e'])\n", 39 | "dict_values([5, 4, 3, 2, 1])\n", 40 | "Counter({1: 4, 2: 3, 0: 2, 3: 2, 4: 1})\n", 41 | "[(1, 4)]\n", 42 | "[0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4]\n" 43 | ] 44 | } 45 | ], 46 | "source": [ 47 | "from collections import Counter\n", 48 | "a = \"aaaaabbbbcccdde\"\n", 49 | "my_counter = Counter(a)\n", 50 | "print(my_counter)\n", 51 | "\n", 52 | "print(my_counter.items())\n", 53 | "print(my_counter.keys())\n", 54 | "print(my_counter.values())\n", 55 | "\n", 56 | "my_list = [0, 1, 0, 1, 2, 1, 1, 3, 2, 3, 2, 4]\n", 57 | "my_counter = Counter(my_list)\n", 58 | "print(my_counter)\n", 59 | "\n", 60 | "# most common items\n", 61 | "print(my_counter.most_common(1))\n", 62 | "\n", 63 | "# Return an iterator over elements repeating each as many times as its count. \n", 64 | "# Elements are returned in arbitrary order.\n", 65 | "print(list(my_counter.elements()))" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "## namedtuple\n", 73 | "namedtuples are easy to create, lightweight object types. They assign meaning to each position in a tuple and allow for more readable, self-documenting code. They can be used wherever regular tuples are used, and they add the ability to access fields by name instead of position index." 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 19, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "Point(x=1, y=-4)\n", 86 | "('x', 'y')\n", 87 | "\n", 88 | "1 -4\n", 89 | "Tom 25\n" 90 | ] 91 | } 92 | ], 93 | "source": [ 94 | "from collections import namedtuple\n", 95 | "# create a namedtuple with its class name as string and its fields as string\n", 96 | "# fields have to be separated by comma or space in the given string\n", 97 | "Point = namedtuple('Point','x, y')\n", 98 | "pt = Point(1, -4)\n", 99 | "print(pt)\n", 100 | "print(pt._fields)\n", 101 | "print(type(pt))\n", 102 | "print(pt.x, pt.y)\n", 103 | "\n", 104 | "Person = namedtuple('Person','name, age')\n", 105 | "friend = Person(name='Tom', age=25)\n", 106 | "print(friend.name, friend.age)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## OrderedDict\n", 114 | "OrderedDicts are just like regular dictionaries but they remember the order that items were inserted. When iterating over an ordered dictionary, the items are returned in the order their keys were first added. If a new entry overwrites an existing entry, the original insertion position is left unchanged. They have become less important now that the built-in dict class gained the ability to remember insertion order (guaranteed since Python 3.7). But some differences still remain, e.g. the OrderedDict is designed to be good at reordering operations." 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 68, 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "name": "stdout", 124 | "output_type": "stream", 125 | "text": [ 126 | "{'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}\n", 127 | "OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)])\n", 128 | "a 1\n", 129 | "b 2\n", 130 | "c 3\n", 131 | "d 4\n", 132 | "e 5\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "from collections import OrderedDict\n", 138 | "ordinary_dict = {}\n", 139 | "ordinary_dict['a'] = 1\n", 140 | "ordinary_dict['b'] = 2\n", 141 | "ordinary_dict['c'] = 3\n", 142 | "ordinary_dict['d'] = 4\n", 143 | "ordinary_dict['e'] = 5\n", 144 | "# this may be in orbitrary order prior to Python 3.7\n", 145 | "print(ordinary_dict)\n", 146 | "\n", 147 | "ordered_dict = OrderedDict()\n", 148 | "ordered_dict['a'] = 1\n", 149 | "ordered_dict['b'] = 2\n", 150 | "ordered_dict['c'] = 3\n", 151 | "ordered_dict['d'] = 4\n", 152 | "ordered_dict['e'] = 5\n", 153 | "print(ordered_dict)\n", 154 | "# same functionality as with ordinary dict, but always ordered\n", 155 | "for k, v in ordinary_dict.items():\n", 156 | " print(k, v)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "## defaultdict\n", 164 | "The defaultdict is a container that's similar to the usual dict container, but the only difference is that a defaultdict will have a default value if that key has not been set yet. If you didn't use a defaultdict you'd have to check to see if that key exists, and if it doesn't, set it to what you want. " 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 49, 170 | "metadata": {}, 171 | "outputs": [ 172 | { 173 | "name": "stdout", 174 | "output_type": "stream", 175 | "text": [ 176 | "dict_items([('yellow', 1), ('blue', 2)])\n", 177 | "0\n", 178 | "dict_items([('yellow', [1, 3]), ('blue', [2, 4]), ('red', [5])])\n", 179 | "[]\n" 180 | ] 181 | } 182 | ], 183 | "source": [ 184 | "from collections import defaultdict\n", 185 | "\n", 186 | "# initialize with a default integer value, i.e 0\n", 187 | "d = defaultdict(int)\n", 188 | "d['yellow'] = 1\n", 189 | "d['blue'] = 2\n", 190 | "print(d.items())\n", 191 | "print(d['green'])\n", 192 | "\n", 193 | "# initialize with a default list value, i.e an empty list\n", 194 | "d = defaultdict(list)\n", 195 | "s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 5)]\n", 196 | "for k, v in s:\n", 197 | " d[k].append(v)\n", 198 | "\n", 199 | "print(d.items())\n", 200 | "print(d['green'])" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "## deque\n", 208 | "A deque is a double-ended queue. It can be used to add or remove elements from both ends. Deques support thread safe, memory efficient appends and pops from either side of the deque with approximately the same __O(1)__ performance in either direction. The more commonly used stacks and queues are degenerate forms of deques, where the inputs and outputs are restricted to a single end." 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 67, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "name": "stdout", 218 | "output_type": "stream", 219 | "text": [ 220 | "deque(['a', 'b'])\n", 221 | "deque(['c', 'a', 'b'])\n", 222 | "b\n", 223 | "deque(['c', 'a'])\n", 224 | "c\n", 225 | "deque(['a'])\n", 226 | "deque([])\n", 227 | "deque(['j', 'i', 'h', 'a', 'b', 'c', 'd', 'e', 'f', 'g'])\n", 228 | "1\n", 229 | "deque(['g', 'j', 'i', 'h', 'a', 'b', 'c', 'd', 'e', 'f'])\n", 230 | "deque(['i', 'h', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'j'])\n" 231 | ] 232 | } 233 | ], 234 | "source": [ 235 | "from collections import deque\n", 236 | "d = deque()\n", 237 | "\n", 238 | "# append() : add elements to the right end \n", 239 | "d.append('a')\n", 240 | "d.append('b')\n", 241 | "print(d)\n", 242 | "\n", 243 | "# appendleft() : add elements to the left end \n", 244 | "d.appendleft('c')\n", 245 | "print(d)\n", 246 | "\n", 247 | "# pop() : return and remove elements from the right\n", 248 | "print(d.pop())\n", 249 | "print(d)\n", 250 | "\n", 251 | "# popleft() : return and remove elements from the left\n", 252 | "print(d.popleft())\n", 253 | "print(d)\n", 254 | "\n", 255 | "# clear() : remove all elements\n", 256 | "d.clear()\n", 257 | "print(d)\n", 258 | "\n", 259 | "d = deque(['a', 'b', 'c', 'd'])\n", 260 | "\n", 261 | "# extend at right or left side\n", 262 | "d.extend(['e', 'f', 'g'])\n", 263 | "d.extendleft(['h', 'i', 'j']) # note that 'j' is now at the left most position\n", 264 | "print(d)\n", 265 | "\n", 266 | "# count(x) : returns the number of found elements\n", 267 | "print(d.count('h'))\n", 268 | "\n", 269 | "# rotate 1 positions to the right\n", 270 | "d.rotate(1)\n", 271 | "print(d)\n", 272 | "\n", 273 | "# rotate 2 positions to the left\n", 274 | "d.rotate(-2)\n", 275 | "print(d)" 276 | ] 277 | } 278 | ], 279 | "metadata": { 280 | "kernelspec": { 281 | "display_name": "Python [conda env:pythonengineer_env]", 282 | "language": "python", 283 | "name": "conda-env-pythonengineer_env-py" 284 | }, 285 | "language_info": { 286 | "codemirror_mode": { 287 | "name": "ipython", 288 | "version": 3 289 | }, 290 | "file_extension": ".py", 291 | "mimetype": "text/x-python", 292 | "name": "python", 293 | "nbconvert_exporter": "python", 294 | "pygments_lexer": "ipython3", 295 | "version": "3.7.3" 296 | } 297 | }, 298 | "nbformat": 4, 299 | "nbformat_minor": 2 300 | } 301 | -------------------------------------------------------------------------------- /advanced-python/19-The Asterisk(*).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# The Asterisk (`*`) in Python and its use cases\n", 8 | "The asterisk sign (`*`) can be used for different cases in Python:\n", 9 | "- Multiplication and power operations\n", 10 | "- Creation of list, tuple, or string with repeated elements\n", 11 | "- `*args` , `**kwargs` , and keyword-only parameters\n", 12 | "- Unpacking lists/tuples/dictionaries for function arguments\n", 13 | "- Unpacking containers\n", 14 | "- Merging containers into list / Merge dictionaries" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## Multiplication and power operations" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 46, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "35\n", 34 | "16\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "# multiplication\n", 40 | "result = 7 * 5\n", 41 | "print(result)\n", 42 | "\n", 43 | "# power operation\n", 44 | "result = 2 ** 4\n", 45 | "print(result)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Creation of list, tuple, or string with repeated elements" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 47, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", 65 | "[1, 2, 1, 2, 1, 2, 1, 2, 1, 2]\n", 66 | "(0, 0, 0, 0, 0, 0, 0, 0, 0, 0)\n", 67 | "(1, 2, 1, 2, 1, 2, 1, 2, 1, 2)\n", 68 | "AAAAAAAAAA\n", 69 | "ABABABABAB\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "# list\n", 75 | "zeros = [0] * 10\n", 76 | "onetwos = [1, 2] * 5\n", 77 | "print(zeros)\n", 78 | "print(onetwos)\n", 79 | "\n", 80 | "# tuple\n", 81 | "zeros = (0,) * 10\n", 82 | "onetwos = (1, 2) * 5\n", 83 | "print(zeros)\n", 84 | "print(onetwos)\n", 85 | "\n", 86 | "# string\n", 87 | "A_string = \"A\" * 10\n", 88 | "AB_string = \"AB\" * 5\n", 89 | "print(A_string)\n", 90 | "print(AB_string)" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "## `*args` , `**kwargs` , and keyword-only arguments\n", 98 | "- Use `*args` for variable-length arguments\n", 99 | "- Use `**kwargs` for variable-length keyword arguments\n", 100 | "- Use `*,` followed by more function parameters to enforce keyword-only arguments" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 48, 106 | "metadata": {}, 107 | "outputs": [ 108 | { 109 | "name": "stdout", 110 | "output_type": "stream", 111 | "text": [ 112 | "Hey\n", 113 | "3\n", 114 | "[0, 1, 2]\n", 115 | "name Alex\n", 116 | "age 8\n", 117 | "Michael\n", 118 | "5\n" 119 | ] 120 | } 121 | ], 122 | "source": [ 123 | "def my_function(*args, **kwargs):\n", 124 | " for arg in args:\n", 125 | " print(arg)\n", 126 | " for key in kwargs:\n", 127 | " print(key, kwargs[key])\n", 128 | " \n", 129 | "my_function(\"Hey\", 3, [0, 1, 2], name=\"Alex\", age=8)\n", 130 | "\n", 131 | "# Parameters after '*' or '*identifier' are keyword-only parameters and may only be passed using keyword arguments.\n", 132 | "def my_function2(name, *, age):\n", 133 | " print(name)\n", 134 | " print(age)\n", 135 | "\n", 136 | "# my_function2(\"Michael\", 5) --> this would raise a TypeError\n", 137 | "my_function2(\"Michael\", age=5)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "## Unpacking for function arguments\n", 145 | "- Lists/tuples/sets/strings can be unpacked into function arguments with one `*` if the length matches the parameters.\n", 146 | "- Dictionaries can be unpacked with two `**` if the length and the keys match the parameters." 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 49, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "1 2 3\n", 159 | "A B C\n", 160 | "4 5 6\n" 161 | ] 162 | } 163 | ], 164 | "source": [ 165 | "def foo(a, b, c):\n", 166 | " print(a, b, c)\n", 167 | "\n", 168 | "# length must match\n", 169 | "my_list = [1, 2, 3]\n", 170 | "foo(*my_list)\n", 171 | "\n", 172 | "my_string = \"ABC\"\n", 173 | "foo(*my_string)\n", 174 | "\n", 175 | "# length and keys must match\n", 176 | "my_dict = {'a': 4, 'b': 5, 'c': 6}\n", 177 | "foo(**my_dict)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "## Unpacking containers\n", 185 | "Unpack the elements of a list, tuple, or set into single and multiple remaining elements.\n", 186 | "Note that multiple elements are combined in a list, even if the unpacked container is a tuple or a set." 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 50, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "[1, 2, 3, 4, 5, 6, 7]\n", 199 | "8\n", 200 | "\n", 201 | "1\n", 202 | "[2, 3, 4, 5, 6, 7, 8]\n", 203 | "\n", 204 | "1\n", 205 | "[2, 3, 4, 5, 6, 7]\n", 206 | "8\n" 207 | ] 208 | } 209 | ], 210 | "source": [ 211 | "numbers = (1, 2, 3, 4, 5, 6, 7, 8)\n", 212 | "\n", 213 | "*beginning, last = numbers\n", 214 | "print(beginning)\n", 215 | "print(last)\n", 216 | "\n", 217 | "print()\n", 218 | "\n", 219 | "first, *end = numbers\n", 220 | "print(first)\n", 221 | "print(end)\n", 222 | "\n", 223 | "print()\n", 224 | "first, *middle, last = numbers\n", 225 | "print(first)\n", 226 | "print(middle)\n", 227 | "print(last)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "## Merge iterables into a list / Merge dictionaries\n", 235 | "This is possible since Python 3.5 thanks to PEP 448 (https://www.python.org/dev/peps/pep-0448/)." 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 51, 241 | "metadata": {}, 242 | "outputs": [ 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "[1, 2, 3, 4, 5, 6]\n", 248 | "{'one': 1, 'two': 2, 'three': 3, 'four': 4}\n" 249 | ] 250 | } 251 | ], 252 | "source": [ 253 | "# dump iterables into a list and merge them\n", 254 | "my_tuple = (1, 2, 3)\n", 255 | "my_set = {4, 5, 6}\n", 256 | "my_list = [*my_tuple, *my_set]\n", 257 | "print(my_list)\n", 258 | "\n", 259 | "# merge two dictionaries with dict unpacking\n", 260 | "dict_a = {'one': 1, 'two': 2}\n", 261 | "dict_b = {'three': 3, 'four': 4}\n", 262 | "dict_c = {**dict_a, **dict_b}\n", 263 | "print(dict_c)" 264 | ] 265 | }, 266 | { 267 | "cell_type": "markdown", 268 | "metadata": {}, 269 | "source": [ 270 | "But be careful with the following merging solution. It does not work if the dictionary has any non-string keys: \n", 271 | "https://stackoverflow.com/questions/38987/how-to-merge-two-dictionaries-in-a-single-expression/39858#39858" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 52, 277 | "metadata": {}, 278 | "outputs": [ 279 | { 280 | "ename": "TypeError", 281 | "evalue": "keywords must be strings", 282 | "output_type": "error", 283 | "traceback": [ 284 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 285 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 286 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mdict_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'one'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'two'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mdict_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'four'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdict_c\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdict_a\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdict_b\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdict_c\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 287 | "\u001b[0;31mTypeError\u001b[0m: keywords must be strings" 288 | ] 289 | } 290 | ], 291 | "source": [ 292 | "dict_a = {'one': 1, 'two': 2}\n", 293 | "dict_b = {3: 3, 'four': 4}\n", 294 | "dict_c = dict(dict_a, **dict_b)\n", 295 | "print(dict_c)\n", 296 | "\n", 297 | "# this works:\n", 298 | "# dict_c = {**dict_a, **dict_b}" 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "metadata": {}, 304 | "source": [ 305 | "Recommended further readings:\n", 306 | "- https://treyhunner.com/2018/10/asterisks-in-python-what-they-are-and-how-to-use-them/\n", 307 | "- https://treyhunner.com/2016/02/how-to-merge-dictionaries-in-python/" 308 | ] 309 | } 310 | ], 311 | "metadata": { 312 | "kernelspec": { 313 | "display_name": "Python [conda env:pythonengineer_env]", 314 | "language": "python", 315 | "name": "conda-env-pythonengineer_env-py" 316 | }, 317 | "language_info": { 318 | "codemirror_mode": { 319 | "name": "ipython", 320 | "version": 3 321 | }, 322 | "file_extension": ".py", 323 | "mimetype": "text/x-python", 324 | "name": "python", 325 | "nbconvert_exporter": "python", 326 | "pygments_lexer": "ipython3", 327 | "version": "3.7.3" 328 | } 329 | }, 330 | "nbformat": 4, 331 | "nbformat_minor": 2 332 | } 333 | -------------------------------------------------------------------------------- /advanced-python/03-Dictionary.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Dictionaries\n", 8 | "A dictionary is a collection which is unordered, changeable and indexed. A dictionary consists of a collection of key-value pairs. Each key-value pair maps the key to its associated value. A dictionary is written in braces. Each key is separated from its value by a colon (:), and the items are separated by commas.\n", 9 | "\n", 10 | "`my_dict = {\"name\":\"Max\", \"age\":28, \"city\":\"New York\"}`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": {}, 16 | "source": [ 17 | "#### Create a dictionary\n", 18 | "Create a dictionary with braces, or with the built-in dict funtion." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "{'name': 'Max', 'age': 28, 'city': 'New York'}\n", 31 | "{'name': 'Lisa', 'age': 27, 'city': 'Boston'}\n" 32 | ] 33 | } 34 | ], 35 | "source": [ 36 | "my_dict = {\"name\":\"Max\", \"age\":28, \"city\":\"New York\"}\n", 37 | "print(my_dict)\n", 38 | "\n", 39 | "# or use the dict constructor, note: no quotes necessary for keys\n", 40 | "my_dict_2 = dict(name=\"Lisa\", age=27, city=\"Boston\")\n", 41 | "print(my_dict_2)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "#### Access items" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "Max\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "name_in_dict = my_dict[\"name\"]\n", 66 | "print(name_in_dict)\n", 67 | "\n", 68 | "# KeyError if no key is found\n", 69 | "# print(my_dict[\"lastname\"])" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "#### Add and change items\n", 77 | "Simply add or access a key and asign the value." 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "{'name': 'Max', 'age': 28, 'city': 'New York', 'email': 'max@xyz.com'}\n", 90 | "{'name': 'Max', 'age': 28, 'city': 'New York', 'email': 'coolmax@xyz.com'}\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "# add a new key\n", 96 | "my_dict[\"email\"] = \"max@xyz.com\"\n", 97 | "print(my_dict)\n", 98 | "\n", 99 | "# or overwrite the now existing key\n", 100 | "my_dict[\"email\"] = \"coolmax@xyz.com\"\n", 101 | "print(my_dict)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "#### Delete items" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 4, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "popped value: 28\n", 121 | "popped item: ('city', 'New York')\n", 122 | "{'name': 'Max'}\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "# delete a key-value pair\n", 128 | "del my_dict[\"email\"]\n", 129 | "\n", 130 | "# this returns the value and removes the key-value pair\n", 131 | "print(\"popped value:\", my_dict.pop(\"age\"))\n", 132 | "\n", 133 | "# return and removes the last inserted key-value pair \n", 134 | "# (in versions before Python 3.7 it removes an arbitrary pair)\n", 135 | "print(\"popped item:\", my_dict.popitem())\n", 136 | "\n", 137 | "print(my_dict)\n", 138 | "\n", 139 | "# clear() : remove all pairs\n", 140 | "# my_dict.clear()" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "#### Check for keys" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 5, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "Max\n", 160 | "No key found\n" 161 | ] 162 | } 163 | ], 164 | "source": [ 165 | "my_dict = {\"name\":\"Max\", \"age\":28, \"city\":\"New York\"}\n", 166 | "# use if .. in ..\n", 167 | "if \"name\" in my_dict:\n", 168 | " print(my_dict[\"name\"])\n", 169 | "\n", 170 | "# use try except\n", 171 | "try:\n", 172 | " print(my_dict[\"firstname\"])\n", 173 | "except KeyError:\n", 174 | " print(\"No key found\") " 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "#### Looping through dictionary" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 6, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "name Max\n", 194 | "age 28\n", 195 | "city New York\n", 196 | "name\n", 197 | "age\n", 198 | "city\n", 199 | "Max\n", 200 | "28\n", 201 | "New York\n", 202 | "name Max\n", 203 | "age 28\n", 204 | "city New York\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "# loop over keys\n", 210 | "for key in my_dict:\n", 211 | " print(key, my_dict[key])\n", 212 | "\n", 213 | "# loop over keys\n", 214 | "for key in my_dict.keys():\n", 215 | " print(key)\n", 216 | "\n", 217 | "# loop over values\n", 218 | "for value in my_dict.values():\n", 219 | " print(value)\n", 220 | "\n", 221 | "# loop over keys and values\n", 222 | "for key, value in my_dict.items():\n", 223 | " print(key, value)" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "#### Copy a dictionary\n", 231 | "Be careful when copying references." 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 7, 237 | "metadata": {}, 238 | "outputs": [ 239 | { 240 | "name": "stdout", 241 | "output_type": "stream", 242 | "text": [ 243 | "{'name': 'Lisa', 'age': 28, 'city': 'New York'}\n", 244 | "{'name': 'Lisa', 'age': 28, 'city': 'New York'}\n", 245 | "{'name': 'Lisa', 'age': 28, 'city': 'New York'}\n", 246 | "{'name': 'Max', 'age': 28, 'city': 'New York'}\n" 247 | ] 248 | } 249 | ], 250 | "source": [ 251 | "dict_org = {\"name\":\"Max\", \"age\":28, \"city\":\"New York\"}\n", 252 | "\n", 253 | "# this just copies the reference to the dict, so be careful\n", 254 | "dict_copy = dict_org\n", 255 | "\n", 256 | "# now modifying the copy also affects the original\n", 257 | "dict_copy[\"name\"] = \"Lisa\"\n", 258 | "print(dict_copy)\n", 259 | "print(dict_org)\n", 260 | "\n", 261 | "# use copy(), or dict(x) to actually copy the dict\n", 262 | "dict_org = {\"name\":\"Max\", \"age\":28, \"city\":\"New York\"}\n", 263 | "\n", 264 | "dict_copy = dict_org.copy()\n", 265 | "# dict_copy = dict(dict_org)\n", 266 | "\n", 267 | "# now modifying the copy does not affect the original\n", 268 | "dict_copy[\"name\"] = \"Lisa\"\n", 269 | "print(dict_copy)\n", 270 | "print(dict_org)" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": {}, 276 | "source": [ 277 | "#### Merge two dictionaries" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": 8, 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "name": "stdout", 287 | "output_type": "stream", 288 | "text": [ 289 | "{'name': 'Lisa', 'age': 27, 'email': 'max@xyz.com', 'city': 'Boston'}\n" 290 | ] 291 | } 292 | ], 293 | "source": [ 294 | "# Use the update() method to merge 2 dicts\n", 295 | "# existing keys are overwritten, new keys are added\n", 296 | "my_dict = {\"name\":\"Max\", \"age\":28, \"email\":\"max@xyz.com\"}\n", 297 | "my_dict_2 = dict(name=\"Lisa\", age=27, city=\"Boston\")\n", 298 | "\n", 299 | "my_dict.update(my_dict_2)\n", 300 | "print(my_dict)" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "#### Possible key types\n", 308 | "Any immutable type, like strings or numbers can be used as a key.\n", 309 | "Also, a tuple can be used if it contains only immutable elements." 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": 9, 315 | "metadata": {}, 316 | "outputs": [ 317 | { 318 | "name": "stdout", 319 | "output_type": "stream", 320 | "text": [ 321 | "9 36 81\n", 322 | "15\n" 323 | ] 324 | } 325 | ], 326 | "source": [ 327 | "# use numbers as key, but be careful\n", 328 | "my_dict = {3: 9, 6: 36, 9:81}\n", 329 | "# do not mistake the keys as indices of a list, e.g my_dict[0] is not possible here\n", 330 | "print(my_dict[3], my_dict[6], my_dict[9])\n", 331 | "\n", 332 | "# use a tuple with immutable elements (e.g. number, string)\n", 333 | "my_tuple = (8, 7)\n", 334 | "my_dict = {my_tuple: 15}\n", 335 | "\n", 336 | "print(my_dict[my_tuple])\n", 337 | "# print(my_dict[8, 7])\n", 338 | "\n", 339 | "# a list is not possible because it is not immutable\n", 340 | "# this will raise an Error:\n", 341 | "# my_list = [8, 7]\n", 342 | "# my_dict = {my_list: 15}" 343 | ] 344 | }, 345 | { 346 | "cell_type": "markdown", 347 | "metadata": {}, 348 | "source": [ 349 | "#### Nested dictionaries\n", 350 | "The values can also be container types (e.g. lists, tuples, dictionaries)." 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 10, 356 | "metadata": {}, 357 | "outputs": [ 358 | { 359 | "name": "stdout", 360 | "output_type": "stream", 361 | "text": [ 362 | "{'dictA': {'name': 'Max', 'age': 28}, 'dictB': {'name': 'Alex', 'age': 25}}\n" 363 | ] 364 | } 365 | ], 366 | "source": [ 367 | "my_dict_1 = {\"name\": \"Max\", \"age\": 28}\n", 368 | "my_dict_2 = {\"name\": \"Alex\", \"age\": 25}\n", 369 | "nested_dict = {\"dictA\": my_dict_1,\n", 370 | " \"dictB\": my_dict_2}\n", 371 | "print(nested_dict)" 372 | ] 373 | } 374 | ], 375 | "metadata": { 376 | "kernelspec": { 377 | "display_name": "Python [conda env:pythonengineer_env]", 378 | "language": "python", 379 | "name": "conda-env-pythonengineer_env-py" 380 | }, 381 | "language_info": { 382 | "codemirror_mode": { 383 | "name": "ipython", 384 | "version": 3 385 | }, 386 | "file_extension": ".py", 387 | "mimetype": "text/x-python", 388 | "name": "python", 389 | "nbconvert_exporter": "python", 390 | "pygments_lexer": "ipython3", 391 | "version": "3.7.3" 392 | } 393 | }, 394 | "nbformat": 4, 395 | "nbformat_minor": 2 396 | } 397 | -------------------------------------------------------------------------------- /advanced-python/21-Context manager.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Context managers and the 'with' statement\n", 8 | "Context managers are a great tool for resource management. They allow you to allocate and release resources precisely when you want to. A well-known example is the `with open()` statemtent:" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 21, 14 | "metadata": { 15 | "collapsed": true 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "with open('notes.txt', 'w') as f:\n", 20 | " f.write('some todo...')" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "This will open a file and makes sure to automatically close it after program execution leaves the context of the with statement. It also handles exceptions and makes sure to properly close the file even in case of an exception. Internally, the above code translates to something like this:" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 22, 33 | "metadata": { 34 | "collapsed": true 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "f = open('notes.txt', 'w')\n", 39 | "try:\n", 40 | " f.write('some todo...')\n", 41 | "finally:\n", 42 | " f.close()" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "We can see that using a context manager and the `with` statement is much shorter and more concise." 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "## Examples of context managers\n", 57 | "- Open and close files\n", 58 | "- open and close database connections\n", 59 | "- Acquire and release locks:\n", 60 | "\n", 61 | "```python\n", 62 | "from threading import Lock\n", 63 | "lock = Lock()\n", 64 | "\n", 65 | "# error-prone:\n", 66 | "lock.acquire()\n", 67 | "# do stuff\n", 68 | "# lock should always be released!\n", 69 | "lock.release()\n", 70 | "\n", 71 | "# Better:\n", 72 | "with lock:\n", 73 | " # do stuff\n", 74 | "```" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "## Implementing a context manager as a class\n", 82 | "To support the `with` statement for our own classes, we have to implement the `__enter__` and `__exit__` methods. Python calls `__enter__` when execution enters the context of the `with` statement. In here the resource should be acquired and returned. When execution leaves the context again, `__exit__` is called and the resource is freed up." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 23, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "name": "stdout", 92 | "output_type": "stream", 93 | "text": [ 94 | "init notes.txt\n", 95 | "enter\n", 96 | "doing stuff...\n", 97 | "exit\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "class ManagedFile:\n", 103 | " def __init__(self, filename):\n", 104 | " print('init', filename)\n", 105 | " self.filename = filename\n", 106 | " \n", 107 | " def __enter__(self):\n", 108 | " print('enter')\n", 109 | " self.file = open(self.filename, 'w')\n", 110 | " return self.file\n", 111 | " \n", 112 | " def __exit__(self, exc_type, exc_value, exc_traceback):\n", 113 | " if self.file:\n", 114 | " self.file.close()\n", 115 | " print('exit')\n", 116 | " \n", 117 | "with ManagedFile('notes.txt') as f:\n", 118 | " print('doing stuff...')\n", 119 | " f.write('some todo...')" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "#### Handling exceptions\n", 127 | "If an exception occurs, Python passes the type, value, and traceback to the `__exit__` method. It can handle the exception here. If anything other than `True` is returned by the `__exit__` method, then the exception is raised by the `with` statement." 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 24, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "name": "stdout", 137 | "output_type": "stream", 138 | "text": [ 139 | "init notes.txt\n", 140 | "enter\n", 141 | "doing stuff...\n", 142 | "exc: None None\n", 143 | "exit\n", 144 | "continuing...\n", 145 | "\n", 146 | "init notes2.txt\n", 147 | "enter\n", 148 | "doing stuff...\n", 149 | "exc: '_io.TextIOWrapper' object has no attribute 'do_something'\n", 150 | "exit\n" 151 | ] 152 | }, 153 | { 154 | "ename": "AttributeError", 155 | "evalue": "'_io.TextIOWrapper' object has no attribute 'do_something'", 156 | "output_type": "error", 157 | "traceback": [ 158 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", 159 | "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", 160 | "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'doing stuff...'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 28\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'some todo...'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 29\u001b[1;33m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdo_something\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 30\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'continuing...'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", 161 | "\u001b[1;31mAttributeError\u001b[0m: '_io.TextIOWrapper' object has no attribute 'do_something'" 162 | ] 163 | } 164 | ], 165 | "source": [ 166 | "class ManagedFile:\n", 167 | " def __init__(self, filename):\n", 168 | " print('init', filename)\n", 169 | " self.filename = filename\n", 170 | " \n", 171 | " def __enter__(self):\n", 172 | " print('enter')\n", 173 | " self.file = open(self.filename, 'w')\n", 174 | " return self.file\n", 175 | " \n", 176 | " def __exit__(self, exc_type, exc_value, exc_traceback):\n", 177 | " if self.file:\n", 178 | " self.file.close()\n", 179 | " print('exc:', exc_type, exc_value)\n", 180 | " print('exit')\n", 181 | "\n", 182 | "# No exception\n", 183 | "with ManagedFile('notes.txt') as f:\n", 184 | " print('doing stuff...')\n", 185 | " f.write('some todo...')\n", 186 | "print('continuing...')\n", 187 | "\n", 188 | "print()\n", 189 | "\n", 190 | "# Exception is raised, but the file can still be closed\n", 191 | "with ManagedFile('notes2.txt') as f:\n", 192 | " print('doing stuff...')\n", 193 | " f.write('some todo...')\n", 194 | " f.do_something()\n", 195 | "print('continuing...')" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": {}, 201 | "source": [ 202 | "We can handle the exception in the `__exit__` method and return `True`." 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 25, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "name": "stdout", 212 | "output_type": "stream", 213 | "text": [ 214 | "init notes2.txt\n", 215 | "enter\n", 216 | "doing stuff...\n", 217 | "Exception has been handled\n", 218 | "exit\n", 219 | "continuing...\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "class ManagedFile:\n", 225 | " def __init__(self, filename):\n", 226 | " print('init', filename)\n", 227 | " self.filename = filename\n", 228 | " \n", 229 | " def __enter__(self):\n", 230 | " print('enter')\n", 231 | " self.file = open(self.filename, 'w')\n", 232 | " return self.file\n", 233 | " \n", 234 | " def __exit__(self, exc_type, exc_value, exc_traceback):\n", 235 | " if self.file:\n", 236 | " self.file.close()\n", 237 | " if exc_type is not None:\n", 238 | " print('Exception has been handled')\n", 239 | " print('exit')\n", 240 | " return True\n", 241 | "\n", 242 | "\n", 243 | "with ManagedFile('notes2.txt') as f:\n", 244 | " print('doing stuff...')\n", 245 | " f.write('some todo...')\n", 246 | " f.do_something()\n", 247 | "print('continuing...')" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "## Implementing a context manager as a generator\n", 255 | "Instead of writing a class, we can also write a generator function and decorate it with the `contextlib.contextmanager` decorator. Then we can also call the function using a `with` statement. For this approach, the function must `yield` the resource in a `try` statement, and all the content of the `__exit__` method to free up the resource goes now inside the corresponding `finally` statement." 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 26, 261 | "metadata": { 262 | "collapsed": true 263 | }, 264 | "outputs": [], 265 | "source": [ 266 | "from contextlib import contextmanager\n", 267 | "\n", 268 | "@contextmanager\n", 269 | "def open_managed_file(filename):\n", 270 | " f = open(filename, 'w')\n", 271 | " try:\n", 272 | " yield f\n", 273 | " finally:\n", 274 | " f.close()\n", 275 | " \n", 276 | "with open_managed_file('notes.txt') as f:\n", 277 | " f.write('some todo...')" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | "The generator first acquires the resource. It then temporarily suspends its own execution and *yields* the resource so it can be used by the caller. When the caller leaves the `with` context, the generator continues to execute and frees up the resource in the `finally` statement." 285 | ] 286 | } 287 | ], 288 | "metadata": { 289 | "kernelspec": { 290 | "display_name": "Python [conda env:pythonengineer_env]", 291 | "language": "python", 292 | "name": "conda-env-pythonengineer_env-py" 293 | }, 294 | "language_info": { 295 | "codemirror_mode": { 296 | "name": "ipython", 297 | "version": 3 298 | }, 299 | "file_extension": ".py", 300 | "mimetype": "text/x-python", 301 | "name": "python", 302 | "nbconvert_exporter": "python", 303 | "pygments_lexer": "ipython3", 304 | "version": "3.7.3" 305 | } 306 | }, 307 | "nbformat": 4, 308 | "nbformat_minor": 2 309 | } 310 | -------------------------------------------------------------------------------- /advanced-python/05-Strings.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Strings\n", 8 | "A string is a sequence of characters. String literals in Python are enclosed by either double or single quotes.\n", 9 | "\n", 10 | "`my_string = 'Hello'`\n", 11 | "\n", 12 | "Python strings are immutable which means they cannot be changed after they are created." 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "#### Creation" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 1, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "name": "stdout", 29 | "output_type": "stream", 30 | "text": [ 31 | "I' m a 'Geek'\n", 32 | "Hello\n", 33 | "World\n", 34 | "Hello World\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "# use singe or double quotes\n", 40 | "my_string = 'Hello'\n", 41 | "my_string = \"Hello\"\n", 42 | "my_string = \"I' m a 'Geek'\"\n", 43 | "\n", 44 | "# escaping backslash\n", 45 | "my_string = 'I\\' m a \"Geek\"'\n", 46 | "my_string = 'I\\' m a \\'Geek\\''\n", 47 | "print(my_string)\n", 48 | "\n", 49 | "# triple quotes for multiline strings\n", 50 | "my_string = \"\"\"Hello\n", 51 | "World\"\"\"\n", 52 | "print(my_string)\n", 53 | "\n", 54 | "# backslash if you want to continue in the next line\n", 55 | "my_string = \"Hello \\\n", 56 | "World\"\n", 57 | "print(my_string)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "#### Access characters and substrings" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 2, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "name": "stdout", 74 | "output_type": "stream", 75 | "text": [ 76 | "H\n", 77 | "el\n", 78 | "Hello\n", 79 | "World\n", 80 | "HloWrd\n", 81 | "dlroW olleH\n" 82 | ] 83 | } 84 | ], 85 | "source": [ 86 | "my_string = \"Hello World\"\n", 87 | "\n", 88 | "# get character by referring to index\n", 89 | "b = my_string[0]\n", 90 | "print(b)\n", 91 | "\n", 92 | "# Substrings with slicing\n", 93 | "b = my_string[1:3] # Note that the last index is not included\n", 94 | "print(b)\n", 95 | "b = my_string[:5] # from beginning\n", 96 | "print(b)\n", 97 | "b = my_string[6:] # until the end\n", 98 | "print(b)\n", 99 | "b = my_string[::2] # start to end with every second item\n", 100 | "print(b)\n", 101 | "b = my_string[::-1] # reverse the string with a negative step:\n", 102 | "print(b)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "#### Concatenate two or more strings" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 3, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "Hello Tom\n" 122 | ] 123 | } 124 | ], 125 | "source": [ 126 | "# concat strings with +\n", 127 | "greeting = \"Hello\"\n", 128 | "name = \"Tom\"\n", 129 | "sentence = greeting + ' ' + name\n", 130 | "print(sentence)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "#### Iterating" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 6, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "name": "stdout", 147 | "output_type": "stream", 148 | "text": [ 149 | "H\n", 150 | "e\n", 151 | "l\n", 152 | "l\n", 153 | "o\n" 154 | ] 155 | } 156 | ], 157 | "source": [ 158 | "# Iterating over a string by using a for in loop\n", 159 | "my_string = 'Hello'\n", 160 | "for i in my_string:\n", 161 | " print(i)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "#### Check if a character or substring exists" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 7, 174 | "metadata": {}, 175 | "outputs": [ 176 | { 177 | "name": "stdout", 178 | "output_type": "stream", 179 | "text": [ 180 | "yes\n", 181 | "yes\n" 182 | ] 183 | } 184 | ], 185 | "source": [ 186 | "if \"e\" in \"Hello\":\n", 187 | " print(\"yes\")\n", 188 | "if \"llo\" in \"Hello\":\n", 189 | " print(\"yes\")" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": {}, 195 | "source": [ 196 | "#### Useful methods" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 11, 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "name": "stdout", 206 | "output_type": "stream", 207 | "text": [ 208 | "Hello World\n", 209 | "11\n", 210 | "HELLO WORLD\n", 211 | "hello world\n", 212 | "['how', 'are', 'you', 'doing']\n", 213 | "['one', 'two', 'three']\n", 214 | "True\n", 215 | "True\n", 216 | "4\n", 217 | "1\n", 218 | "Hello Universe\n", 219 | "How are you doing\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "my_string = \" Hello World \"\n", 225 | "\n", 226 | "# remove white space\n", 227 | "my_string = my_string.strip()\n", 228 | "print(my_string)\n", 229 | "\n", 230 | "# number of characters\n", 231 | "print(len(my_string))\n", 232 | "\n", 233 | "# Upper and lower cases\n", 234 | "print(my_string.upper())\n", 235 | "print(my_string.lower())\n", 236 | "\n", 237 | "# startswith and endswith\n", 238 | "print(\"hello\".startswith(\"he\"))\n", 239 | "print(\"hello\".endswith(\"llo\"))\n", 240 | "\n", 241 | "# find first index of a given substring, -1 otherwise\n", 242 | "print(\"Hello\".find(\"o\"))\n", 243 | "\n", 244 | "# count number of characters/substrings\n", 245 | "print(\"Hello\".count(\"e\"))\n", 246 | "\n", 247 | "# replace a substring with another string (only if the substring is found)\n", 248 | "# Note: The original string stays the same\n", 249 | "message = \"Hello World\"\n", 250 | "new_message = message.replace(\"World\", \"Universe\")\n", 251 | "print(new_message)\n", 252 | "\n", 253 | "# split the string into a list\n", 254 | "my_string = \"how are you doing\"\n", 255 | "a = my_string.split() # default argument is \" \"\n", 256 | "print(a)\n", 257 | "my_string = \"one,two,three\"\n", 258 | "a = my_string.split(\",\")\n", 259 | "print(a)\n", 260 | "\n", 261 | "# join elements of a list into a string\n", 262 | "my_list = ['How', 'are', 'you', 'doing']\n", 263 | "a = ' '.join(my_list) # the given string is the separator, e.g. ' ' between each argument\n", 264 | "print(a)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "metadata": {}, 270 | "source": [ 271 | "#### Format\n", 272 | "New style is with .format() and old style is with % operator." 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 12, 278 | "metadata": {}, 279 | "outputs": [ 280 | { 281 | "name": "stdout", 282 | "output_type": "stream", 283 | "text": [ 284 | "Hello Bob and Tom\n", 285 | "Hello Bob and Tom\n", 286 | "The integer value is 2\n", 287 | "The float value is 2.123\n", 288 | "The float value is 2.123400e+00\n", 289 | "The binary value is 10\n", 290 | "Hello Bob and Tom\n", 291 | "The decimal value is 10\n", 292 | "The float value is 10.123450\n", 293 | "The float value is 10.12\n" 294 | ] 295 | } 296 | ], 297 | "source": [ 298 | "# use braces as placeholders\n", 299 | "a = \"Hello {0} and {1}\".format(\"Bob\", \"Tom\")\n", 300 | "print(a)\n", 301 | "\n", 302 | "# the positions are optional for the default order\n", 303 | "a = \"Hello {} and {}\".format(\"Bob\", \"Tom\")\n", 304 | "print(a)\n", 305 | "\n", 306 | "a = \"The integer value is {}\".format(2)\n", 307 | "print(a)\n", 308 | "\n", 309 | "# some special format rules for numbers\n", 310 | "a = \"The float value is {0:.3f}\".format(2.1234)\n", 311 | "print(a)\n", 312 | "a = \"The float value is {0:e}\".format(2.1234)\n", 313 | "print(a)\n", 314 | "a = \"The binary value is {0:b}\".format(2)\n", 315 | "print(a)\n", 316 | "\n", 317 | "# old style formatting by using % operator\n", 318 | "print(\"Hello %s and %s\" % (\"Bob\", \"Tom\")) # must be a tuple for multiple arguments\n", 319 | "val = 3.14159265359\n", 320 | "print(\"The decimal value is %d\" % val)\n", 321 | "print(\"The float value is %f\" % val)\n", 322 | "print(\"The float value is %.2f\" % val)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "#### f-Strings\n", 330 | "New since Python 3.6. Use the variables directly inside the braces." 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": 13, 336 | "metadata": {}, 337 | "outputs": [ 338 | { 339 | "name": "stdout", 340 | "output_type": "stream", 341 | "text": [ 342 | "Hello, Eric. You are 25.\n", 343 | "Pi is 3.142\n", 344 | "The value is 120\n" 345 | ] 346 | } 347 | ], 348 | "source": [ 349 | "name = \"Eric\"\n", 350 | "age = 25\n", 351 | "a = f\"Hello, {name}. You are {age}.\"\n", 352 | "print(a)\n", 353 | "pi = 3.14159\n", 354 | "a = f\"Pi is {pi:.3f}\"\n", 355 | "print(a)\n", 356 | "# f-Strings are evaluated at runtime, which allows expressions\n", 357 | "a = f\"The value is {2*60}\"\n", 358 | "print(a)" 359 | ] 360 | }, 361 | { 362 | "cell_type": "markdown", 363 | "metadata": {}, 364 | "source": [ 365 | "#### More on immutability and concatenation" 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "execution_count": 14, 371 | "metadata": {}, 372 | "outputs": [ 373 | { 374 | "name": "stdout", 375 | "output_type": "stream", 376 | "text": [ 377 | "concat string with + : 0.34527\n", 378 | "concat string with join(): 0.01191\n" 379 | ] 380 | } 381 | ], 382 | "source": [ 383 | "# since a string is immutable, adding strings with +, or += always \n", 384 | "# creates a new string, and therefore is expensive for multiple operations\n", 385 | "# --> join method is much faster\n", 386 | "from timeit import default_timer as timer\n", 387 | "my_list = [\"a\"] * 1000000\n", 388 | "\n", 389 | "# bad\n", 390 | "start = timer()\n", 391 | "a = \"\"\n", 392 | "for i in my_list:\n", 393 | " a += i\n", 394 | "end = timer()\n", 395 | "print(\"concatenate string with + : %.5f\" % (end - start))\n", 396 | "\n", 397 | "# good\n", 398 | "start = timer()\n", 399 | "a = \"\".join(my_list)\n", 400 | "end = timer()\n", 401 | "print(\"concatenate string with join(): %.5f\" % (end - start))" 402 | ] 403 | } 404 | ], 405 | "metadata": { 406 | "kernelspec": { 407 | "display_name": "Python [conda env:pythonengineer_env]", 408 | "language": "python", 409 | "name": "conda-env-pythonengineer_env-py" 410 | }, 411 | "language_info": { 412 | "codemirror_mode": { 413 | "name": "ipython", 414 | "version": 3 415 | }, 416 | "file_extension": ".py", 417 | "mimetype": "text/x-python", 418 | "name": "python", 419 | "nbconvert_exporter": "python", 420 | "pygments_lexer": "ipython3", 421 | "version": "3.7.3" 422 | } 423 | }, 424 | "nbformat": 4, 425 | "nbformat_minor": 2 426 | } 427 | -------------------------------------------------------------------------------- /advanced-python/02-Tuple.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tuple\n", 8 | "\n", 9 | "A tuple is a collection of objects which is ordered and immutable. Tuples are similar to lists, the main difference ist the immutability. In Python tuples are written with round brackets and comma separated values.\n", 10 | "\n", 11 | "`my_tuple = (\"Max\", 28, \"New York\")`\n", 12 | "\n", 13 | "#### Reasons to use a tuple over a list\n", 14 | "\n", 15 | "- Generally used for objects that belong together.\n", 16 | "- Use tuple for heterogeneous (different) datatypes and list for homogeneous (similar) datatypes.\n", 17 | "- Since tuple are immutable, iterating through tuple is slightly faster than with list.\n", 18 | "- Tuples with their immutable elements can be used as key for a dictionary. This is not possible with lists.\n", 19 | "- If you have data that doesn't change, implementing it as tuple will guarantee that it remains write-protected." 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "#### Create a tuple\n", 27 | "\n", 28 | "Tuples are created with round brackets and comma separated values. Or use the built-in tuple function." 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "('Max', 28, 'New York')\n", 41 | "('Linda', 25, 'Miami')\n", 42 | "(25,)\n", 43 | "(1, 2, 3)\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "tuple_1 = (\"Max\", 28, \"New York\")\n", 49 | "tuple_2 = \"Linda\", 25, \"Miami\" # Parentheses are optional\n", 50 | "\n", 51 | "# Special case: a tuple with only one element needs to have a comma at the end, \n", 52 | "# otherwise it is not recognized as tuple\n", 53 | "tuple_3 = (25,)\n", 54 | "print(tuple_1)\n", 55 | "print(tuple_2)\n", 56 | "print(tuple_3)\n", 57 | "\n", 58 | "# Or convert an iterable (list, dict, string) with the built-in tuple function\n", 59 | "tuple_4 = tuple([1,2,3])\n", 60 | "print(tuple_4)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "#### Access elements\n", 68 | "You access the tuple items by referring to the index number. Note that the indices start at 0." 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [ 76 | { 77 | "name": "stdout", 78 | "output_type": "stream", 79 | "text": [ 80 | "Max\n", 81 | "New York\n" 82 | ] 83 | } 84 | ], 85 | "source": [ 86 | "item = tuple_1[0]\n", 87 | "print(item)\n", 88 | "# You can also use negative indexing, e.g -1 refers to the last item,\n", 89 | "# -2 to the second last item, and so on\n", 90 | "item = tuple_1[-1]\n", 91 | "print(item)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "#### Add or change items \n", 99 | "Not possible and will raise a TypeError." 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 5, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "ename": "TypeError", 109 | "evalue": "'tuple' object does not support item assignment", 110 | "output_type": "error", 111 | "traceback": [ 112 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 113 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 114 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtuple_1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Boston\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 115 | "\u001b[0;31mTypeError\u001b[0m: 'tuple' object does not support item assignment" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "tuple_1[2] = \"Boston\"" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "#### Delete a tuple" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": { 134 | "collapsed": true 135 | }, 136 | "outputs": [], 137 | "source": [ 138 | "del tuple_2" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "#### Iterating" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 7, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "Max\n", 158 | "28\n", 159 | "New York\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "# Iterating over a tuple by using a for in loop\n", 165 | "for i in tuple_1:\n", 166 | " print(i)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "#### Check if an item exists" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 6, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "name": "stdout", 183 | "output_type": "stream", 184 | "text": [ 185 | "yes\n" 186 | ] 187 | } 188 | ], 189 | "source": [ 190 | "if \"New York\" in tuple_1:\n", 191 | " print(\"yes\")\n", 192 | "else:\n", 193 | " print(\"no\")" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "#### Usefule methods" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 2, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "name": "stdout", 210 | "output_type": "stream", 211 | "text": [ 212 | "5\n", 213 | "2\n", 214 | "3\n", 215 | "('a', 'b', 'a', 'b', 'a', 'b', 'a', 'b', 'a', 'b')\n", 216 | "(1, 2, 3, 4, 5, 6)\n", 217 | "('a', 'b', 'c', 'd')\n", 218 | "['a', 'b', 'c', 'd']\n", 219 | "('H', 'e', 'l', 'l', 'o')\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "my_tuple = ('a','p','p','l','e',)\n", 225 | "\n", 226 | "# len() : get the number of elements in a tuple\n", 227 | "print(len(my_tuple))\n", 228 | "\n", 229 | "# count(x) : Return the number of items that is equal to x\n", 230 | "print(my_tuple.count('p'))\n", 231 | "\n", 232 | "# index(x) : Return index of first item that is equal to x\n", 233 | "print(my_tuple.index('l'))\n", 234 | "\n", 235 | "# repetition\n", 236 | "my_tuple = ('a', 'b') * 5\n", 237 | "print(my_tuple)\n", 238 | "\n", 239 | "# concatenation\n", 240 | "my_tuple = (1,2,3) + (4,5,6)\n", 241 | "print(my_tuple)\n", 242 | "\n", 243 | "# convert list to a tuple and vice versa\n", 244 | "my_list = ['a', 'b', 'c', 'd']\n", 245 | "list_to_tuple = tuple(my_list)\n", 246 | "print(list_to_tuple)\n", 247 | "\n", 248 | "tuple_to_list = list(list_to_tuple)\n", 249 | "print(tuple_to_list)\n", 250 | "\n", 251 | "# convert string to tuple\n", 252 | "string_to_tuple = tuple('Hello')\n", 253 | "print(string_to_tuple)" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "metadata": {}, 259 | "source": [ 260 | "#### Slicing\n", 261 | "Access sub parts of the tuple wih the use of colon (:), just as with strings." 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 8, 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "name": "stdout", 271 | "output_type": "stream", 272 | "text": [ 273 | "(2, 3)\n", 274 | "(3, 4, 5, 6, 7, 8, 9, 10)\n", 275 | "(1, 2, 3)\n", 276 | "(1, 3, 5, 7, 9)\n", 277 | "(10, 9, 8, 7, 6, 5, 4, 3, 2, 1)\n" 278 | ] 279 | } 280 | ], 281 | "source": [ 282 | "# a[start:stop:step], default step is 1\n", 283 | "a = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)\n", 284 | "b = a[1:3] # Note that the last index is not included\n", 285 | "print(b)\n", 286 | "b = a[2:] # until the end\n", 287 | "print(b)\n", 288 | "b = a[:3] # from beginning\n", 289 | "print(b)\n", 290 | "b = a[::2] # start to end with every second item\n", 291 | "print(b)\n", 292 | "b = a[::-1] # reverse tuple\n", 293 | "print(b)" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "#### Unpack tuple" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 4, 306 | "metadata": {}, 307 | "outputs": [ 308 | { 309 | "name": "stdout", 310 | "output_type": "stream", 311 | "text": [ 312 | "Max\n", 313 | "28\n", 314 | "New York\n", 315 | "0\n", 316 | "[1, 2, 3, 4]\n", 317 | "5\n" 318 | ] 319 | } 320 | ], 321 | "source": [ 322 | "# number of variables have to match number of tuple elements\n", 323 | "tuple_1 = (\"Max\", 28, \"New York\")\n", 324 | "name, age, city = tuple_1\n", 325 | "print(name)\n", 326 | "print(age)\n", 327 | "print(city)\n", 328 | "\n", 329 | "# tip: unpack multiple elements to a list with *\n", 330 | "my_tuple = (0, 1, 2, 3, 4, 5)\n", 331 | "item_first, *items_between, item_last = my_tuple\n", 332 | "print(item_first)\n", 333 | "print(items_between)\n", 334 | "print(item_last)" 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": {}, 340 | "source": [ 341 | "#### Nested tuples\n", 342 | "Tuples can contain other tuples (or other container types)." 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": 10, 348 | "metadata": {}, 349 | "outputs": [ 350 | { 351 | "name": "stdout", 352 | "output_type": "stream", 353 | "text": [ 354 | "((0, 1), ('age', 'height'))\n", 355 | "(0, 1)\n" 356 | ] 357 | } 358 | ], 359 | "source": [ 360 | "a = ((0, 1), ('age', 'height'))\n", 361 | "print(a)\n", 362 | "print(a[0])" 363 | ] 364 | }, 365 | { 366 | "cell_type": "markdown", 367 | "metadata": {}, 368 | "source": [ 369 | "#### Compare tuple and list\n", 370 | "The immutability of tuples enables Python to make internal optimizations. Thus, tuples can be more efficient when working with large data." 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": 11, 376 | "metadata": {}, 377 | "outputs": [ 378 | { 379 | "name": "stdout", 380 | "output_type": "stream", 381 | "text": [ 382 | "104 bytes\n", 383 | "88 bytes\n", 384 | "0.12474981700000853\n", 385 | "0.014836141000017733\n" 386 | ] 387 | } 388 | ], 389 | "source": [ 390 | "# compare the size\n", 391 | "import sys\n", 392 | "my_list = [0, 1, 2, \"hello\", True]\n", 393 | "my_tuple = (0, 1, 2, \"hello\", True)\n", 394 | "print(sys.getsizeof(my_list), \"bytes\")\n", 395 | "print(sys.getsizeof(my_tuple), \"bytes\")\n", 396 | "\n", 397 | "# compare the execution time of a list vs. tuple creation statement\n", 398 | "import timeit\n", 399 | "print(timeit.timeit(stmt=\"[0, 1, 2, 3, 4, 5]\", number=1000000))\n", 400 | "print(timeit.timeit(stmt=\"(0, 1, 2, 3, 4, 5)\", number=1000000))" 401 | ] 402 | } 403 | ], 404 | "metadata": { 405 | "kernelspec": { 406 | "display_name": "Python [conda env:pythonengineer_env]", 407 | "language": "python", 408 | "name": "conda-env-pythonengineer_env-py" 409 | }, 410 | "language_info": { 411 | "codemirror_mode": { 412 | "name": "ipython", 413 | "version": 3 414 | }, 415 | "file_extension": ".py", 416 | "mimetype": "text/x-python", 417 | "name": "python", 418 | "nbconvert_exporter": "python", 419 | "pygments_lexer": "ipython3", 420 | "version": "3.7.3" 421 | } 422 | }, 423 | "nbformat": 4, 424 | "nbformat_minor": 2 425 | } 426 | -------------------------------------------------------------------------------- /advanced-python/01-Lists.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lists\n", 8 | "\n", 9 | "List is a collection data type which is ordered and mutable. Unlike Sets, Lists allow duplicate elements.\n", 10 | "They are useful for preserving a sequence of data and further iterating over it. Lists are created with square brackets.\n", 11 | "\n", 12 | "`my_list = [\"banana\", \"cherry\", \"apple\"]`\n", 13 | "\n", 14 | "#### Comparison of basic built-in collection data types in Python:\n", 15 | "\n", 16 | "- List is a collection which is ordered and mutable. Allows duplicate members.\n", 17 | "- Tuple is a collection which is ordered and immutable. Allows duplicate members.\n", 18 | "- Set is a collection which is unordered and unindexed. No duplicate members.\n", 19 | "- Dictionary is a collection which is unordered, mutable and indexed. No duplicate members.\n", 20 | "- Strings are immutable sequences of Unicode code points." 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "#### Creating a list\n", 28 | "Lists are created with square brackets or the built-in list function." 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 1, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "['banana', 'cherry', 'apple']\n", 41 | "[]\n", 42 | "[5, True, 'apple']\n", 43 | "[0, 0, 1, 1]\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "list_1 = [\"banana\", \"cherry\", \"apple\"]\n", 49 | "print(list_1)\n", 50 | "\n", 51 | "# Or create an empty list with the list function\n", 52 | "list_2 = list()\n", 53 | "print(list_2)\n", 54 | "\n", 55 | "# Lists allow different data types\n", 56 | "list_3 = [5, True, \"apple\"]\n", 57 | "print(list_3)\n", 58 | "\n", 59 | "# Lists allow duplicates\n", 60 | "list_4 = [0, 0, 1, 1]\n", 61 | "print(list_4)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Access elements\n", 69 | "You access the list items by referring to the index number. Note that the indices start at 0." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 2, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "banana\n", 82 | "apple\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "item = list_1[0]\n", 88 | "print(item)\n", 89 | "\n", 90 | "# You can also use negative indexing, e.g -1 refers to the last item,\n", 91 | "# -2 to the second last item, and so on\n", 92 | "item = list_1[-1]\n", 93 | "print(item)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "#### Change items\n", 101 | "Just refer to the index number and assign a new value." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 3, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "['banana', 'cherry', 'lemon']\n" 114 | ] 115 | } 116 | ], 117 | "source": [ 118 | "# Lists can be altered after their creation\n", 119 | "list_1[2] = \"lemon\"\n", 120 | "print(list_1)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "#### Useful methods\n", 128 | "Have a look at the Python Documentation to see all list methods: \n", 129 | "https://docs.python.org/3/tutorial/datastructures.html " 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 4, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "Length: 3\n", 142 | "['banana', 'blueberry', 'cherry', 'apple', 'orange']\n", 143 | "Popped item: orange\n", 144 | "['banana', 'blueberry', 'apple']\n", 145 | "[]\n", 146 | "Reversed: ['apple', 'cherry', 'banana']\n", 147 | "Sorted: ['apple', 'banana', 'cherry']\n", 148 | "[0, 0, 0, 0, 0]\n", 149 | "[0, 0, 0, 0, 0, 'banana', 'cherry', 'apple']\n", 150 | "['H', 'e', 'l', 'l', 'o']\n" 151 | ] 152 | } 153 | ], 154 | "source": [ 155 | "my_list = [\"banana\", \"cherry\", \"apple\"]\n", 156 | "\n", 157 | "# len() : get the number of elements in a list\n", 158 | "print(\"Length:\", len(my_list))\n", 159 | "\n", 160 | "# append() : adds an element to the end of the list\n", 161 | "my_list.append(\"orange\")\n", 162 | "\n", 163 | "# insert() : adds an element at the specified position\n", 164 | "my_list.insert(1, \"blueberry\")\n", 165 | "print(my_list)\n", 166 | "\n", 167 | "# pop() : removes and returns the item at the given position, default is the last item\n", 168 | "item = my_list.pop()\n", 169 | "print(\"Popped item: \", item)\n", 170 | "\n", 171 | "# remove() : removes an item from the list\n", 172 | "my_list.remove(\"cherry\") # Value error if not in the list\n", 173 | "print(my_list)\n", 174 | "\n", 175 | "# clear() : removes all items from the list\n", 176 | "my_list.clear()\n", 177 | "print(my_list)\n", 178 | "\n", 179 | "# reverse() : reverse the items\n", 180 | "my_list = [\"banana\", \"cherry\", \"apple\"]\n", 181 | "my_list.reverse()\n", 182 | "print('Reversed: ', my_list)\n", 183 | "\n", 184 | "# sort() : sort items in ascending order\n", 185 | "my_list.sort()\n", 186 | "print('Sorted: ', my_list)\n", 187 | "\n", 188 | "# use sorted() to get a new list, and leave the original unaffected.\n", 189 | "# sorted() works on any iterable type, not just lists\n", 190 | "my_list = [\"banana\", \"cherry\", \"apple\"]\n", 191 | "new_list = sorted(my_list)\n", 192 | "\n", 193 | "# create list with repeated elements\n", 194 | "list_with_zeros = [0] * 5\n", 195 | "print(list_with_zeros)\n", 196 | "\n", 197 | "# concatenation\n", 198 | "list_concat = list_with_zeros + my_list\n", 199 | "print(list_concat)\n", 200 | "\n", 201 | "# convert string to list\n", 202 | "string_to_list = list('Hello')\n", 203 | "print(string_to_list)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "#### Copy a list\n", 211 | "Be careful when copying references." 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 5, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "['banana', 'cherry', 'apple', True]\n", 224 | "['banana', 'cherry', 'apple', True]\n", 225 | "['banana', 'cherry', 'apple', True]\n", 226 | "['banana', 'cherry', 'apple']\n" 227 | ] 228 | } 229 | ], 230 | "source": [ 231 | "list_org = [\"banana\", \"cherry\", \"apple\"]\n", 232 | "\n", 233 | "# this just copies the reference to the list, so be careful\n", 234 | "list_copy = list_org\n", 235 | "\n", 236 | "# now modifying the copy also affects the original\n", 237 | "list_copy.append(True)\n", 238 | "print(list_copy)\n", 239 | "print(list_org)\n", 240 | "\n", 241 | "# use copy(), or list(x) to actually copy the list\n", 242 | "# slicing also works: list_copy = list_org[:]\n", 243 | "list_org = [\"banana\", \"cherry\", \"apple\"]\n", 244 | "\n", 245 | "list_copy = list_org.copy()\n", 246 | "# list_copy = list(list_org)\n", 247 | "# list_copy = list_org[:]\n", 248 | "\n", 249 | "# now modifying the copy does not affect the original\n", 250 | "list_copy.append(True)\n", 251 | "print(list_copy)\n", 252 | "print(list_org)" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "#### Iterating" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 6, 265 | "metadata": {}, 266 | "outputs": [ 267 | { 268 | "name": "stdout", 269 | "output_type": "stream", 270 | "text": [ 271 | "banana\n", 272 | "cherry\n", 273 | "lemon\n" 274 | ] 275 | } 276 | ], 277 | "source": [ 278 | "# Iterating over a list by using a for in loop\n", 279 | "for i in list_1:\n", 280 | " print(i)" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "#### Check if an item exists" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 7, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "name": "stdout", 297 | "output_type": "stream", 298 | "text": [ 299 | "yes\n" 300 | ] 301 | } 302 | ], 303 | "source": [ 304 | "if \"banana\" in list_1:\n", 305 | " print(\"yes\")\n", 306 | "else:\n", 307 | " print(\"no\")" 308 | ] 309 | }, 310 | { 311 | "cell_type": "markdown", 312 | "metadata": {}, 313 | "source": [ 314 | "#### Slicing\n", 315 | "Access sub parts of the list wih the use of colon (:), just as with strings." 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 8, 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "name": "stdout", 325 | "output_type": "stream", 326 | "text": [ 327 | "[2, 3]\n", 328 | "[3, 4, 5, 6, 7, 8, 9, 10]\n", 329 | "[1, 2, 3]\n", 330 | "[0, 4, 5, 6, 7, 8, 9, 10]\n", 331 | "[0, 5, 7, 9]\n", 332 | "[10, 9, 8, 7, 6, 5, 4, 0]\n", 333 | "[10, 9, 8, 7, 6, 5, 4, 0]\n" 334 | ] 335 | } 336 | ], 337 | "source": [ 338 | "# a[start:stop:step], default step is 1\n", 339 | "a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", 340 | "b = a[1:3] # Note that the last index is not included\n", 341 | "print(b)\n", 342 | "b = a[2:] # until the end\n", 343 | "print(b)\n", 344 | "b = a[:3] # from beginning\n", 345 | "print(b)\n", 346 | "a[0:3] = [0] # replace sub-parts, you need an iterable here\n", 347 | "print(a)\n", 348 | "b = a[::2] # start to end with every second item\n", 349 | "print(b)\n", 350 | "a = a[::-1] # reverse the list with a negative step:\n", 351 | "print(a)\n", 352 | "b = a[:] # copy a list with slicing\n", 353 | "print(b)" 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": {}, 359 | "source": [ 360 | "#### List comprehension\n", 361 | "A elegant and fast way to create a new list from an existing list.\n", 362 | "\n", 363 | "List comprehension consists of an expression followed by a for statement inside square brackets." 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 9, 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "name": "stdout", 373 | "output_type": "stream", 374 | "text": [ 375 | "[1, 4, 9, 16, 25, 36, 49, 64]\n" 376 | ] 377 | } 378 | ], 379 | "source": [ 380 | "a = [1, 2, 3, 4, 5, 6, 7, 8]\n", 381 | "b = [i * i for i in a] # squares each element\n", 382 | "print(b)" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "#### Nested lists\n", 390 | "Lists can contain other lists (or other container types)." 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": 10, 396 | "metadata": {}, 397 | "outputs": [ 398 | { 399 | "name": "stdout", 400 | "output_type": "stream", 401 | "text": [ 402 | "[[1, 2], [3, 4]]\n", 403 | "[1, 2]\n" 404 | ] 405 | } 406 | ], 407 | "source": [ 408 | "a = [[1, 2], [3, 4]]\n", 409 | "print(a)\n", 410 | "print(a[0])" 411 | ] 412 | } 413 | ], 414 | "metadata": { 415 | "kernelspec": { 416 | "display_name": "Python [conda env:pythonengineer_env]", 417 | "language": "python", 418 | "name": "conda-env-pythonengineer_env-py" 419 | }, 420 | "language_info": { 421 | "codemirror_mode": { 422 | "name": "ipython", 423 | "version": 3 424 | }, 425 | "file_extension": ".py", 426 | "mimetype": "text/x-python", 427 | "name": "python", 428 | "nbconvert_exporter": "python", 429 | "pygments_lexer": "ipython3", 430 | "version": "3.7.3" 431 | } 432 | }, 433 | "nbformat": 4, 434 | "nbformat_minor": 2 435 | } 436 | -------------------------------------------------------------------------------- /advanced-python/04-Sets.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sets\n", 8 | "A Set is an unordered collection data type that is unindexed, mutable, and has no duplicate elements. Sets are created with braces.\n", 9 | "\n", 10 | "`my_set = {\"apple\", \"banana\", \"cherry\"}`" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": {}, 16 | "source": [ 17 | "#### Create a set\n", 18 | "Use curly braces or the built-in set function." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 8, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "{'banana', 'apple', 'cherry'}\n", 31 | "{'three', 'one', 'two'}\n", 32 | "{'b', 'c', 'd', 'e', 'f', 'a'}\n", 33 | "\n", 34 | "\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "my_set = {\"apple\", \"banana\", \"cherry\"}\n", 40 | "print(my_set)\n", 41 | "\n", 42 | "# or use the set function and create from an iterable, e.g. list, tuple, string\n", 43 | "my_set_2 = set([\"one\", \"two\", \"three\"])\n", 44 | "my_set_2 = set((\"one\", \"two\", \"three\"))\n", 45 | "print(my_set_2)\n", 46 | "\n", 47 | "my_set_3 = set(\"aaabbbcccdddeeeeeffff\")\n", 48 | "print(my_set_3)\n", 49 | "\n", 50 | "# careful: an empty set cannot be created with {}, as this is interpreted as dict\n", 51 | "# use set() instead\n", 52 | "a = {}\n", 53 | "print(type(a))\n", 54 | "a = set()\n", 55 | "print(type(a))" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "#### Add elements" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 9, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "{True, 42, 'Hello'}\n", 75 | "{True, 42, 'Hello'}\n" 76 | ] 77 | } 78 | ], 79 | "source": [ 80 | "my_set = set()\n", 81 | "\n", 82 | "# use the add() method to add elements\n", 83 | "my_set.add(42)\n", 84 | "my_set.add(True)\n", 85 | "my_set.add(\"Hello\")\n", 86 | "\n", 87 | "# note: the order does not matter, and might differ when printed\n", 88 | "print(my_set)\n", 89 | "\n", 90 | "# nothing happens when the element is already present:\n", 91 | "my_set.add(42)\n", 92 | "print(my_set)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "#### Remove elements" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 10, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "name": "stdout", 109 | "output_type": "stream", 110 | "text": [ 111 | "{'banana', 'cherry'}\n", 112 | "{'banana'}\n", 113 | "set()\n", 114 | "False\n", 115 | "{True, 2, 'hi', 'hello'}\n" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "# remove(x): removes x, raises a KeyError if element is not present\n", 121 | "my_set = {\"apple\", \"banana\", \"cherry\"}\n", 122 | "my_set.remove(\"apple\")\n", 123 | "print(my_set)\n", 124 | "\n", 125 | "# KeyError:\n", 126 | "# my_set.remove(\"orange\")\n", 127 | "\n", 128 | "# discard(x): removes x, does nothing if element is not present\n", 129 | "my_set.discard(\"cherry\")\n", 130 | "my_set.discard(\"blueberry\")\n", 131 | "print(my_set)\n", 132 | "\n", 133 | "# clear() : remove all elements\n", 134 | "my_set.clear()\n", 135 | "print(my_set)\n", 136 | "\n", 137 | "# pop() : return and remove a random element\n", 138 | "a = {True, 2, False, \"hi\", \"hello\"}\n", 139 | "print(a.pop())\n", 140 | "print(a)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "#### Check if element is in Set" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 11, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "yes\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "my_set = {\"apple\", \"banana\", \"cherry\"}\n", 165 | "if \"apple\" in my_set:\n", 166 | " print(\"yes\")" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "#### Iterating " 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 12, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "name": "stdout", 183 | "output_type": "stream", 184 | "text": [ 185 | "banana\n", 186 | "apple\n", 187 | "cherry\n" 188 | ] 189 | } 190 | ], 191 | "source": [ 192 | "# Iterating over a set by using a for in loop\n", 193 | "# Note: order is not important\n", 194 | "my_set = {\"apple\", \"banana\", \"cherry\"}\n", 195 | "for i in my_set:\n", 196 | " print(i)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "#### Union and Intersection" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 13, 209 | "metadata": {}, 210 | "outputs": [ 211 | { 212 | "name": "stdout", 213 | "output_type": "stream", 214 | "text": [ 215 | "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 216 | "set()\n", 217 | "{3, 5, 7}\n", 218 | "{2}\n" 219 | ] 220 | } 221 | ], 222 | "source": [ 223 | "odds = {1, 3, 5, 7, 9}\n", 224 | "evens = {0, 2, 4, 6, 8}\n", 225 | "primes = {2, 3, 5, 7}\n", 226 | "\n", 227 | "# union() : combine elements from both sets, no duplication\n", 228 | "# note that this does not change the two sets\n", 229 | "u = odds.union(evens)\n", 230 | "print(u)\n", 231 | "\n", 232 | "# intersection(): take elements that are in both sets\n", 233 | "i = odds.intersection(evens)\n", 234 | "print(i)\n", 235 | "\n", 236 | "i = odds.intersection(primes)\n", 237 | "print(i)\n", 238 | "\n", 239 | "i = evens.intersection(primes)\n", 240 | "print(i)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "#### Difference of sets" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 14, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "name": "stdout", 257 | "output_type": "stream", 258 | "text": [ 259 | "{4, 5, 6, 7, 8, 9}\n", 260 | "{10, 11, 12}\n", 261 | "{4, 5, 6, 7, 8, 9, 10, 11, 12}\n", 262 | "{4, 5, 6, 7, 8, 9, 10, 11, 12}\n" 263 | ] 264 | } 265 | ], 266 | "source": [ 267 | "setA = {1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 268 | "setB = {1, 2, 3, 10, 11, 12}\n", 269 | "\n", 270 | "# difference() : returns a set with all the elements from the setA that are not in setB.\n", 271 | "diff_set = setA.difference(setB)\n", 272 | "print(diff_set)\n", 273 | "\n", 274 | "# A.difference(B) is not the same as B.difference(A)\n", 275 | "diff_set = setB.difference(setA)\n", 276 | "print(diff_set)\n", 277 | "\n", 278 | "# symmetric_difference() : returns a set with all the elements that are in setA and setB but not in both\n", 279 | "diff_set = setA.symmetric_difference(setB)\n", 280 | "print(diff_set)\n", 281 | "\n", 282 | "# A.symmetric_difference(B) = B.symmetric_difference(A)\n", 283 | "diff_set = setB.symmetric_difference(setA)\n", 284 | "print(diff_set)" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "#### Updating sets" 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": 15, 297 | "metadata": {}, 298 | "outputs": [ 299 | { 300 | "name": "stdout", 301 | "output_type": "stream", 302 | "text": [ 303 | "{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}\n", 304 | "{1, 2, 3}\n", 305 | "{4, 5, 6, 7, 8, 9}\n", 306 | "{4, 5, 6, 7, 8, 9, 10, 11, 12}\n" 307 | ] 308 | } 309 | ], 310 | "source": [ 311 | "setA = {1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 312 | "setB = {1, 2, 3, 10, 11, 12}\n", 313 | "\n", 314 | "# update() : Update the set by adding elements from another set.\n", 315 | "setA.update(setB)\n", 316 | "print(setA)\n", 317 | "\n", 318 | "# intersection_update() : Update the set by keeping only the elements found in both\n", 319 | "setA = {1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 320 | "setA.intersection_update(setB)\n", 321 | "print(setA)\n", 322 | "\n", 323 | "# difference_update() : Update the set by removing elements found in another set.\n", 324 | "setA = {1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 325 | "setA.difference_update(setB)\n", 326 | "print(setA)\n", 327 | "\n", 328 | "# symmetric_difference_update() : Update the set by only keeping the elements found in either set, but not in both\n", 329 | "setA = {1, 2, 3, 4, 5, 6, 7, 8, 9}\n", 330 | "setA.symmetric_difference_update(setB)\n", 331 | "print(setA)\n", 332 | "\n", 333 | "# Note: all update methods also work with other iterables as argument, e.g lists, tuples\n", 334 | "# setA.update([1, 2, 3, 4, 5, 6])" 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": {}, 340 | "source": [ 341 | "#### Copying" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 16, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "name": "stdout", 351 | "output_type": "stream", 352 | "text": [ 353 | "{1, 2, 3, 4, 5, 6, 7}\n", 354 | "{1, 2, 3, 4, 5, 6, 7}\n", 355 | "{1, 2, 3, 4, 5, 6, 7}\n", 356 | "{1, 2, 3, 4, 5}\n" 357 | ] 358 | } 359 | ], 360 | "source": [ 361 | "set_org = {1, 2, 3, 4, 5}\n", 362 | "\n", 363 | "# this just copies the reference to the set, so be careful\n", 364 | "set_copy = set_org\n", 365 | "\n", 366 | "# now modifying the copy also affects the original\n", 367 | "set_copy.update([3, 4, 5, 6, 7])\n", 368 | "print(set_copy)\n", 369 | "print(set_org)\n", 370 | "\n", 371 | "# use copy() to actually copy the set\n", 372 | "set_org = {1, 2, 3, 4, 5}\n", 373 | "set_copy = set_org.copy()\n", 374 | "\n", 375 | "# now modifying the copy does not affect the original\n", 376 | "set_copy.update([3, 4, 5, 6, 7])\n", 377 | "print(set_copy)\n", 378 | "print(set_org)" 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "#### Subset, Superset, and Disjoint" 386 | ] 387 | }, 388 | { 389 | "cell_type": "code", 390 | "execution_count": 17, 391 | "metadata": {}, 392 | "outputs": [ 393 | { 394 | "name": "stdout", 395 | "output_type": "stream", 396 | "text": [ 397 | "False\n", 398 | "True\n", 399 | "True\n", 400 | "False\n", 401 | "False\n", 402 | "True\n" 403 | ] 404 | } 405 | ], 406 | "source": [ 407 | "setA = {1, 2, 3, 4, 5, 6}\n", 408 | "setB = {1, 2, 3}\n", 409 | "# issubset(setX): Returns True if setX contains the set\n", 410 | "print(setA.issubset(setB))\n", 411 | "print(setB.issubset(setA)) # True\n", 412 | "\n", 413 | "# issuperset(setX): Returns True if the set contains setX\n", 414 | "print(setA.issuperset(setB)) # True\n", 415 | "print(setB.issuperset(setA))\n", 416 | "\n", 417 | "# isdisjoint(setX) : Return True if both sets have a null intersection, i.e. no same elements\n", 418 | "setC = {7, 8, 9}\n", 419 | "print(setA.isdisjoint(setB))\n", 420 | "print(setA.isdisjoint(setC))" 421 | ] 422 | }, 423 | { 424 | "cell_type": "markdown", 425 | "metadata": {}, 426 | "source": [ 427 | "#### Frozenset\n", 428 | "Frozen set is just an immutable version of normal set. While elements of a set can be modified at any time, elements of frozen set remains the same after creation.\n", 429 | "Creation with: `my_frozenset = frozenset(iterable)`" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 18, 435 | "metadata": {}, 436 | "outputs": [ 437 | { 438 | "name": "stdout", 439 | "output_type": "stream", 440 | "text": [ 441 | "frozenset({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})\n", 442 | "frozenset()\n", 443 | "frozenset({1, 3, 5, 7, 9})\n" 444 | ] 445 | } 446 | ], 447 | "source": [ 448 | "a = frozenset([0, 1, 2, 3, 4])\n", 449 | "\n", 450 | "# The following is not allowed:\n", 451 | "# a.add(5)\n", 452 | "# a.remove(1)\n", 453 | "# a.discard(1)\n", 454 | "# a.clear()\n", 455 | "\n", 456 | "# Also no update methods are allowed:\n", 457 | "# a.update([1,2,3])\n", 458 | "\n", 459 | "# Other set operations work\n", 460 | "odds = frozenset({1, 3, 5, 7, 9})\n", 461 | "evens = frozenset({0, 2, 4, 6, 8})\n", 462 | "print(odds.union(evens))\n", 463 | "print(odds.intersection(evens))\n", 464 | "print(odds.difference(evens))" 465 | ] 466 | } 467 | ], 468 | "metadata": { 469 | "kernelspec": { 470 | "display_name": "Python [conda env:pythonengineer_env]", 471 | "language": "python", 472 | "name": "conda-env-pythonengineer_env-py" 473 | }, 474 | "language_info": { 475 | "codemirror_mode": { 476 | "name": "ipython", 477 | "version": 3 478 | }, 479 | "file_extension": ".py", 480 | "mimetype": "text/x-python", 481 | "name": "python", 482 | "nbconvert_exporter": "python", 483 | "pygments_lexer": "ipython3", 484 | "version": "3.7.3" 485 | } 486 | }, 487 | "nbformat": 4, 488 | "nbformat_minor": 2 489 | } 490 | -------------------------------------------------------------------------------- /advanced-python/09-Exceptions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Errors and Exceptions\n", 8 | "A Python program terminates as soon as it encounters an error. In Python, an error can be a syntax error or an exception. In this article we will have a look at:\n", 9 | "- Syntax Error vs. Exception\n", 10 | "- How to raise Exceptions\n", 11 | "- How to handle Exceptions\n", 12 | "- Most common built-in Exceptions\n", 13 | "- How to define your own Exception" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Syntax Errors\n", 21 | "A **Syntax Error** occurs when the parser detects a syntactically incorrect statement. A syntax error can be for example a typo, missing brackets, no new line (see code below), or wrong identation (this will actually raise its own IndentationError, but its subclassed from a SyntaxError)." 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 5, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "ename": "SyntaxError", 31 | "evalue": "invalid syntax (, line 1)", 32 | "output_type": "error", 33 | "traceback": [ 34 | "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m a = 5 print(a)\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" 35 | ] 36 | } 37 | ], 38 | "source": [ 39 | "a = 5 print(a)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "## Exceptions\n", 47 | "Even if a statement is syntactically correct, it may cause an error when it is executed. This is called an **Exception Error**. There are several different error classes, for example trying to add a number and a string will raise a TypeError." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 6, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "ename": "TypeError", 57 | "evalue": "unsupported operand type(s) for +: 'int' and 'str'", 58 | "output_type": "error", 59 | "traceback": [ 60 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 61 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 62 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'10'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 63 | "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'int' and 'str'" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "a = 5 + '10'" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "## Raising an Exception\n", 76 | "If you want to force an exception to occur when a certain condition is met, you can use the `raise` keyword." 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 4, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "ename": "Exception", 86 | "evalue": "x should not be negative.", 87 | "output_type": "error", 88 | "traceback": [ 89 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 90 | "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", 91 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'x should not be negative.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 92 | "\u001b[0;31mException\u001b[0m: x should not be negative." 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "x = -5\n", 98 | "if x < 0:\n", 99 | " raise Exception('x should not be negative.')" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "You can also use the `assert` statement, which will throw an AssertionError if your assertion is **not** True.\n", 107 | "This way, you can actively test some conditions that have to be fulfilled instead of waiting for your program to unexpectedly crash midway. Assertion is also used in **unit testing**." 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 7, 113 | "metadata": {}, 114 | "outputs": [ 115 | { 116 | "ename": "AssertionError", 117 | "evalue": "x is not positive.", 118 | "output_type": "error", 119 | "traceback": [ 120 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 121 | "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", 122 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'x is not positive.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;31m# --> Your code will be fine if x >= 0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 123 | "\u001b[0;31mAssertionError\u001b[0m: x is not positive." 124 | ] 125 | } 126 | ], 127 | "source": [ 128 | "x = -5\n", 129 | "assert (x >= 0), 'x is not positive.'\n", 130 | "# --> Your code will be fine if x >= 0" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "## Handling Exceptions\n", 138 | "You can use a `try` and `except` block to catch and handle exceptions. If you can catch an exceptions your program won't terminate, and can continue." 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 8, 144 | "metadata": {}, 145 | "outputs": [ 146 | { 147 | "name": "stdout", 148 | "output_type": "stream", 149 | "text": [ 150 | "some error occured.\n", 151 | "division by zero\n", 152 | "Only a ZeroDivisionError is handled here\n", 153 | "A TypeError occured: unsupported operand type(s) for +: 'float' and 'str'\n" 154 | ] 155 | } 156 | ], 157 | "source": [ 158 | "# This will catch all possible exceptions\n", 159 | "try:\n", 160 | " a = 5 / 0\n", 161 | "except:\n", 162 | " print('some error occured.')\n", 163 | " \n", 164 | "# You can also catch the type of exception\n", 165 | "try:\n", 166 | " a = 5 / 0\n", 167 | "except Exception as e:\n", 168 | " print(e)\n", 169 | " \n", 170 | "# It is good practice to specify the type of Exception you want to catch.\n", 171 | "# Therefore, you have to know the possible errors\n", 172 | "try:\n", 173 | " a = 5 / 0\n", 174 | "except ZeroDivisionError:\n", 175 | " print('Only a ZeroDivisionError is handled here')\n", 176 | " \n", 177 | "# You can run multiple statements in a try block, and catch different possible exceptions\n", 178 | "try:\n", 179 | " a = 5 / 1 # Note: No ZeroDivisionError here\n", 180 | " b = a + '10'\n", 181 | "except ZeroDivisionError as e:\n", 182 | " print('A ZeroDivisionError occured:', e)\n", 183 | "except TypeError as e:\n", 184 | " print('A TypeError occured:', e)\n" 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "metadata": {}, 190 | "source": [ 191 | "#### `else` clause\n", 192 | "You can use an else statement that is run if no exception occured." 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 9, 198 | "metadata": { 199 | "scrolled": true 200 | }, 201 | "outputs": [ 202 | { 203 | "name": "stdout", 204 | "output_type": "stream", 205 | "text": [ 206 | "Everything is ok\n" 207 | ] 208 | } 209 | ], 210 | "source": [ 211 | "try:\n", 212 | " a = 5 / 1\n", 213 | "except ZeroDivisionError as e:\n", 214 | " print('A ZeroDivisionError occured:', e)\n", 215 | "else:\n", 216 | " print('Everything is ok')" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "metadata": {}, 222 | "source": [ 223 | "#### `finally` clause\n", 224 | "You can use a finally statement that always runs, no matter if there was an exception or not. This is for example used to make some cleanup operations." 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "metadata": {}, 231 | "outputs": [ 232 | { 233 | "name": "stdout", 234 | "output_type": "stream", 235 | "text": [ 236 | "A TypeError occured: unsupported operand type(s) for +: 'float' and 'str'\n", 237 | "Cleaning up some stuff...\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "try:\n", 243 | " a = 5 / 1 # Note: No ZeroDivisionError here\n", 244 | " b = a + '10'\n", 245 | "except ZeroDivisionError as e:\n", 246 | " print('A ZeroDivisionError occured:', e)\n", 247 | "except TypeError as e:\n", 248 | " print('A TypeError occured:', e)\n", 249 | "else:\n", 250 | " print('Everything is ok')\n", 251 | "finally:\n", 252 | " print('Cleaning up some stuff...')" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "## Common built-in Exceptions\n", 260 | "You can find all built-in Exceptions here: https://docs.python.org/3/library/exceptions.html\n", 261 | "- ImportError: If a module cannot be imported\n", 262 | "- NameError: If you try to use a variable that was not defined\n", 263 | "- FileNotFoundError: If you try to open a file that does not exist or you specify the wrong path\n", 264 | "- ValueError: When an operation or function receives an argument that has the right type but an inappropriate value,\n", 265 | "e.g. try to remove a value from a list that does not exist\n", 266 | "- TypeError: Raised when an operation or function is applied to an object of inappropriate type.\n", 267 | "- IndexError: If you try to access an invalid index of a sequence, e.g a list or a tuple.\n", 268 | "- KeyError: If you try to access a non existing key of a dictionary." 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [ 277 | "# ImportError\n", 278 | "import nonexistingmodule\n", 279 | "\n", 280 | "# NameError\n", 281 | "a = someundefinedvariable\n", 282 | "\n", 283 | "# FileNotFoundError\n", 284 | "with open('nonexistingfile.txt') as f:\n", 285 | " read_data = f.read()\n", 286 | "\n", 287 | "# ValueError\n", 288 | "a = [0, 1, 2]\n", 289 | "a.remove(3)\n", 290 | "\n", 291 | "# TypeError\n", 292 | "a = 5 + \"10\"\n", 293 | "\n", 294 | "# IndexError\n", 295 | "a = [0, 1, 2]\n", 296 | "value = a[5]\n", 297 | "\n", 298 | "# KeyError\n", 299 | "my_dict = {\"name\": \"Max\", \"city\": \"Boston\"}\n", 300 | "age = my_dict[\"age\"]" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": {}, 306 | "source": [ 307 | "## Define your own Exceptions\n", 308 | "You can define your own exception class that should be derived from the built-in `Exception` class. Most exceptions are defined with names that end in 'Error', similar to the naming of the standard exceptions. Exception classes can be defined like any other class, but are usually kept simple, often only offering a number of attributes that allow information about the error to be extracted by handlers." 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": 11, 314 | "metadata": {}, 315 | "outputs": [ 316 | { 317 | "name": "stdout", 318 | "output_type": "stream", 319 | "text": [ 320 | "Value is too low. The value is: 1\n" 321 | ] 322 | } 323 | ], 324 | "source": [ 325 | "# minimal example for own exception class\n", 326 | "class ValueTooHighError(Exception):\n", 327 | " pass\n", 328 | "\n", 329 | "# or add some more information for handlers\n", 330 | "class ValueTooLowError(Exception):\n", 331 | " def __init__(self, message, value):\n", 332 | " self.message = message\n", 333 | " self.value = value\n", 334 | "\n", 335 | "def test_value(a):\n", 336 | " if a > 1000:\n", 337 | " raise ValueTooHighError('Value is too high.')\n", 338 | " if a < 5:\n", 339 | " raise ValueTooLowError('Value is too low.', a) # Note that the constructor takes 2 arguments here\n", 340 | " return a\n", 341 | "\n", 342 | "try:\n", 343 | " test_value(1)\n", 344 | "except ValueTooHighError as e:\n", 345 | " print(e)\n", 346 | "except ValueTooLowError as e:\n", 347 | " print(e.message, 'The value is:', e.value)" 348 | ] 349 | } 350 | ], 351 | "metadata": { 352 | "kernelspec": { 353 | "display_name": "Python [conda env:pythonengineer_env]", 354 | "language": "python", 355 | "name": "conda-env-pythonengineer_env-py" 356 | }, 357 | "language_info": { 358 | "codemirror_mode": { 359 | "name": "ipython", 360 | "version": 3 361 | }, 362 | "file_extension": ".py", 363 | "mimetype": "text/x-python", 364 | "name": "python", 365 | "nbconvert_exporter": "python", 366 | "pygments_lexer": "ipython3", 367 | "version": "3.7.3" 368 | } 369 | }, 370 | "nbformat": 4, 371 | "nbformat_minor": 2 372 | } 373 | -------------------------------------------------------------------------------- /advanced-python/13-Decoratos.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Decorators\n", 8 | "A decorator is a function that takes another function and extends the behavior of this function without explicitly modifying it. It is a very powerful tool that allows to add new functionality to an existing function. \n", 9 | "There are 2 kinds of decorators:\n", 10 | "- Function decoratos\n", 11 | "- Class decorators\n", 12 | "\n", 13 | "A function is decorated with the `@` symbol: \n", 14 | "```python\n", 15 | "@my_decorator\n", 16 | "def my_function():\n", 17 | " pass\n", 18 | "```" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "## Function decorators\n", 26 | "In order to understand the decorator pattern, we have to understand that functions in Python are first class objects, which means that – like any other object – they can be defined inside another function, passed as argument to another function, or returned from other functions. A decorator is a function that takes another function as argument, wraps its behaviour inside an inner function. and returns the wrapped function. As a consequence, the decorated function no has extended functionality!" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 1, 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "name": "stdout", 36 | "output_type": "stream", 37 | "text": [ 38 | "Alex\n", 39 | "\n", 40 | "Start\n", 41 | "Alex\n", 42 | "End\n" 43 | ] 44 | } 45 | ], 46 | "source": [ 47 | "# A decorator function takes another function as argument, wraps its behaviour inside\n", 48 | "# an inner function, and returns the wrapped function.\n", 49 | "def start_end_decorator(func):\n", 50 | " \n", 51 | " def wrapper():\n", 52 | " print('Start')\n", 53 | " func()\n", 54 | " print('End')\n", 55 | " return wrapper\n", 56 | "\n", 57 | "def print_name():\n", 58 | " print('Alex')\n", 59 | " \n", 60 | "print_name()\n", 61 | "\n", 62 | "print()\n", 63 | "\n", 64 | "# Now wrap the function by passing it as argument to the decorator function\n", 65 | "# and asign it to itself -> Our function has extended behaviour!\n", 66 | "print_name = start_end_decorator(print_name)\n", 67 | "print_name()" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "## The decorator syntax\n", 75 | "Instead of wrapping our function and asigning it to itself, we can achieve the same thing simply by decorating our function with an `@`." 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 2, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Start\n", 88 | "Alex\n", 89 | "End\n" 90 | ] 91 | } 92 | ], 93 | "source": [ 94 | "@start_end_decorator\n", 95 | "def print_name():\n", 96 | " print('Alex')\n", 97 | " \n", 98 | "print_name()" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "## What about function arguments\n", 106 | "If our function has input arguments and we try to wrap it with our decorator above, it will raise a `TypeError` since \n", 107 | "we have to call our function inside the wrapper with this arguments, too. However, we can fix this by \n", 108 | "using `*args` and `**kwargs` in the inner function:" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 3, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "Start\n", 121 | "End\n", 122 | "None\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "def start_end_decorator_2(func):\n", 128 | " \n", 129 | " def wrapper(*args, **kwargs):\n", 130 | " print('Start')\n", 131 | " func(*args, **kwargs)\n", 132 | " print('End')\n", 133 | " return wrapper\n", 134 | "\n", 135 | "@start_end_decorator_2\n", 136 | "def add_5(x):\n", 137 | " return x + 5\n", 138 | "\n", 139 | "result = add_5(10)\n", 140 | "print(result)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "metadata": {}, 146 | "source": [ 147 | "## Return values\n", 148 | "Note that in the example above, we do not get the result back, so as next step we also have to return the value from our inner function:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 4, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "Start\n", 161 | "End\n", 162 | "15\n" 163 | ] 164 | } 165 | ], 166 | "source": [ 167 | "def start_end_decorator_3(func):\n", 168 | " \n", 169 | " def wrapper(*args, **kwargs):\n", 170 | " print('Start')\n", 171 | " result = func(*args, **kwargs)\n", 172 | " print('End')\n", 173 | " return result\n", 174 | " return wrapper\n", 175 | "\n", 176 | "@start_end_decorator_3\n", 177 | "def add_5(x):\n", 178 | " return x + 5\n", 179 | "\n", 180 | "result = add_5(10)\n", 181 | "print(result)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "## What about the function identity?\n", 189 | "If we have a look at the name of our decorated function, and inspect it with the built-in `help` function, we notice that Python thinks our function is now the wrapped inner function of the decorator function." 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 5, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "wrapper\n", 202 | "Help on function wrapper in module __main__:\n", 203 | "\n", 204 | "wrapper(*args, **kwargs)\n", 205 | "\n" 206 | ] 207 | } 208 | ], 209 | "source": [ 210 | "print(add_5.__name__)\n", 211 | "help(add_5)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "To fix this, use the `functools.wraps` decorator, which will preserve the information about the original function. This is helpful for introspection purposes, i.e. the ability of an object to know about its own attributes at runtime:" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 6, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "Start\n", 231 | "End\n", 232 | "15\n", 233 | "add_5\n", 234 | "Help on function add_5 in module __main__:\n", 235 | "\n", 236 | "add_5(x)\n", 237 | "\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "import functools\n", 243 | "def start_end_decorator_4(func):\n", 244 | " \n", 245 | " @functools.wraps(func)\n", 246 | " def wrapper(*args, **kwargs):\n", 247 | " print('Start')\n", 248 | " result = func(*args, **kwargs)\n", 249 | " print('End')\n", 250 | " return result\n", 251 | " return wrapper\n", 252 | "\n", 253 | "@start_end_decorator_4\n", 254 | "def add_5(x):\n", 255 | " return x + 5\n", 256 | "result = add_5(10)\n", 257 | "print(result)\n", 258 | "print(add_5.__name__)\n", 259 | "help(add_5)" 260 | ] 261 | }, 262 | { 263 | "cell_type": "markdown", 264 | "metadata": {}, 265 | "source": [ 266 | "## The final template for own decorators\n", 267 | "Now that we have all parts, our template for any decorator looks like this:" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 7, 273 | "metadata": { 274 | "collapsed": true 275 | }, 276 | "outputs": [], 277 | "source": [ 278 | "import functools\n", 279 | "\n", 280 | "def my_decorator(func):\n", 281 | " @functools.wraps(func)\n", 282 | " def wrapper(*args, **kwargs):\n", 283 | " # Do something before\n", 284 | " result = func(*args, **kwargs)\n", 285 | " # Do something after\n", 286 | " return result\n", 287 | " return wrapper" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "## Decorator function arguments\n", 295 | "Note that `functools.wraps` is a decorator that takes an argument for itself. We can think of this as 2 inner functions, so an inner function within an inner function. To make this clearer, we look at another example: A `repeat` decorator that takes a number as input. Within this function, we have the actual decorator function that wraps our function and extends its behaviour within another inner function. In this case, it repeats the input function the given number of times." 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": 8, 301 | "metadata": {}, 302 | "outputs": [ 303 | { 304 | "name": "stdout", 305 | "output_type": "stream", 306 | "text": [ 307 | "Hello Alex\n", 308 | "Hello Alex\n", 309 | "Hello Alex\n" 310 | ] 311 | } 312 | ], 313 | "source": [ 314 | "def repeat(num_times):\n", 315 | " def decorator_repeat(func):\n", 316 | " @functools.wraps(func)\n", 317 | " def wrapper(*args, **kwargs):\n", 318 | " for _ in range(num_times):\n", 319 | " result = func(*args, **kwargs)\n", 320 | " return result\n", 321 | " return wrapper\n", 322 | " return decorator_repeat\n", 323 | "\n", 324 | "@repeat(num_times=3)\n", 325 | "def greet(name):\n", 326 | " print(f\"Hello {name}\")\n", 327 | " \n", 328 | "greet('Alex')" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "## Nested Decorators\n", 336 | "We can apply several decorators to a function by stacking them on top of each other. The decorators are being executed in the order they are listed." 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": 9, 342 | "metadata": {}, 343 | "outputs": [ 344 | { 345 | "name": "stdout", 346 | "output_type": "stream", 347 | "text": [ 348 | "Calling say_hello(name='Alex')\n", 349 | "Start\n", 350 | "Hello Alex\n", 351 | "End\n", 352 | "'say_hello' returned 'Hello Alex'\n" 353 | ] 354 | }, 355 | { 356 | "data": { 357 | "text/plain": [ 358 | "'Hello Alex'" 359 | ] 360 | }, 361 | "execution_count": 9, 362 | "metadata": {}, 363 | "output_type": "execute_result" 364 | } 365 | ], 366 | "source": [ 367 | "# a decorator function that prints debug information about the wrapped function\n", 368 | "def debug(func):\n", 369 | " @functools.wraps(func)\n", 370 | " def wrapper(*args, **kwargs):\n", 371 | " args_repr = [repr(a) for a in args]\n", 372 | " kwargs_repr = [f\"{k}={v!r}\" for k, v in kwargs.items()]\n", 373 | " signature = \", \".join(args_repr + kwargs_repr)\n", 374 | " print(f\"Calling {func.__name__}({signature})\")\n", 375 | " result = func(*args, **kwargs)\n", 376 | " print(f\"{func.__name__!r} returned {result!r}\")\n", 377 | " return result\n", 378 | " return wrapper\n", 379 | "\n", 380 | "@debug\n", 381 | "@start_end_decorator_4\n", 382 | "def say_hello(name):\n", 383 | " greeting = f'Hello {name}'\n", 384 | " print(greeting)\n", 385 | " return greeting\n", 386 | "\n", 387 | "# now `debug` is executed first and calls `@start_end_decorator_4`, which then calls `say_hello`\n", 388 | "say_hello(name='Alex')" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "## Class decorators\n", 396 | "We can also use a class as a decorator. Therefore, we have to implement the `__call__()` method to make our object callable. Class decorators are typically used to maintain a state, e.g. here we keep track of the number of times our function is executed. The `__call__` method does essentially the same thing as the `wrapper()` method we have seen earlier. It adds some functionality, executes the function, and returns its result. Note that here we use `functools.update_wrapper()` instead of `functools.wraps` to preserve the information about our function." 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": 1, 402 | "metadata": {}, 403 | "outputs": [ 404 | { 405 | "name": "stdout", 406 | "output_type": "stream", 407 | "text": [ 408 | "Call 1 of 'say_hello'\n", 409 | "Hello!\n", 410 | "Call 2 of 'say_hello'\n", 411 | "Hello!\n" 412 | ] 413 | } 414 | ], 415 | "source": [ 416 | "import functools\n", 417 | "\n", 418 | "class CountCalls:\n", 419 | " # the init needs to have the func as argument and stores it\n", 420 | " def __init__(self, func):\n", 421 | " functools.update_wrapper(self, func)\n", 422 | " self.func = func\n", 423 | " self.num_calls = 0\n", 424 | " \n", 425 | " # extend functionality, execute function, and return the result\n", 426 | " def __call__(self, *args, **kwargs):\n", 427 | " self.num_calls += 1\n", 428 | " print(f\"Call {self.num_calls} of {self.func.__name__!r}\")\n", 429 | " return self.func(*args, **kwargs)\n", 430 | "\n", 431 | "@CountCalls\n", 432 | "def say_hello(num):\n", 433 | " print(\"Hello!\")\n", 434 | " \n", 435 | "say_hello(5)\n", 436 | "say_hello(5)" 437 | ] 438 | }, 439 | { 440 | "cell_type": "markdown", 441 | "metadata": {}, 442 | "source": [ 443 | "## Some typical use cases\n", 444 | "- Use a timer decorator to calculate the execution time of a function\n", 445 | "- Use a debug decorator to print out some more information about the called function and its arguments\n", 446 | "- Use a check decorator to check if the arguments fulfill some requirements and adapt the bevaviour accordingly\n", 447 | "- Register functions (plugins)\n", 448 | "- Slow down code with `time.sleep()` to check network behaviour\n", 449 | "- Cache the return values for memoization (https://en.wikipedia.org/wiki/Memoization)\n", 450 | "- Add information or update a state" 451 | ] 452 | } 453 | ], 454 | "metadata": { 455 | "kernelspec": { 456 | "display_name": "Python [conda env:pythonengineer_env]", 457 | "language": "python", 458 | "name": "conda-env-pythonengineer_env-py" 459 | }, 460 | "language_info": { 461 | "codemirror_mode": { 462 | "name": "ipython", 463 | "version": 3 464 | }, 465 | "file_extension": ".py", 466 | "mimetype": "text/x-python", 467 | "name": "python", 468 | "nbconvert_exporter": "python", 469 | "pygments_lexer": "ipython3", 470 | "version": "3.7.3" 471 | } 472 | }, 473 | "nbformat": 4, 474 | "nbformat_minor": 2 475 | } 476 | -------------------------------------------------------------------------------- /advanced-python/18-Functions arguments.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Function arguments\n", 8 | "In this article we will talk about function parameters and function arguments in detail. We will learn:\n", 9 | "\n", 10 | "- The difference between arguments and parameters\n", 11 | "- Positional and keyword arguments\n", 12 | "- Default arguments\n", 13 | "- Variable-length arguments (`*args` and `**kwargs`)\n", 14 | "- Container unpacking into function arguments\n", 15 | "- Local vs. global arguments\n", 16 | "- Parameter passing (by value or by reference?)" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Arguments and parameters\n", 24 | "- Parameters are the variables that are defined or used inside parentheses while defining a function\n", 25 | "- Arguments are the value passed for these parameters while calling a function" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 1, 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "name": "stdout", 35 | "output_type": "stream", 36 | "text": [ 37 | "Alex\n" 38 | ] 39 | } 40 | ], 41 | "source": [ 42 | "def print_name(name): # name is the parameter\n", 43 | " print(name)\n", 44 | "\n", 45 | "print_name('Alex') # 'Alex' is the argument" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## Positional and keyword arguments\n", 53 | "We can pass arguments as positional or keyword arguments. Some benefits of keyword arguments can be:\n", 54 | "- We can call arguments by their names to make it more clear what they represent\n", 55 | "- We can rearrange arguments in a way that makes them most readable" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 5, 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stdout", 65 | "output_type": "stream", 66 | "text": [ 67 | "1 2 3\n", 68 | "1 2 3\n", 69 | "1 2 3\n", 70 | "1 2 3\n" 71 | ] 72 | } 73 | ], 74 | "source": [ 75 | "def foo(a, b, c):\n", 76 | " print(a, b, c)\n", 77 | " \n", 78 | "# positional arguments\n", 79 | "foo(1, 2, 3)\n", 80 | "\n", 81 | "# keyword arguments\n", 82 | "foo(a=1, b=2, c=3)\n", 83 | "foo(c=3, b=2, a=1) # Note that the order is not important here\n", 84 | "\n", 85 | "# mix of both\n", 86 | "foo(1, b=2, c=3)\n", 87 | "\n", 88 | "# This is not allowed:\n", 89 | "# foo(1, b=2, 3) # positional argument after keyword argument\n", 90 | "# foo(1, b=2, a=3) # multiple values for argument 'a'" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "## Default arguments\n", 98 | "Functions can have default arguments with a predefined value. This argument can be left out and the default value is then passed to the function, or the argument can be used with a different value. Note that default arguments must be defined as the last parameters in a function." 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 11, 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "name": "stdout", 108 | "output_type": "stream", 109 | "text": [ 110 | "1 2 3 4\n", 111 | "1 2 3 100\n" 112 | ] 113 | } 114 | ], 115 | "source": [ 116 | "# default arguments\n", 117 | "def foo(a, b, c, d=4):\n", 118 | " print(a, b, c, d)\n", 119 | "\n", 120 | "foo(1, 2, 3, 4)\n", 121 | "foo(1, b=2, c=3, d=100)\n", 122 | "\n", 123 | "# not allowed: default arguments must be at the end\n", 124 | "# def foo(a, b=2, c, d=4):\n", 125 | "# print(a, b, c, d)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "## Variable-length arguments (`*args` and `**kwargs`)\n", 133 | "- If you mark a parameter with one asterisk (`*`), you can pass any number of positional arguments to your function (Typically called `*args`)\n", 134 | "- If you mark a parameter with two asterisks (`**`), you can pass any number of keyword arguments to this function (Typically called `**kwargs`)." 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 14, 140 | "metadata": {}, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "1 2\n", 147 | "3\n", 148 | "4\n", 149 | "5\n", 150 | "six 6\n", 151 | "seven 7\n", 152 | "\n", 153 | "1 2\n", 154 | "three 3\n" 155 | ] 156 | } 157 | ], 158 | "source": [ 159 | "def foo(a, b, *args, **kwargs):\n", 160 | " print(a, b)\n", 161 | " for arg in args:\n", 162 | " print(arg)\n", 163 | " for kwarg in kwargs:\n", 164 | " print(kwarg, kwargs[kwarg])\n", 165 | "\n", 166 | "# 3, 4, 5 are combined into args\n", 167 | "# six and seven are combined into kwargs\n", 168 | "foo(1, 2, 3, 4, 5, six=6, seven=7)\n", 169 | "print()\n", 170 | "\n", 171 | "# omitting of args or kwargs is also possible\n", 172 | "foo(1, 2, three=3)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "## Forced keyword arguments\n", 180 | "Sometimes you want to have keyword-only arguments. You can enforce that with:\n", 181 | "- If you write '`*,`' in your function parameter list, all parameters after that must be passed as keyword arguments.\n", 182 | "- Arguments after variable-length arguments must be keyword arguments." 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 19, 188 | "metadata": {}, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "1 2 3 4\n", 195 | "8\n", 196 | "9\n", 197 | "10\n", 198 | "50\n" 199 | ] 200 | } 201 | ], 202 | "source": [ 203 | "def foo(a, b, *, c, d):\n", 204 | " print(a, b, c, d)\n", 205 | "\n", 206 | "foo(1, 2, c=3, d=4)\n", 207 | "# not allowed:\n", 208 | "# foo(1, 2, 3, 4)\n", 209 | "\n", 210 | "# arguments after variable-length arguments must be keyword arguments\n", 211 | "def foo(*args, last):\n", 212 | " for arg in args:\n", 213 | " print(arg)\n", 214 | " print(last)\n", 215 | "\n", 216 | "foo(8, 9, 10, last=50)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "metadata": {}, 222 | "source": [ 223 | "## Unpacking into agruments\n", 224 | "- Lists or tuples can be unpacked into arguments with one asterisk (`*`) if the length of the container matches the number of function parameters.\n", 225 | "- Dictionaries can be unpacked into arguments with two asterisks (`**`) the the length and the keys match the function parameters." 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 21, 231 | "metadata": {}, 232 | "outputs": [ 233 | { 234 | "name": "stdout", 235 | "output_type": "stream", 236 | "text": [ 237 | "4 5 6\n", 238 | "1 2 3\n" 239 | ] 240 | } 241 | ], 242 | "source": [ 243 | "def foo(a, b, c):\n", 244 | " print(a, b, c)\n", 245 | "\n", 246 | "\n", 247 | "# list/tuple unpacking, length must match\n", 248 | "my_list = [4, 5, 6] # or tuple\n", 249 | "foo(*my_list)\n", 250 | "\n", 251 | "# dict unpacking, keys and length must match\n", 252 | "my_dict = {'a': 1, 'b': 2, 'c': 3}\n", 253 | "foo(**my_dict)\n", 254 | "\n", 255 | "# my_dict = {'a': 1, 'b': 2, 'd': 3} # not possible since wrong keyword" 256 | ] 257 | }, 258 | { 259 | "cell_type": "markdown", 260 | "metadata": {}, 261 | "source": [ 262 | "## Local vs global variables\n", 263 | "Global variables can be accessed within a function body, but to modify them, we first must state `global var_name` in order to change the global variable." 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 30, 269 | "metadata": {}, 270 | "outputs": [ 271 | { 272 | "name": "stdout", 273 | "output_type": "stream", 274 | "text": [ 275 | "number in function: 0\n", 276 | "number before foo2(): 0\n", 277 | "number after foo2(): 3\n" 278 | ] 279 | } 280 | ], 281 | "source": [ 282 | "def foo1():\n", 283 | " x = number # global variable can only be accessed here\n", 284 | " print('number in function:', x)\n", 285 | "\n", 286 | "number = 0\n", 287 | "foo1()\n", 288 | "\n", 289 | "# modifying the global variable\n", 290 | "def foo2():\n", 291 | " global number # global variable can now be accessed and modified\n", 292 | " number = 3\n", 293 | "\n", 294 | "print('number before foo2(): ', number)\n", 295 | "foo2() # modifies the global variable\n", 296 | "print('number after foo2(): ', number)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "If we do not write `global var_name` and asign a new value to a variable with the same name as the global variable, this will create a local variable within the function. The global variable remains unchanged." 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": 31, 309 | "metadata": {}, 310 | "outputs": [ 311 | { 312 | "name": "stdout", 313 | "output_type": "stream", 314 | "text": [ 315 | "number before foo3(): 0\n", 316 | "number after foo3(): 0\n" 317 | ] 318 | } 319 | ], 320 | "source": [ 321 | "number = 0\n", 322 | "\n", 323 | "def foo3():\n", 324 | " number = 3 # this is a local variable\n", 325 | "\n", 326 | "print('number before foo3(): ', number)\n", 327 | "foo3() # does not modify the global variable\n", 328 | "print('number after foo3(): ', number)" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "## Parameter passing\n", 336 | "Python uses a mechanism, which is known as \"Call-by-Object\" or \"Call-by-Object-Reference. The following rules must be considered:\n", 337 | "- The parameter passed in is actually a reference to an object (but the reference is passed by value)\n", 338 | "- Difference between mutable and immutable data types \n", 339 | "\n", 340 | "This means that:\n", 341 | "\n", 342 | "1. Mutable objects (e.g. lists,dict) can be changed within a method.\n", 343 | " * But if you rebind the reference in the method, the outer reference will still point at the original object.\n", 344 | "3. Immutable objects (e.g. int, string) cannot be changed within a method.\n", 345 | " * But immutable object CONTAINED WITHIN a mutable object can be re-assigned within a method." 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": 46, 351 | "metadata": {}, 352 | "outputs": [ 353 | { 354 | "name": "stdout", 355 | "output_type": "stream", 356 | "text": [ 357 | "var before foo(): 10\n", 358 | "var after foo(): 10\n" 359 | ] 360 | } 361 | ], 362 | "source": [ 363 | "# immutable objects -> no change\n", 364 | "def foo(x):\n", 365 | " x = 5 # x += 5 also no effect since x is immutable and a new variable must be created\n", 366 | "\n", 367 | "var = 10\n", 368 | "print('var before foo():', var)\n", 369 | "foo(var)\n", 370 | "print('var after foo():', var)" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": 47, 376 | "metadata": {}, 377 | "outputs": [ 378 | { 379 | "name": "stdout", 380 | "output_type": "stream", 381 | "text": [ 382 | "my_list before foo(): [1, 2, 3]\n", 383 | "my_list after foo(): [1, 2, 3, 4]\n" 384 | ] 385 | } 386 | ], 387 | "source": [ 388 | "# mutable objects -> change\n", 389 | "def foo(a_list):\n", 390 | " a_list.append(4)\n", 391 | " \n", 392 | "my_list = [1, 2, 3]\n", 393 | "print('my_list before foo():', my_list)\n", 394 | "foo(my_list)\n", 395 | "print('my_list after foo():', my_list)" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 48, 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "name": "stdout", 405 | "output_type": "stream", 406 | "text": [ 407 | "my_list before foo(): [1, 2, 'Max']\n", 408 | "my_list after foo(): [-100, 2, 'Paul']\n" 409 | ] 410 | } 411 | ], 412 | "source": [ 413 | "# immutable objects within a mutable object -> change\n", 414 | "def foo(a_list):\n", 415 | " a_list[0] = -100\n", 416 | " a_list[2] = \"Paul\"\n", 417 | " \n", 418 | "my_list = [1, 2, \"Max\"]\n", 419 | "print('my_list before foo():', my_list)\n", 420 | "foo(my_list)\n", 421 | "print('my_list after foo():', my_list)" 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": 53, 427 | "metadata": {}, 428 | "outputs": [ 429 | { 430 | "name": "stdout", 431 | "output_type": "stream", 432 | "text": [ 433 | "my_list before foo(): [1, 2, 3]\n", 434 | "my_list after foo(): [1, 2, 3]\n" 435 | ] 436 | } 437 | ], 438 | "source": [ 439 | "# Rebind a mutable reference -> no change\n", 440 | "def foo(a_list):\n", 441 | " a_list = [50, 60, 70] # a_list is now a new local variable within the function\n", 442 | " a_list.append(50)\n", 443 | " \n", 444 | "my_list = [1, 2, 3]\n", 445 | "print('my_list before foo():', my_list)\n", 446 | "foo(my_list)\n", 447 | "print('my_list after foo():', my_list)" 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "metadata": {}, 453 | "source": [ 454 | "Be careful with `+=` and `=` operations for mutable types. The first operation has an effect on the passed argument while the latter has not:" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 52, 460 | "metadata": {}, 461 | "outputs": [ 462 | { 463 | "name": "stdout", 464 | "output_type": "stream", 465 | "text": [ 466 | "my_list before foo(): [1, 2, 3]\n", 467 | "my_list after foo(): [1, 2, 3, 4, 5]\n", 468 | "my_list before bar(): [1, 2, 3]\n", 469 | "my_list after bar(): [1, 2, 3]\n" 470 | ] 471 | } 472 | ], 473 | "source": [ 474 | "# another example with rebinding references:\n", 475 | "def foo(a_list):\n", 476 | " a_list += [4, 5] # this chanches the outer variable\n", 477 | " \n", 478 | "def bar(a_list):\n", 479 | " a_list = a_list + [4, 5] # this rebinds the reference to a new local variable\n", 480 | "\n", 481 | "my_list = [1, 2, 3]\n", 482 | "print('my_list before foo():', my_list)\n", 483 | "foo(my_list)\n", 484 | "print('my_list after foo():', my_list)\n", 485 | "\n", 486 | "my_list = [1, 2, 3]\n", 487 | "print('my_list before bar():', my_list)\n", 488 | "bar(my_list)\n", 489 | "print('my_list after bar():', my_list)" 490 | ] 491 | } 492 | ], 493 | "metadata": { 494 | "kernelspec": { 495 | "display_name": "Python [conda env:pythonengineer_env]", 496 | "language": "python", 497 | "name": "conda-env-pythonengineer_env-py" 498 | }, 499 | "language_info": { 500 | "codemirror_mode": { 501 | "name": "ipython", 502 | "version": 3 503 | }, 504 | "file_extension": ".py", 505 | "mimetype": "text/x-python", 506 | "name": "python", 507 | "nbconvert_exporter": "python", 508 | "pygments_lexer": "ipython3", 509 | "version": "3.7.3" 510 | } 511 | }, 512 | "nbformat": 4, 513 | "nbformat_minor": 2 514 | } 515 | -------------------------------------------------------------------------------- /advanced-python/16-Threading in Python.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Threading in Python\n", 8 | "In this article we talk about how to use the `threading` module in Python.\n", 9 | "\n", 10 | "- How to create and start multiple threads\n", 11 | "- How to wait for threads to complete\n", 12 | "- How to share data between threads\n", 13 | "- How to use `Locks` to prevent race conditions\n", 14 | "- What is a daemon thread\n", 15 | "- How to use a `Queue` for thread-safe data/task processing." 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "## Create and run threads\n", 23 | "You create a thread with `threading.Thread()`. It takes two important arguments:\n", 24 | " \n", 25 | "- `target`: a callable object (function) for this thread to be invoked when the thread starts\n", 26 | "- `args`: the (function) arguments for the target function. This must be a tuple\n", 27 | "\n", 28 | "Start a thread with `thread.start()`\n", 29 | "\n", 30 | "Call `thread.join()` to tell the program that it should wait for this thread to complete before it continues with the rest of the code." 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": { 37 | "collapsed": true 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "from threading import Thread\n", 42 | "\n", 43 | "def square_numbers():\n", 44 | " for i in range(1000):\n", 45 | " result = i * i\n", 46 | "\n", 47 | " \n", 48 | "if __name__ == \"__main__\": \n", 49 | " threads = []\n", 50 | " num_threads = 10\n", 51 | "\n", 52 | " # create threads and asign a function for each thread\n", 53 | " for i in range(num_threads):\n", 54 | " thread = Thread(target=square_numbers)\n", 55 | " threads.append(thread)\n", 56 | "\n", 57 | " # start all threads\n", 58 | " for thread in threads:\n", 59 | " thread.start()\n", 60 | "\n", 61 | " # wait for all threads to finish\n", 62 | " # block the main thread until these threads are finished\n", 63 | " for thread in threads:\n", 64 | " thread.join()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "## Share data between threads\n", 72 | "Since threads live in the same memory space, they have access to the same (public) data. Thus, you can for example simply use\n", 73 | "a global variable to which all threads have read and write access.\n", 74 | "\n", 75 | "Task: Create two threads, each thread should access the current database value, modify it (in this case only increase it by 1), and write the new value back into the database value. Each thread should do this operation 10 times." 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 3, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Start value: 0\n", 88 | "End value: 1\n", 89 | "end main\n" 90 | ] 91 | } 92 | ], 93 | "source": [ 94 | "from threading import Thread\n", 95 | "import time\n", 96 | "\n", 97 | "\n", 98 | "# all threads can access this global variable\n", 99 | "database_value = 0\n", 100 | "\n", 101 | "def increase():\n", 102 | " global database_value # needed to modify the global value\n", 103 | " \n", 104 | " # get a local copy (simulate data retrieving)\n", 105 | " local_copy = database_value\n", 106 | " \n", 107 | " # simulate some modifying operation\n", 108 | " local_copy += 1\n", 109 | " time.sleep(0.1)\n", 110 | " \n", 111 | " # write the calculated new value into the global variable\n", 112 | " database_value = local_copy\n", 113 | "\n", 114 | "\n", 115 | "if __name__ == \"__main__\":\n", 116 | "\n", 117 | " print('Start value: ', database_value)\n", 118 | "\n", 119 | " t1 = Thread(target=increase)\n", 120 | " t2 = Thread(target=increase)\n", 121 | "\n", 122 | " t1.start()\n", 123 | " t2.start()\n", 124 | "\n", 125 | " t1.join()\n", 126 | " t2.join()\n", 127 | "\n", 128 | " print('End value:', database_value)\n", 129 | "\n", 130 | " print('end main')" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "## How to use `Locks`\n", 138 | "Notice that in the above example, the 2 threads should increment the value by 1, so 2 increment operations are performed. But why is the end value 1 and not 2?\n", 139 | "\n", 140 | "#### Race condition\n", 141 | "A race condition happened here. A race condition occurs when two or more threads can access shared data and they try to change it at the same time. Because the thread scheduling algorithm can swap between threads at any time, you don't know the order in which the threads will attempt to access the shared data. In our case, the first thread accesses the `database_value` (0) and stores it in a local copy. It then increments it (`local_copy` is now 1). With our `time.sleep()` function that just simulates some time consuming operations, the programm will swap to the second thread in the meantime. This will also retrieve the current `database_value` (still 0) and increment the `local_copy` to 1. Now both threads have a local copy with value 1, so both will write the 1 into the global `database_value`. This is why the end value is 1 and not 2.\n", 142 | "\n", 143 | "#### Avoid race conditions with `Locks`\n", 144 | "A lock (also known as mutex) is a synchronization mechanism for enforcing limits on access to a resource in an environment where there are many threads of execution. A Lock has two states: **locked** and **unlocked**. \n", 145 | "If the state is locked, it does not allow other concurrent threads to enter this code section until the state is unlocked again.\n", 146 | "\n", 147 | "Two functions are important:\n", 148 | "- `lock.acquire()` : This will lock the state and block\n", 149 | "- `lock.release()` : This will unlock the state again.\n", 150 | "\n", 151 | "Important: You should always release the block again after it was acquired!\n", 152 | "\n", 153 | "In our example the critical code section where database values are retrieved and modified is now locked. This prevents the second thread from modyfing the global data at the same time. Not much has changed in our code. All new changes are commented in the code below." 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 6, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "Start value: 0\n", 166 | "End value: 2\n", 167 | "end main\n" 168 | ] 169 | } 170 | ], 171 | "source": [ 172 | "# import Lock\n", 173 | "from threading import Thread, Lock\n", 174 | "import time\n", 175 | "\n", 176 | "\n", 177 | "database_value = 0\n", 178 | "\n", 179 | "def increase(lock):\n", 180 | " global database_value \n", 181 | " \n", 182 | " # lock the state\n", 183 | " lock.acquire()\n", 184 | " \n", 185 | " local_copy = database_value\n", 186 | " local_copy += 1\n", 187 | " time.sleep(0.1)\n", 188 | " database_value = local_copy\n", 189 | " \n", 190 | " # unlock the state\n", 191 | " lock.release()\n", 192 | "\n", 193 | "\n", 194 | "if __name__ == \"__main__\":\n", 195 | "\n", 196 | " # create a lock\n", 197 | " lock = Lock()\n", 198 | " \n", 199 | " print('Start value: ', database_value)\n", 200 | "\n", 201 | " # pass the lock to the target function\n", 202 | " t1 = Thread(target=increase, args=(lock,)) # notice the comma after lock since args must be a tuple\n", 203 | " t2 = Thread(target=increase, args=(lock,))\n", 204 | "\n", 205 | " t1.start()\n", 206 | " t2.start()\n", 207 | "\n", 208 | " t1.join()\n", 209 | " t2.join()\n", 210 | "\n", 211 | " print('End value:', database_value)\n", 212 | "\n", 213 | " print('end main')" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "#### Use the lock as a context manager\n", 221 | "After `lock.acquire()` you should never forget to call `lock.release()` to unblock the code. You can also use a lock as a context manager, wich will safely lock and unlock your code. It is recommended to use a lock this way:" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 8, 227 | "metadata": { 228 | "collapsed": true 229 | }, 230 | "outputs": [], 231 | "source": [ 232 | "def increase(lock):\n", 233 | " global database_value \n", 234 | " \n", 235 | " with lock: \n", 236 | " local_copy = database_value\n", 237 | " local_copy += 1\n", 238 | " time.sleep(0.1)\n", 239 | " database_value = local_copy" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "metadata": { 245 | "collapsed": true 246 | }, 247 | "source": [ 248 | "## Using Queues in Python\n", 249 | "Queues can be used for thread-safe/process-safe data exchanges and data processing both in a multithreaded and a multiprocessing environment.\n", 250 | "\n", 251 | "### The queue\n", 252 | "A queue is a linear data structure that follows the First In First Out (FIFO) principle. A good example is a queue of customers that are waiting in line, where the customer that came first is served first." 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 1, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "name": "stdout", 262 | "output_type": "stream", 263 | "text": [ 264 | "1\n" 265 | ] 266 | } 267 | ], 268 | "source": [ 269 | "from queue import Queue\n", 270 | "\n", 271 | "# create queue\n", 272 | "q = Queue()\n", 273 | "\n", 274 | "# add elements\n", 275 | "q.put(1) # 1\n", 276 | "q.put(2) # 2 1\n", 277 | "q.put(3) # 3 2 1 \n", 278 | "\n", 279 | "# now q looks like this:\n", 280 | "# back --> 3 2 1 --> front\n", 281 | "\n", 282 | "# get and remove first element\n", 283 | "first = q.get() # --> 1\n", 284 | "print(first) \n", 285 | "\n", 286 | "# q looks like this:\n", 287 | "# back --> 3 2 --> front" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "### Using a queue in multithreading\n", 295 | "Operations with a queue are thread-safe. Important methods are:\n", 296 | "\n", 297 | "- `q.get()` : Remove and return the first item. By default, it blocks until the item is available.\n", 298 | "- `q.put(item)` : Puts element at the end of the queue. By default, it blocks until a free slot is available.\n", 299 | "- `q.task_done()` : Indicate that a formerly enqueued task is complete. For each `get()` you should call this after you are done with your task for this item.\n", 300 | "- `q.join()` : Blocks until all items in the queue have been gotten and proccessed (`task_done()` has been called for each item).\n", 301 | "- `q.empty()` : Return True if the queue is empty.\n", 302 | "\n", 303 | "The following example uses a queue to exchange numbers from 0...19. Each thread invokes the `worker` method.\n", 304 | "Inside the infinite loop the thread is waiting until items are available due to the blocking `q.get()` call. When items are available, they are processed (i.e. just printed here), and then `q.task_done()` tells the queue that processing is complete. In the main thread, 10 **daemon** threads are created. This means that they automatically die when the main thread dies, and thus the worker method and infinite loop is no longer invoked. Then the queue is filled with items and the worker method can continue with available items. At the end `q.join()` is necessary to block the main thread until all items have been gotten and proccessed." 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": 7, 310 | "metadata": {}, 311 | "outputs": [ 312 | { 313 | "name": "stdout", 314 | "output_type": "stream", 315 | "text": [ 316 | "in Thread1 got 0\n", 317 | "in Thread2 got 1\n", 318 | "in Thread2 got 11\n", 319 | "in Thread2 got 12\n", 320 | "in Thread2 got 13\n", 321 | "in Thread2 got 14\n", 322 | "in Thread2 got 15\n", 323 | "in Thread2 got 16\n", 324 | "in Thread2 got 17\n", 325 | "in Thread2 got 18\n", 326 | "in Thread2 got 19\n", 327 | "in Thread8 got 5\n", 328 | "in Thread4 got 9\n", 329 | "in Thread1 got 10\n", 330 | "in Thread5 got 2\n", 331 | "in Thread6 got 3\n", 332 | "in Thread9 got 6\n", 333 | "in Thread7 got 4\n", 334 | "in Thread10 got 7\n", 335 | "in Thread3 got 8\n", 336 | "main done\n" 337 | ] 338 | } 339 | ], 340 | "source": [ 341 | "from threading import Thread, Lock, current_thread\n", 342 | "from queue import Queue\n", 343 | "\n", 344 | "def worker(q, lock):\n", 345 | " while True:\n", 346 | " value = q.get() # blocks until the item is available\n", 347 | "\n", 348 | " # do stuff...\n", 349 | " with lock:\n", 350 | " # prevent printing at the same time with this lock\n", 351 | " print(f\"in {current_thread().name} got {value}\")\n", 352 | " # ...\n", 353 | "\n", 354 | " # For each get(), a subsequent call to task_done() tells the queue\n", 355 | " # that the processing on this item is complete.\n", 356 | " # If all tasks are done, q.join() can unblock\n", 357 | " q.task_done()\n", 358 | "\n", 359 | "\n", 360 | "if __name__ == '__main__':\n", 361 | " q = Queue()\n", 362 | " num_threads = 10\n", 363 | " lock = Lock()\n", 364 | "\n", 365 | " for i in range(num_threads):\n", 366 | " t = Thread(name=f\"Thread{i+1}\", target=worker, args=(q, lock))\n", 367 | " t.daemon = True # dies when the main thread dies\n", 368 | " t.start()\n", 369 | " \n", 370 | " # fill the queue with items\n", 371 | " for x in range(20):\n", 372 | " q.put(x)\n", 373 | "\n", 374 | " q.join() # Blocks until all items in the queue have been gotten and processed.\n", 375 | "\n", 376 | " print('main done')" 377 | ] 378 | }, 379 | { 380 | "cell_type": "markdown", 381 | "metadata": {}, 382 | "source": [ 383 | "### Daemon threads\n", 384 | "In the above example, daemon threads are used. Daemon threads are background threads that automatically die when the main program ends. This is why the infinite loops inside the worker methods can be exited. Without a daemon process we would have to use a signalling mechanism such as a `threading.Event` to stop the worker. But be careful with daemon processes: They are abruptly stopped and their resources (e.g. open files or database transactions) may not be released/completed properly. " 385 | ] 386 | } 387 | ], 388 | "metadata": { 389 | "kernelspec": { 390 | "display_name": "Python [conda env:pythonengineer_env]", 391 | "language": "python", 392 | "name": "conda-env-pythonengineer_env-py" 393 | }, 394 | "language_info": { 395 | "codemirror_mode": { 396 | "name": "ipython", 397 | "version": 3 398 | }, 399 | "file_extension": ".py", 400 | "mimetype": "text/x-python", 401 | "name": "python", 402 | "nbconvert_exporter": "python", 403 | "pygments_lexer": "ipython3", 404 | "version": "3.7.3" 405 | } 406 | }, 407 | "nbformat": 4, 408 | "nbformat_minor": 2 409 | } 410 | -------------------------------------------------------------------------------- /advanced-python/11-JSON.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# JSON\n", 8 | "JSON (JavaScript Object Notation) is a leightweight data format for data exchange. In Python you have the built-in `json` module for encoding and decoding JSON data. Simply import it and you are ready to work with JSON data:\n", 9 | "\n", 10 | "```python\n", 11 | "import json\n", 12 | "``` \n", 13 | "\n", 14 | "Some advantages of JSON:\n", 15 | "- JSON exists as a \"sequence of bytes\" which is very useful in the case we need to transmit (stream) data over a network.\n", 16 | "- Compared to XML, JSON is much smaller, translating into faster data transfers, and better experiences.\n", 17 | "- JSON is extremely human-friendly since it is textual, and simultaneously machine-friendly." 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "## JSON format" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "{\n", 34 | " \"firstName\": \"Jane\",\n", 35 | " \"lastName\": \"Doe\",\n", 36 | " \"hobbies\": [\"running\", \"swimming\", \"singing\"],\n", 37 | " \"age\": 28,\n", 38 | " \"children\": [\n", 39 | " {\n", 40 | " \"firstName\": \"Alex\",\n", 41 | " \"age\": 5\n", 42 | " },\n", 43 | " {\n", 44 | " \"firstName\": \"Bob\",\n", 45 | " \"age\": 7\n", 46 | " }\n", 47 | " ]\n", 48 | "}" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "JSON supports primitive types (strings, numbers, boolean), as well as nested arrays and objects. Simple Python objects are translated to JSON according to the following conversion: \n", 56 | "\n", 57 | "| Python | JSON |\n", 58 | "| :--- | ---: |\n", 59 | "| dict | object |\n", 60 | "| list, tuple | array |\n", 61 | "| str | string |\n", 62 | "| int, long, float | number |\n", 63 | "| True | true |\n", 64 | "| False | false |\n", 65 | "| None | null |" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "## From Python to JSON (Serialization, Encode)\n", 73 | "Convert Python objects into a JSON string with the `json.dumps()` method." 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 33, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "{\"name\": \"John\", \"age\": 30, \"city\": \"New York\", \"hasChildren\": false, \"titles\": [\"engineer\", \"programmer\"]}\n", 86 | "{\n", 87 | " \"age\"= 30; \n", 88 | " \"city\"= \"New York\"; \n", 89 | " \"hasChildren\"= false; \n", 90 | " \"name\"= \"John\"; \n", 91 | " \"titles\"= [\n", 92 | " \"engineer\"; \n", 93 | " \"programmer\"\n", 94 | " ]\n", 95 | "}\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "import json\n", 101 | "\n", 102 | "person = {\"name\": \"John\", \"age\": 30, \"city\": \"New York\", \"hasChildren\": False, \"titles\": [\"engineer\", \"programmer\"]}\n", 103 | "\n", 104 | "# convert into JSON:\n", 105 | "person_json = json.dumps(person)\n", 106 | "# use different formatting style\n", 107 | "person_json2 = json.dumps(person, indent=4, separators=(\"; \", \"= \"), sort_keys=True)\n", 108 | "\n", 109 | "# the result is a JSON string:\n", 110 | "print(person_json) \n", 111 | "print(person_json2) " 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "Or convert Python objects into JSON objects and save them into a file with the `json.dump()` method." 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 34, 124 | "metadata": { 125 | "collapsed": true 126 | }, 127 | "outputs": [], 128 | "source": [ 129 | "import json\n", 130 | "\n", 131 | "person = {\"name\": \"John\", \"age\": 30, \"city\": \"New York\", \"hasChildren\": False, \"titles\": [\"engineer\", \"programmer\"]}\n", 132 | "\n", 133 | "with open('person.json', 'w') as f:\n", 134 | " json.dump(person, f) # you can also specify indent etc..." 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "## FROM JSON to Python (Deserialization, Decode)\n", 142 | "Convert a JSON string into a Python object with the `json.loads()` method. The result will be a **Python dictionary**." 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 35, 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "{'age': 30, 'city': 'New York', 'hasChildren': False, 'name': 'John', 'titles': ['engineer', 'programmer']}\n" 155 | ] 156 | } 157 | ], 158 | "source": [ 159 | "import json\n", 160 | "person_json = \"\"\"\n", 161 | "{\n", 162 | " \"age\": 30, \n", 163 | " \"city\": \"New York\",\n", 164 | " \"hasChildren\": false, \n", 165 | " \"name\": \"John\",\n", 166 | " \"titles\": [\n", 167 | " \"engineer\",\n", 168 | " \"programmer\"\n", 169 | " ]\n", 170 | "}\n", 171 | "\"\"\"\n", 172 | "person = json.loads(person_json)\n", 173 | "print(person)" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "Or load data from a file and convert it to a Python object with the `json.load()` method." 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 36, 186 | "metadata": {}, 187 | "outputs": [ 188 | { 189 | "name": "stdout", 190 | "output_type": "stream", 191 | "text": [ 192 | "{'name': 'John', 'age': 30, 'city': 'New York', 'hasChildren': False, 'titles': ['engineer', 'programmer']}\n" 193 | ] 194 | } 195 | ], 196 | "source": [ 197 | "import json\n", 198 | "\n", 199 | "with open('person.json', 'r') as f:\n", 200 | " person = json.load(f)\n", 201 | " print(person)" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "## Working with Custom Objects\n", 209 | "#### Encoding\n", 210 | "Encoding a custom object with the default `JSONEncoder` will raise a `TypeError`. We can specify a custom encoding function that will store the class name and all object variables in a dictionary. Use this function for the `default` argument in the `json.dump()` method." 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 1, 216 | "metadata": {}, 217 | "outputs": [ 218 | { 219 | "name": "stdout", 220 | "output_type": "stream", 221 | "text": [ 222 | "{\"complex\": true, \"real\": 5.0, \"imag\": 9.0}\n" 223 | ] 224 | } 225 | ], 226 | "source": [ 227 | "import json\n", 228 | "def encode_complex(z):\n", 229 | " if isinstance(z, complex):\n", 230 | " # just the key of the class name is important, the value can be arbitrary.\n", 231 | " return {z.__class__.__name__: True, \"real\":z.real, \"imag\":z.imag}\n", 232 | " else:\n", 233 | " raise TypeError(f\"Object of type '{z.__class__.__name__}' is not JSON serializable\")\n", 234 | "\n", 235 | "z = 5 + 9j\n", 236 | "zJSON = json.dumps(z, default=encode_complex)\n", 237 | "print(zJSON)" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "You can also create a custom Encoder class, and overwrite the `default()` method. Use this for the `cls` argument in the `json.dump()` method, or use the encoder directly." 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 38, 250 | "metadata": {}, 251 | "outputs": [ 252 | { 253 | "name": "stdout", 254 | "output_type": "stream", 255 | "text": [ 256 | "{\"complex\": true, \"real\": 5.0, \"imag\": 9.0}\n", 257 | "{\"complex\": true, \"real\": 5.0, \"imag\": 9.0}\n" 258 | ] 259 | } 260 | ], 261 | "source": [ 262 | "from json import JSONEncoder\n", 263 | "class ComplexEncoder(JSONEncoder):\n", 264 | " \n", 265 | " def default(self, o):\n", 266 | " if isinstance(z, complex):\n", 267 | " return {z.__class__.__name__: True, \"real\":z.real, \"imag\":z.imag}\n", 268 | " # Let the base class default method handle other objects or raise a TypeError\n", 269 | " return JSONEncoder.default(self, o)\n", 270 | " \n", 271 | "z = 5 + 9j\n", 272 | "zJSON = json.dumps(z, cls=ComplexEncoder)\n", 273 | "print(zJSON)\n", 274 | "# or use encoder directly:\n", 275 | "zJson = ComplexEncoder().encode(z)\n", 276 | "print(zJSON)" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "#### Decoding\n", 284 | "Decoding a custom object with the defaut `JSONDecoder` is possible, but it will be decoded into a dictionary. Write a custom decode function that will take a dictionary as input, and creates your custom object if it can find the object class name in the dictionary. Use this function for the `object_hook` argument in the `json.load()` method." 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 39, 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "name": "stdout", 294 | "output_type": "stream", 295 | "text": [ 296 | "\n", 297 | "{'complex': True, 'real': 5.0, 'imag': 9.0}\n", 298 | "\n", 299 | "(5+9j)\n" 300 | ] 301 | } 302 | ], 303 | "source": [ 304 | "# Possible but decoded as a dictionary\n", 305 | "z = json.loads(zJSON)\n", 306 | "print(type(z))\n", 307 | "print(z)\n", 308 | "\n", 309 | "def decode_complex(dct):\n", 310 | " if complex.__name__ in dct:\n", 311 | " return complex(dct[\"real\"], dct[\"imag\"])\n", 312 | " return dct\n", 313 | "\n", 314 | "# Now the object is of type complex after decoding\n", 315 | "z = json.loads(zJSON, object_hook=decode_complex)\n", 316 | "print(type(z))\n", 317 | "print(z)" 318 | ] 319 | }, 320 | { 321 | "cell_type": "markdown", 322 | "metadata": {}, 323 | "source": [ 324 | "## Template encode and decode functions\n", 325 | "This works for all custom classes if all instance variables are given in the `__init__` method." 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": 2, 331 | "metadata": {}, 332 | "outputs": [ 333 | { 334 | "name": "stdout", 335 | "output_type": "stream", 336 | "text": [ 337 | "{\n", 338 | " \"__class__\": \"User\",\n", 339 | " \"__module__\": \"__main__\",\n", 340 | " \"active\": true,\n", 341 | " \"age\": 28,\n", 342 | " \"balance\": 20.7,\n", 343 | " \"friends\": [\n", 344 | " \"Jane\",\n", 345 | " \"Tom\"\n", 346 | " ],\n", 347 | " \"name\": \"John\"\n", 348 | "}\n", 349 | "\n", 350 | "{\n", 351 | " \"__class__\": \"Player\",\n", 352 | " \"__module__\": \"__main__\",\n", 353 | " \"level\": 5,\n", 354 | " \"name\": \"Max\",\n", 355 | " \"nickname\": \"max1234\"\n", 356 | "}\n", 357 | "\n" 358 | ] 359 | } 360 | ], 361 | "source": [ 362 | "class User:\n", 363 | " # Custom class with all instance variables given in the __init__()\n", 364 | " def __init__(self, name, age, active, balance, friends):\n", 365 | " self.name = name\n", 366 | " self.age = age\n", 367 | " self.active = active\n", 368 | " self.balance = balance\n", 369 | " self.friends = friends\n", 370 | " \n", 371 | "class Player:\n", 372 | " # Other custom class\n", 373 | " def __init__(self, name, nickname, level):\n", 374 | " self.name = name\n", 375 | " self.nickname = nickname\n", 376 | " self.level = level\n", 377 | " \n", 378 | " \n", 379 | "def encode_obj(obj):\n", 380 | " \"\"\"\n", 381 | " Takes in a custom object and returns a dictionary representation of the object.\n", 382 | " This dict representation also includes the object's module and class names.\n", 383 | " \"\"\"\n", 384 | " \n", 385 | " # Populate the dictionary with object meta data \n", 386 | " obj_dict = {\n", 387 | " \"__class__\": obj.__class__.__name__,\n", 388 | " \"__module__\": obj.__module__\n", 389 | " }\n", 390 | " \n", 391 | " # Populate the dictionary with object properties\n", 392 | " obj_dict.update(obj.__dict__)\n", 393 | " \n", 394 | " return obj_dict\n", 395 | "\n", 396 | "\n", 397 | "def decode_dct(dct):\n", 398 | " \"\"\"\n", 399 | " Takes in a dict and returns a custom object associated with the dict.\n", 400 | " It makes use of the \"__module__\" and \"__class__\" metadata in the dictionary\n", 401 | " to know which object type to create.\n", 402 | " \"\"\"\n", 403 | " if \"__class__\" in dct:\n", 404 | " # Pop ensures we remove metadata from the dict to leave only the instance arguments\n", 405 | " class_name = dct.pop(\"__class__\")\n", 406 | " \n", 407 | " # Get the module name from the dict and import it\n", 408 | " module_name = dct.pop(\"__module__\")\n", 409 | " \n", 410 | " # We use the built in __import__ function since the module name is not yet known at runtime\n", 411 | " module = __import__(module_name)\n", 412 | " \n", 413 | " # Get the class from the module\n", 414 | " class_ = getattr(module,class_name)\n", 415 | "\n", 416 | " # Use dictionary unpacking to initialize the object\n", 417 | " # Note: This only works if all __init__() arguments of the class are exactly the dict keys\n", 418 | " obj = class_(**dct)\n", 419 | " else:\n", 420 | " obj = dct\n", 421 | " return obj\n", 422 | "\n", 423 | "# User class works with our encoding and decoding methods\n", 424 | "user = User(name = \"John\",age = 28, friends = [\"Jane\", \"Tom\"], balance = 20.70, active = True)\n", 425 | "\n", 426 | "userJSON = json.dumps(user,default=encode_obj,indent=4, sort_keys=True)\n", 427 | "print(userJSON)\n", 428 | "\n", 429 | "user_decoded = json.loads(userJSON, object_hook=decode_dct)\n", 430 | "print(type(user_decoded))\n", 431 | "\n", 432 | "# Player class also works with our custom encoding and decoding\n", 433 | "player = Player('Max', 'max1234', 5)\n", 434 | "playerJSON = json.dumps(player,default=encode_obj,indent=4, sort_keys=True)\n", 435 | "print(playerJSON)\n", 436 | "\n", 437 | "player_decoded = json.loads(playerJSON, object_hook=decode_dct)\n", 438 | "print(type(player_decoded))" 439 | ] 440 | } 441 | ], 442 | "metadata": { 443 | "kernelspec": { 444 | "display_name": "Python [conda env:pythonengineer_env]", 445 | "language": "python", 446 | "name": "conda-env-pythonengineer_env-py" 447 | }, 448 | "language_info": { 449 | "codemirror_mode": { 450 | "name": "ipython", 451 | "version": 3 452 | }, 453 | "file_extension": ".py", 454 | "mimetype": "text/x-python", 455 | "name": "python", 456 | "nbconvert_exporter": "python", 457 | "pygments_lexer": "ipython3", 458 | "version": "3.7.3" 459 | } 460 | }, 461 | "nbformat": 4, 462 | "nbformat_minor": 2 463 | } 464 | --------------------------------------------------------------------------------