├── .gitignore ├── LICENSE.md ├── README.md ├── default.nix ├── example4 ├── hello_world │ ├── Makefile │ ├── default.nix │ └── hello_world.c └── spinsfast.nix └── notebooks ├── nix-introduction.ipynb └── nix-presentation.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .ipynb_checkpoints 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright © `2018` `Chris Ostrouchov` 5 | 6 | Permission is hereby granted, free of charge, to any person 7 | obtaining a copy of this software and associated documentation 8 | files (the “Software”), to deal in the Software without 9 | restriction, including without limitation the rights to use, 10 | copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the 12 | Software is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice shall be 16 | included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, 19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 22 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | OTHER DEALINGS IN THE SOFTWARE. 26 | 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Nixpkgs](https://github.com/nixos/nixpkgs) BinderHub example 2 | 3 | [![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/costrouc/nix-binder-example/master) 4 | 5 | # Why Nix? 6 | 7 | [Nix](https://github.com/nixos/nixpkgs) would be a great addition to reproducible data science. It is a unique package manager. Some notable features: 8 | 9 | - 100% reproducible environments (pin to exact commit in repository) 10 | - both a source and binary package repository 11 | - allows customized compilation and version of every package 12 | - can run identical environment outside of docker (all linux distros + dawin) 13 | - as of now [45,000+ packages](https://repology.org/repositories/statistics/total) 14 | - fully declarative environments 15 | - packages: python, javascript, julia, R, haskell, perl, and many other languages (some better than others). 16 | 17 | Assuming that you have [`nix` 18 | installed](https://nixos.org/nix/download.html) (compatible with all 19 | linux distributions and darwin (Mac OS)) you can run this repository 20 | locally (no need for binderhub). It will be identical assuming you 21 | have pinned repositories. Nix can coexist fine with other package 22 | managers. 23 | 24 | ``` 25 | # is optional if in current directory 26 | nix-shell --run "jupyter lab" 27 | ``` 28 | 29 | # Example 1 30 | 31 | Lets start with the simplest `default.nix` I can imagine. 32 | 33 | ```nix 34 | { pkgs ? import { }, pythonPackages ? pkgs.python36Packages }: 35 | 36 | pkgs.mkShell { 37 | buildInputs = [ 38 | pythonPackages.numpy 39 | pythonPackages.scipy 40 | pythonPackages.jupyterlab 41 | ]; 42 | } 43 | ``` 44 | 45 | This will give you `python 3.6` with `jupyterlab`, `scipy`, and 46 | `numpy` installed. However there is one downside to this simple 47 | expression. The [packages within the `nixpkgs` derivations are not 48 | pinned](https://vaibhavsagar.com/blog/2018/05/27/quick-easy-nixpkgs-pinning/). 49 | This means that you have no guarantee of reproducibility and fixed 50 | versions. Don't worry this can be easily fixed and is why this is not 51 | the recommended way. Also while this demonstration only shows python 52 | packages nix has many more. For example [searching 53 | nixpkgs](https://nixos.org/nixos/packages) you could add 54 | `pkgs.google-cloud-sdk` and `pkgs.nodejs`. 55 | 56 | # Example 2 (with shellHook) 57 | 58 | ```nix 59 | { pkgs ? import { }, pythonPackages ? pkgs.python36Packages }: 60 | 61 | pkgs.mkShell { 62 | buildInputs = [ 63 | pythonPackages.numpy 64 | pythonPackages.scipy 65 | pythonPackages.jupyterlab 66 | ]; 67 | 68 | shellHook = '' 69 | if [ ! -f $HOME/.dockerbuildphase ]; then 70 | touch $HOME/.dockerbuildphase 71 | export DOCKER_BUILD_PHASE=true 72 | fi 73 | 74 | if [ "$DOCKER_BUILD_PHASE" = true ]; then 75 | echo "Do some action in build phase" 76 | fi 77 | 78 | if [ "$DOCKER_BUILD_PHASE" = false ]; then 79 | echo "Do some action in run phase" 80 | fi 81 | 82 | echo "Do some action in both phases" 83 | ''; 84 | } 85 | ``` 86 | 87 | Exactly the same example one except now we are able to execute shell 88 | commands before launching `jupyter`. This can include anything you can 89 | imagine but it will be run as a normal user (not root). A quick caveat 90 | with the `shellHooks` is that they are actually run twice. Once in the 91 | build phase (so that all of the `nixpkgs` dependencies are built and 92 | cached. And a second time to start a 93 | [nix-shell](https://nixos.org/nix/manual/#sec-nix-shell). I highly 94 | recommend that you do not put state into your `shellHook`. However, 95 | sometimes this is unavoidable when you want to start a database for 96 | instance before launching `jupyter lab`. 97 | 98 | # Example 3 (with pinned packages) 99 | 100 | ```nix 101 | let 102 | # Pinning nixpkgs to specific release 103 | # To get sha256 use "nix-prefetch-git --rev " 104 | commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a"; 105 | nixpkgs = builtins.fetchTarball { 106 | url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz"; 107 | sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg"; 108 | }; 109 | pkgs = import nixpkgs { config = { allowUnfree = true; }; }; 110 | pythonPackages = pkgs.python36Packages; 111 | in 112 | pkgs.mkShell { 113 | buildInputs = [ 114 | pythonPackages.numpy 115 | pythonPackages.scipy 116 | pythonPackages.jupyterlab 117 | ]; 118 | 119 | shellHook = '' 120 | echo "execute any bash commands before starting jupyterlab"; 121 | ''; 122 | } 123 | ``` 124 | 125 | Like before `python 36` will be installed with `jupyterlab`, `numpy`, 126 | and `scipy`. All this extra work guarantees that the versions of every 127 | package and configuration are pinned and fully reproducible to a git 128 | commit. `allowUnfree = true;` allows you to include unfree software in 129 | your environment. 130 | 131 | # Example 4 (Using nix for package building from source) 132 | 133 | ```nix 134 | { pkgs ? import { }, pythonPackages ? python36Packages }: 135 | 136 | pythonPackages.buildPythonPackage { 137 | pname = "spinsfast"; 138 | version = "unstable-528606f06d0dcd06c78de77cd2eeef404136f0ca"; 139 | 140 | src = fetchFromGitHub { 141 | owner = "moble"; 142 | repo = "spinsfast"; 143 | rev = "528606f06d0dcd06c78de77cd2eeef404136f0ca"; 144 | sha256 = "15hzrk2rji4v4qm26q8swyj4aqh8nsichybj6n12fwh067i8jzgf"; 145 | }; 146 | 147 | propagatedBuildInputs = [ pythonPackages.numpy pkgs.gsl pkgs.fftw ]; 148 | 149 | FFTW3_HOME = pkgs.fftw; 150 | GSL_HOME = pkgs.gsl; 151 | } 152 | ``` 153 | 154 | ```nix 155 | { pkgs ? import { }, pythonPackages ? pkgs.python36Packages }: 156 | 157 | let 158 | python-spinsfast = import example4/spinsfast.nix { }; 159 | my-local-hello-world = import example4/hello_world { }; 160 | in 161 | pkgs.mkShell { 162 | buildInputs = [ 163 | pythonPackages.numpy 164 | pythonPackages.scipy 165 | pythonPackages.jupyterlab 166 | python-spinsfast 167 | my-local-hello-world 168 | ]; 169 | } 170 | ``` 171 | 172 | Previously I have shown nix as a configuration tool for package 173 | management but it is also a great tool for building 174 | packages. Sometimes nixpkgs may not have the package that you want, 175 | might not have the most recent unstable release, or you need to 176 | package software that exists somewhere in a repository. Nixpkgs can 177 | handle all of this. [spinsfasts](https://github.com/moble/spinsfast) 178 | is a random project that I picked off of `pypi trending repositories` 179 | and yet the build derivation is quite simple. I wanted to also show 180 | that you can package local files (see `hello_world` and I set `src = 181 | ./.`). Nix will never rebuild a package if the configuration does not 182 | change and it exists in the cache (`/nix/store/...`). Notice that this 183 | means to nix there is no difference between a monorepo or distributed 184 | repositories. A huge win for developers. 185 | 186 | # Documentation and Further Reading 187 | 188 | This README was not designed to teach you to fully understand how nix 189 | works. Instead there is much better documentation below including blog 190 | posts. Nix can be confusing becuase there are many pieces: `nixpkgs` 191 | (packages), `nix` (the language), `nixos` (the operating system fully 192 | configured with nix). 193 | 194 | Contributions to [nixpkgs](https://github.com/nixos/nixpkgs) are 195 | always welcome and I am proud to say that we have a welcoming 196 | community. 197 | 198 | - [search for available packages](https://nixos.org/nixos/packages.html) 199 | - [great tutorial on nix language](http://www.binaryphile.com/nix/2018/07/22/nix-language-primer.html) 200 | - [nix language manual](https://nixos.org/nix/manual/) 201 | - [nixpkgs manual](https://nixos.org/nixpkgs/manual/) 202 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | let 2 | # Pinning nixpkgs to specific release 3 | # To get sha256 use "nix-prefetch-git --rev " 4 | commitRev="5574b6a152b1b3ae5f93ba37c4ffd1981f62bf5a"; 5 | nixpkgs = builtins.fetchTarball { 6 | url = "https://github.com/NixOS/nixpkgs/archive/${commitRev}.tar.gz"; 7 | sha256 = "1pqdddp4aiz726c7qs1dwyfzixi14shp0mbzi1jhapl9hrajfsjg"; 8 | }; 9 | pkgs = import nixpkgs { config = { allowUnfree = true; }; }; 10 | pythonPackages = pkgs.python36Packages; 11 | 12 | python-spinsfast = import example4/spinsfast.nix { pythonPackages = pythonPackages; }; 13 | my-local-hello-world = import example4/hello_world { }; 14 | in 15 | pkgs.mkShell { 16 | buildInputs = [ 17 | pythonPackages.numpy 18 | pythonPackages.scipy 19 | pythonPackages.jupyterlab 20 | 21 | # for demo and notebooks 22 | python-spinsfast 23 | my-local-hello-world 24 | pythonPackages.pythonix 25 | pythonPackages.graphviz 26 | pkgs.tree 27 | ]; 28 | 29 | shellHook = '' 30 | if [ ! -f $HOME/.dockerbuildphase ]; then 31 | touch $HOME/.dockerbuildphase 32 | export DOCKER_BUILD_PHASE=true 33 | fi 34 | 35 | if [ "$DOCKER_BUILD_PHASE" = true ]; then 36 | echo "Do some action in build phase" 37 | fi 38 | 39 | if [ "$DOCKER_BUILD_PHASE" = false ]; then 40 | echo "Do some action in run phase like start db" 41 | fi 42 | 43 | echo "Do some action in both phases" 44 | ''; 45 | } 46 | -------------------------------------------------------------------------------- /example4/hello_world/Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | 3 | build: 4 | gcc hello_world.c -o hello_world 5 | 6 | install: 7 | mkdir -p ${PREFIX}/bin/ 8 | cp hello_world ${PREFIX}/bin/ 9 | -------------------------------------------------------------------------------- /example4/hello_world/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | pkgs.stdenv.mkDerivation rec { 4 | name = "my-hello-world-${version}"; 5 | version = "unstable-foobarbaz"; 6 | 7 | src = ./.; 8 | 9 | installPhase = '' 10 | make install PREFIX=$out 11 | ''; 12 | } 13 | -------------------------------------------------------------------------------- /example4/hello_world/hello_world.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | printf("hello world"); 5 | } 6 | -------------------------------------------------------------------------------- /example4/spinsfast.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { }, pythonPackages ? pkgs.python36Packages }: 2 | 3 | pythonPackages.buildPythonPackage rec { 4 | pname = "spinsfast"; 5 | version = "unstable-528606f06d0dcd06c78de77cd2eeef404136f0ca"; 6 | 7 | src = pkgs.fetchFromGitHub { 8 | owner = "moble"; 9 | repo = "spinsfast"; 10 | rev = "528606f06d0dcd06c78de77cd2eeef404136f0ca"; 11 | sha256 = "15hzrk2rji4v4qm26q8swyj4aqh8nsichybj6n12fwh067i8jzgf"; 12 | }; 13 | 14 | propagatedBuildInputs = [ pythonPackages.numpy pkgs.gsl pkgs.fftw ]; 15 | 16 | FFTW3_HOME = pkgs.fftw; 17 | GSL_HOME = pkgs.gsl; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /notebooks/nix-introduction.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Welcome to a quick Nix Introduction!\n", 8 | "\n", 9 | "This is an overview of the nix commands for now. Maybe this will grow into a bigger example.\n", 10 | "\n", 11 | "First notice that the packages that we built are included in the environment.\n", 12 | " - Our local `hello_world` program\n", 13 | " - Our `spinsfast` python program is installed for the correct python version and included in our environment\n", 14 | " - python 3.6 is installed with `numpy`, `scipy`, and `jupyterlab` (along with all dependencies)\n", 15 | " \n", 16 | "Lets quickly inspect what we have" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 7, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "name": "stdout", 26 | "output_type": "stream", 27 | "text": [ 28 | "hello world" 29 | ] 30 | } 31 | ], 32 | "source": [ 33 | "# check output of hello_world program\n", 34 | "! hello_world" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 11, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "name": "stdout", 44 | "output_type": "stream", 45 | "text": [ 46 | "default.nix example4 notebooks README.md\n", 47 | "hello_world spinsfast.nix\n", 48 | "default.nix hello_world.c Makefile\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "# notice how clean nix leaves our directory\n", 54 | "# everything is copied to an isolated build directory\n", 55 | "# for building\n", 56 | "! ls ../\n", 57 | "! ls ../example4/\n", 58 | "! ls ../example4/hello_world/" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 12, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "import spinsfast\n", 68 | "import numpy\n", 69 | "import scipy\n", 70 | "import jupyterlab" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 4, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "3.6.6 (default, Jun 27 2018, 05:47:41) \n", 83 | "[GCC 7.3.0]\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "import sys\n", 89 | "print(sys.version)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 6, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "data": { 99 | "text/plain": [ 100 | "['',\n", 101 | " '/nix/store/3aqb8gxmjy4bd98zh01as7nrcyszf3c2-python3.6-numpy-1.15.1/lib/python3.6/site-packages',\n", 102 | " '/nix/store/hy65mn4wjswqih75gfr6g4q3xgqdm325-python3-3.6.6/lib/python3.6/site-packages',\n", 103 | " '/nix/store/2a8lm8w8ihlhnwid4xq6amvi755dbyz7-python3.6-setuptools-40.2.0/lib/python3.6/site-packages',\n", 104 | " '/nix/store/w3424y8ggxky8wjy8axqi024gf7ivbi2-python3.6-scipy-1.1.0/lib/python3.6/site-packages',\n", 105 | " '/nix/store/irnrpx2lkzajhhbwki2rwqvbzmh75as1-python3.6-jupyterlab-0.34.6/lib/python3.6/site-packages',\n", 106 | " '/nix/store/bwi2i35g11a049832g0zzsqhqz45y7xm-python3.6-ipython_genutils-0.2.0/lib/python3.6/site-packages',\n", 107 | " '/nix/store/21lxdak14zn4z4p8pxb4nrsivxfc1xvv-python3.6-jupyterlab_launcher-0.13.1/lib/python3.6/site-packages',\n", 108 | " '/nix/store/vgsbzajx3wdallz9hb0yywzsc3ypypaq-python3.6-jsonschema-2.6.0/lib/python3.6/site-packages',\n", 109 | " '/nix/store/94vqszxlf8basd5s3ni8wf5bq3h8p6x4-python3.6-notebook-5.6.0/lib/python3.6/site-packages',\n", 110 | " '/nix/store/f7vh7y804vqlahfsq1f0f9afdrcbdyq3-python3.6-Jinja2-2.10/lib/python3.6/site-packages',\n", 111 | " '/nix/store/qv1d6n1lwiv7skfz89cnz1xkfk5x8j2q-python3.6-markupsafe-1.0/lib/python3.6/site-packages',\n", 112 | " '/nix/store/j8yd57mbs9fis0dacl5yjwypgvpyv9z4-python3.6-tornado-5.1/lib/python3.6/site-packages',\n", 113 | " '/nix/store/ca86qlq73gbrb9hb3nphxph6cqmrzwac-python3.6-backports_abc-0.5/lib/python3.6/site-packages',\n", 114 | " '/nix/store/qhla67d2vh6xmpksy4lb43rr00qkh5yg-python3.6-certifi-2018.8.24/lib/python3.6/site-packages',\n", 115 | " '/nix/store/q7v4yvj8gzh06mlgap8ca27daldwg3pc-python3.6-singledispatch-3.4.0.3/lib/python3.6/site-packages',\n", 116 | " '/nix/store/xh9mfh0rhd7wi8by6hn57d14qlyqnpph-python3.6-six-1.11.0/lib/python3.6/site-packages',\n", 117 | " '/nix/store/zna268428fij3yrg1vi6l3d1ivl7wqgb-python3.6-traitlets-4.3.2/lib/python3.6/site-packages',\n", 118 | " '/nix/store/i36bzblisbis3pfkn00s1msr809ryhv0-python3.6-decorator-4.3.0/lib/python3.6/site-packages',\n", 119 | " '/nix/store/38cfzhxpvk97kahaz525s8wf87a46cfi-python3.6-jupyter_core-4.4.0/lib/python3.6/site-packages',\n", 120 | " '/nix/store/j8f2cj0axqhvjgm39qpr3calibdc9fzn-python3.6-ipython-6.5.0/lib/python3.6/site-packages',\n", 121 | " '/nix/store/5jwwcflsjpl1zwwxzzvv42x3vh5pyq0l-python3.6-jedi-0.12.1/lib/python3.6/site-packages',\n", 122 | " '/nix/store/0dnaxfngc1za833za1jgsbz70m97b8zq-python3.6-parso-0.3.1/lib/python3.6/site-packages',\n", 123 | " '/nix/store/bj6yhk4zhi2h0qnac3fdznz2k6bpcdjs-python3.6-pickleshare-0.7.4/lib/python3.6/site-packages',\n", 124 | " '/nix/store/kzn4zbw5vk6xqcvyjjsrkz62jlkw6gni-python3.6-path.py-11.0.1/lib/python3.6/site-packages',\n", 125 | " '/nix/store/dw3d7q66b6j27hayv4dxggg0vicbh9gs-python3.6-simplegeneric-0.8.1/lib/python3.6/site-packages',\n", 126 | " '/nix/store/zdn5w039qxalhrpgcqxms1ah7s1hlrgm-python3.6-prompt_toolkit-1.0.15/lib/python3.6/site-packages',\n", 127 | " '/nix/store/w1bsp4r7ndcdjlw6qjj28h89srilpmpd-python3.6-docopt-0.6.2/lib/python3.6/site-packages',\n", 128 | " '/nix/store/im7qrqspgl291dhx8885dismj3s4lx25-python3.6-wcwidth-0.1.7/lib/python3.6/site-packages',\n", 129 | " '/nix/store/fxvb6bczjxnai4aakbfdi452h0y0i246-python3.6-Pygments-2.2.0/lib/python3.6/site-packages',\n", 130 | " '/nix/store/pzvnw72cgd57jmrnr3wqc18g6kd23505-python3.6-docutils-0.14/lib/python3.6/site-packages',\n", 131 | " '/nix/store/6zq887wbbk2zlqd7j0jpkfvap1lc90ln-python3.6-pexpect-4.6.0/lib/python3.6/site-packages',\n", 132 | " '/nix/store/48iw34820bi03qxajqadbkcbffppi9bn-python3.6-ptyprocess-0.6.0/lib/python3.6/site-packages',\n", 133 | " '/nix/store/rnywq4i7zjmr8kxjhkhs7jrbx5r6wrmd-python3.6-backcall-0.1.0/lib/python3.6/site-packages',\n", 134 | " '/nix/store/mjgi2rjn6npw2w4wrkm5l5gzqb2cz8j9-python3.6-Send2Trash-1.4.2/lib/python3.6/site-packages',\n", 135 | " '/nix/store/mldl2xklwa03zc4ppi8ms2p72gyac0r8-python3.6-jupyter_client-5.2.3/lib/python3.6/site-packages',\n", 136 | " '/nix/store/5kk1n0b1sdxqzs56y6galig2p1yvq8nz-python3.6-pyzmq-17.1.2/lib/python3.6/site-packages',\n", 137 | " '/nix/store/bccb8jmszflv9hnwiinqwrdb83bfmr4j-python3.6-py-1.5.4/lib/python3.6/site-packages',\n", 138 | " '/nix/store/k9c78j8z4q20j27vjyjs411w1nla0809-python3.6-python-dateutil-2.7.3/lib/python3.6/site-packages',\n", 139 | " '/nix/store/1sl7znqlhifw81i3da6w8whqgdrv2i8r-python3.6-setuptools_scm-3.1.0/lib/python3.6/site-packages',\n", 140 | " '/nix/store/fqndazrpdiqcms7ivcqxy8gnmbw3xswx-python3.6-nbformat-4.4.0/lib/python3.6/site-packages',\n", 141 | " '/nix/store/pq26z30kkyb8229l8an6l6asq1wi2h9m-python3.6-testpath-0.3/lib/python3.6/site-packages',\n", 142 | " '/nix/store/qdivvcm3cqii4bwf609gdpkpas45iapf-python3.6-nbconvert-5.3.1/lib/python3.6/site-packages',\n", 143 | " '/nix/store/rsdwsww06w728n1bz7cswpj1lvh85xb6-python3.6-entrypoints-0.2.3/lib/python3.6/site-packages',\n", 144 | " '/nix/store/008fss5alrk8wp0f2p2pj1wsk3vcrwdw-python3.6-bleach-2.1.4/lib/python3.6/site-packages',\n", 145 | " '/nix/store/3hqfvm2s3lmai5gw27lxpm02zi0m0xg1-python3.6-html5lib-1.0.1/lib/python3.6/site-packages',\n", 146 | " '/nix/store/kh44p1x4z8cwwhav29csrdig0jskzndr-python3.6-webencodings-0.5.1/lib/python3.6/site-packages',\n", 147 | " '/nix/store/3y6fkpzjfqcalq8z769i7l07cj4fj2cz-python3.6-mistune-0.8.3/lib/python3.6/site-packages',\n", 148 | " '/nix/store/d70vr7s8v2ncj53mbp2v4w6nb5bk8zzz-python3.6-ipykernel-4.8.2/lib/python3.6/site-packages',\n", 149 | " '/nix/store/vky6qk0p6bigzw0xfq6a2myp1dh3vbmh-python3.6-pandocfilters-1.4.1/lib/python3.6/site-packages',\n", 150 | " '/nix/store/ajm635nxfwwag11pxjzgcibk2y4h42qc-python3.6-terminado-0.8.1/lib/python3.6/site-packages',\n", 151 | " '/nix/store/wphh2s31x39mqhb7188zyjv7sp08l5xn-python3.6-urllib3-1.23/lib/python3.6/site-packages',\n", 152 | " '/nix/store/sndz163kpv95n1rw8v11v2xw66vac8xq-python3.6-idna-2.7/lib/python3.6/site-packages',\n", 153 | " '/nix/store/1dn5q7j2ascy368vwdr6q9b0kxzr216l-python3.6-asn1crypto-0.24.0/lib/python3.6/site-packages',\n", 154 | " '/nix/store/4q1wccglx7l7vvhfdr46bfq1a3z9as1r-python3.6-packaging-17.1/lib/python3.6/site-packages',\n", 155 | " '/nix/store/55lagsqjr0yjv491vc7ia3d64iwmfwi8-python3.6-pyparsing-2.2.0/lib/python3.6/site-packages',\n", 156 | " '/nix/store/qq0058w2ydsa0a0gk984z5dk36jc1qjs-python3.6-pycparser-2.18/lib/python3.6/site-packages',\n", 157 | " '/nix/store/x5v2d0mh41yxc146w4p34zf3ka4q0ddn-python3.6-cffi-1.11.5/lib/python3.6/site-packages',\n", 158 | " '/nix/store/fjkrgww4vi05lz5vdpa3wwg4q08h1bmv-python3.6-cryptography-2.3.1/lib/python3.6/site-packages',\n", 159 | " '/nix/store/0g4mvk847nq8r6gq83d90wm8lvvhhr3z-python3.6-pyasn1-0.4.4/lib/python3.6/site-packages',\n", 160 | " '/nix/store/gzrgq8xlxsxnh18v2wm4aq2bh33063g7-python3.6-pyOpenSSL-18.0.0/lib/python3.6/site-packages',\n", 161 | " '/nix/store/hx8njjmhccmh6ngazc4dq4fiqmqkqvn2-python3.6-pysocks-1.6.6/lib/python3.6/site-packages',\n", 162 | " '/nix/store/w98icbhliwcisj6w3r8dragj2b2j5a4b-python3.6-chardet-3.0.4/lib/python3.6/site-packages',\n", 163 | " '/nix/store/66614b4gc9cw1vdyly98ahqdf0aapgb8-python3.6-requests-2.19.1/lib/python3.6/site-packages',\n", 164 | " '/nix/store/nsvi389pycl03hj7m5m6yvzn2b6vss1p-python3.6-prometheus_client-0.3.1/lib/python3.6/site-packages',\n", 165 | " '/nix/store/yx187xakxfxm3m4yf1k1i06m0j9gbdiy-python3.6-spinsfast-unstable-528606f06d0dcd06c78de77cd2eeef404136f0ca/lib/python3.6/site-packages',\n", 166 | " '/nix/store/2a8lm8w8ihlhnwid4xq6amvi755dbyz7-python3.6-setuptools-40.2.0/lib/python3.6/site-packages/setuptools-40.2.0-py3.6.egg',\n", 167 | " '/nix/store/hy65mn4wjswqih75gfr6g4q3xgqdm325-python3-3.6.6/lib/python36.zip',\n", 168 | " '/nix/store/hy65mn4wjswqih75gfr6g4q3xgqdm325-python3-3.6.6/lib/python3.6',\n", 169 | " '/nix/store/hy65mn4wjswqih75gfr6g4q3xgqdm325-python3-3.6.6/lib/python3.6/lib-dynload',\n", 170 | " '/nix/store/j8f2cj0axqhvjgm39qpr3calibdc9fzn-python3.6-ipython-6.5.0/lib/python3.6/site-packages/IPython/extensions',\n", 171 | " '/home/costrouc/.ipython']" 172 | ] 173 | }, 174 | "execution_count": 6, 175 | "metadata": {}, 176 | "output_type": "execute_result" 177 | } 178 | ], 179 | "source": [ 180 | "# here we learn the magic that nix does\n", 181 | "# all nix builds are isolated (including network) and only provided the recources they specify as dependencies\n", 182 | "# all nix builds are put in a flat directory `/nix/store/*`\n", 183 | "# by sacrificing POSIX compatibility we get many useful features\n", 184 | "# here nix sets `PYTHONPATH` to all of our python dependencies.\n", 185 | "sys.path" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "# Searching for nix packages\n", 193 | "\n", 194 | "Repology stats on nixpkgs https://repology.org/repositories/statistics/total\n", 195 | "\n", 196 | "Over 42,000 packages with 87% being the latest version.\n", 197 | "\n", 198 | "![nixpkgs image](https://repology.org/graph/map_repo_size_fresh.svg)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 15, 204 | "metadata": {}, 205 | "outputs": [ 206 | { 207 | "name": "stdout", 208 | "output_type": "stream", 209 | "text": [ 210 | "\u001b[K\u001b[31;1mwarning:\u001b[0m using cached results; pass '-u' to update the cache\u001b[0m\n", 211 | "\u001b[K* \u001b[0;1mnixpkgs.\u001b[31;1mprometheus-s\u001b[0;1mnmp-exporter\u001b[0m (\u001b[0;2msnmp_exporter-0.13.0\u001b[0m)\n", 212 | " SNMP Exporter for Prometheus\n", 213 | "\n", 214 | "* \u001b[0;1mnixpkgs.\u001b[31;1mprometheus-s\u001b[0;1mtatsd-bridge\u001b[0m (\u001b[0;2mstatsd_exporter-0.4.0\u001b[0m)\n", 215 | " Receives StatsD-style metrics and exports them to Prometheus\n", 216 | "\n", 217 | "* \u001b[0;1mnixpkgs.\u001b[31;1mprometheus-s\u001b[0;1mtatsd-exporter\u001b[0m (\u001b[0;2mstatsd_exporter-0.4.0\u001b[0m)\n", 218 | " Receives StatsD-style metrics and exports them to Prometheus\n", 219 | "\n", 220 | "* \u001b[0;1mnixpkgs.\u001b[31;1mprometheus-s\u001b[0;1murfboard-exporter\u001b[0m (\u001b[0;2msurfboard_exporter-2.0.0\u001b[0m)\n", 221 | " Arris Surfboard signal metrics exporter\n", 222 | "\n", 223 | "\u001b[K" 224 | ] 225 | } 226 | ], 227 | "source": [ 228 | "! nix search prometheus-s" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "# Nix stores all packages in `/nix/store/*`" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 17, 241 | "metadata": {}, 242 | "outputs": [ 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "24G\t/nix/store\n" 248 | ] 249 | } 250 | ], 251 | "source": [ 252 | "! du -sh /nix/store" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": null, 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [] 261 | } 262 | ], 263 | "metadata": { 264 | "kernelspec": { 265 | "display_name": "Python 3", 266 | "language": "python", 267 | "name": "python3" 268 | }, 269 | "language_info": { 270 | "codemirror_mode": { 271 | "name": "ipython", 272 | "version": 3 273 | }, 274 | "file_extension": ".py", 275 | "mimetype": "text/x-python", 276 | "name": "python", 277 | "nbconvert_exporter": "python", 278 | "pygments_lexer": "ipython3", 279 | "version": "3.6.6" 280 | } 281 | }, 282 | "nbformat": 4, 283 | "nbformat_minor": 2 284 | } 285 | -------------------------------------------------------------------------------- /notebooks/nix-presentation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Nix / Nixpkgs / NixOS\n", 8 | "\n", 9 | "In this presentation when I refer to `nix` I am refering to the community of tools. `nix` is an ecosystem of tools and ideas that rethink packaging, development, OS configuration, containers, and configuration management in general. I hope that this presentation can show how `nix` can be used. Nix is compatible with Darwin (OSX) and all linux distributions (including ARM and FreeBSD).\n", 10 | "\n", 11 | "To confuse things the community is based on a programming language called `nix`. Some noteable features.\n", 12 | " - functional programming language\n", 13 | " - lazy lazy evaluation (only evaluate once and never evaluate if not used)\n", 14 | " - evaluations are compatible with json/yaml representation\n", 15 | " \n", 16 | "# Nix" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 14, 22 | "metadata": {}, 23 | "outputs": [ 24 | { 25 | "data": { 26 | "text/plain": [ 27 | "{'x': 1, 'y': 12, 'z': [13, 10]}" 28 | ] 29 | }, 30 | "execution_count": 14, 31 | "metadata": {}, 32 | "output_type": "execute_result" 33 | } 34 | ], 35 | "source": [ 36 | "import nix\n", 37 | "\n", 38 | "nix.eval('''\n", 39 | "let a = 1;\n", 40 | " b = { c = a; }.d;\n", 41 | " f = v: v + 1;\n", 42 | "in\n", 43 | "rec {\n", 44 | " x = a;\n", 45 | " y = f 11;\n", 46 | " z = [ (x + y) 10 ];\n", 47 | "}\n", 48 | "''')" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "The expression above shows many nix concepts. `let ... in` allows for local variable declarations outside of the returned expression. `rec` allows for recursive attribute definitions. `{ ... }` are equivalent to dictionaries. `[ ]` are lists. We also have a function declaration `f = v: v + 1;` which takes a given value and adds 1 to it.\n", 56 | "\n", 57 | "Notice that there is an error in the expression above but it is not evaluated in nix so there is no problem. `b = { c = a; }.d;` -> `b = { c = 1; }.d;` -> \"no d attribute in `{ c = 1; }`. This shows just how lazy nix is! For a much better introduction see [nix language primer](http://www.binaryphile.com/nix/2018/07/22/nix-language-primer.html). Nix fundamental types are ints, floats, strings, filepaths(including http://...), lists, and dicts. Note that filepaths are a type in nix which is one reason why nix is a great configuration language. Also supports `imports` that can be remote or local allowing truely composable configuration.\n", 58 | "\n", 59 | "A question that many commonly come up is why not [yaml, json, or X](https://discourse.nixos.org/t/why-not-use-yaml-for-configuration-and-package-declaration/1333). Take a look around who is using plain `json` or `yaml` for package configuration? Conda uses `meta.yaml` which is yaml + jinja2. Teraform configuration is [HashiCorp Configuration Language (HCL)](https://www.terraform.io/docs/configuration/syntax.html) which is sometimes json \"like\". Ansible is `yaml` + jinja2 and their custom magic. The point is that configuration tools require **abstration** to remove the tedious work. It is my personal belief that `nix` is an excelent replacement for these with a strong theoretical backbone based on a [dissertation by Eelco Dolstra](https://grosskurth.ca/bib/2006/dolstra-thesis.pdf).\n", 60 | "\n", 61 | "# Nixpkgs\n", 62 | "\n", 63 | "[nixpkgs](https://github.com/NixOS/nixpkgs) is the realization of a purely functional package manager." 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 6, 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "data": { 73 | "image/svg+xml": [ 74 | "\n", 75 | "\n", 77 | "\n", 79 | "\n", 80 | "\n", 82 | "\n", 83 | "%3\n", 84 | "\n", 85 | "\n", 86 | "\n", 87 | "A\n", 88 | "\n", 89 | "configuration\n", 90 | "\n", 91 | "\n", 92 | "\n", 93 | "E\n", 94 | "\n", 95 | "package derivation\n", 96 | "\n", 97 | "\n", 98 | "\n", 99 | "A->E\n", 100 | "\n", 101 | "\n", 102 | "\n", 103 | "\n", 104 | "\n", 105 | "B\n", 106 | "\n", 107 | "package source\n", 108 | "\n", 109 | "\n", 110 | "\n", 111 | "B->A\n", 112 | "\n", 113 | "\n", 114 | "\n", 115 | "\n", 116 | "\n", 117 | "C\n", 118 | "\n", 119 | "dependency A\n", 120 | "\n", 121 | "\n", 122 | "\n", 123 | "C->A\n", 124 | "\n", 125 | "\n", 126 | "\n", 127 | "\n", 128 | "\n", 129 | "D\n", 130 | "\n", 131 | "dependency B\n", 132 | "\n", 133 | "\n", 134 | "\n", 135 | "D->A\n", 136 | "\n", 137 | "\n", 138 | "\n", 139 | "\n", 140 | "\n" 141 | ], 142 | "text/plain": [ 143 | "" 144 | ] 145 | }, 146 | "execution_count": 6, 147 | "metadata": {}, 148 | "output_type": "execute_result" 149 | } 150 | ], 151 | "source": [ 152 | "from graphviz import Digraph\n", 153 | "\n", 154 | "dot = Digraph()\n", 155 | "dot.node('A', 'configuration')\n", 156 | "dot.node('B', 'package source')\n", 157 | "dot.node('C', 'dependency A')\n", 158 | "dot.node('D', 'dependency B')\n", 159 | "dot.node('E', 'package derivation')\n", 160 | "dot.edges(['BA', 'CA', 'DA', 'AE'])\n", 161 | "dot" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "Nixpkgs applies the following constraints:\n", 169 | " - all packages are built in a sandboxed environment (no network, chroot with only dependencies in filepath, etc)\n", 170 | " - each package derivation is put in a flat read-only directory `/nix/store/--`\n", 171 | " \n", 172 | "Interesting consequences:\n", 173 | "\n", 174 | "1. If a package builds on your machine then it is nearly guarenteed to build on elsewhere (nixpkgs never sees issues of \"it builds here but not there\")\n", 175 | "2. Since we have all dependecies packaged in a directory installation is simply downloading a tarball (no execution)\n", 176 | "3. The sha256 hash of the configuration and its dependencies allows us to `cache` builds thus nix is a source and binary package manager\n", 177 | "4. Easily support multiple versions of same package along with different configurations (e.g. single vs double precision)\n", 178 | "5. Package builds can be distributed (since all dependencies are well defined)\n", 179 | "6. Packages are now the new \"Container\"\n", 180 | "7. All build configuration is completely reproducible and feasable to manage in source control\n", 181 | "\n", 182 | "# Nixpkgs Derivation\n", 183 | "\n", 184 | "What is a derivation? A derivation is a configuration for a given package that defines all dependencies and configuration to build a given package.\n", 185 | "\n", 186 | "Example packaging `c` [xnd library](https://xnd.io/)\n", 187 | "\n", 188 | "```nix\n", 189 | "{ lib\n", 190 | ", stdenv\n", 191 | ", fetchFromGitHub\n", 192 | ", libndtypes\n", 193 | "}:\n", 194 | "\n", 195 | "stdenv.mkDerivation rec {\n", 196 | " name = \"libxnd-${version}\";\n", 197 | " version = \"0.2.0dev3\";\n", 198 | "\n", 199 | " src = fetchFromGitHub {\n", 200 | " owner = \"plures\";\n", 201 | " repo = \"xnd\";\n", 202 | " rev = \"v${version}\";\n", 203 | " sha256 = \"0byq7jspyr2wxrhihw4q7nf0y4sb6j5ax0ndd5dnq5dz88c7qqm2\";\n", 204 | " };\n", 205 | "\n", 206 | " buildInputs = [ libndtypes ];\n", 207 | "\n", 208 | " # Override linker with cc (symlink to either gcc or clang)\n", 209 | " # Library expects to use cc for linking\n", 210 | " configureFlags = [ \"LD=${stdenv.cc.targetPrefix}cc\" ];\n", 211 | "\n", 212 | " meta = {\n", 213 | " description = \"General container that maps a wide range of Python values directly to memory\";\n", 214 | " homepage = https://xnd.io/;\n", 215 | " license = lib.licenses.bsdOriginal;\n", 216 | " maintainers = with lib.maintainers; [ costrouc ];\n", 217 | " };\n", 218 | "}\n", 219 | "```\n", 220 | "\n", 221 | "Great documentation on package in [nixpkgs manual](https://nixos.org/nixpkgs/manual/). Lets explain some of the attributes here `name` is the name of the package that is appended to the configuration sha256 hash. `src` is how to collect the source core pertaining to a given package (notice the `sha256` hash). `buildInputs` specifies dependencies that are required to build packages. `mkDerivation` is a function that creates a built script from all of the attributes. changing `configureFlags` allows you to specifiy additional `./configure` flags. `meta` is metadata about the given package.\n", 222 | "\n", 223 | "Below we will looks at what a derivation really is." 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 7, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "name": "stdout", 233 | "output_type": "stream", 234 | "text": [ 235 | "{'attributes': ['__ignoreNulls', 'all', 'args', 'buildInputs', 'builder', 'configureFlags', 'configurePhase', 'depsBuildBuild', 'depsBuildBuildPropagated', 'depsBuildTarget', 'depsBuildTargetPropagated', 'depsHostHost', 'depsHostHostPropagated', 'depsTargetTarget', 'depsTargetTargetPropagated', 'doCheck', 'doInstallCheck', 'drvAttrs', 'drvPath', 'meta', 'name', 'nativeBuildInputs', 'out', 'outPath', 'outputName', 'outputUnspecified', 'outputs', 'override', 'overrideAttrs', 'overrideDerivation', 'passthru', 'propagatedBuildInputs', 'propagatedNativeBuildInputs', 'src', 'stdenv', 'strictDeps', 'system', 'type', 'userHook'], 'storePath': '/nix/store/bskfav26x2xify79w2kc824k3fiwyika-tree-1.7.0'}\n" 236 | ] 237 | } 238 | ], 239 | "source": [ 240 | "import nix\n", 241 | "\n", 242 | "print(nix.eval('''\n", 243 | "let pkgs = import { };\n", 244 | "in { \n", 245 | " attributes = builtins.attrNames pkgs.tree;\n", 246 | " storePath = pkgs.tree;\n", 247 | "}\n", 248 | "'''))" 249 | ] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": 9, 254 | "metadata": {}, 255 | "outputs": [ 256 | { 257 | "name": "stdout", 258 | "output_type": "stream", 259 | "text": [ 260 | "/nix/store/bskfav26x2xify79w2kc824k3fiwyika-tree-1.7.0\n", 261 | "├── bin\n", 262 | "│   └── tree\n", 263 | "└── share\n", 264 | " └── man\n", 265 | " └── man1\n", 266 | " └── tree.1.gz\n", 267 | "\n", 268 | "4 directories, 2 files\n", 269 | "\tlinux-vdso.so.1 (0x00007ffcb695a000)\n", 270 | "\tlibc.so.6 => /nix/store/fg4yq8i8wd08xg3fy58l6q73cjy8hjr2-glibc-2.27/lib/libc.so.6 (0x00007fe295470000)\n", 271 | "\t/nix/store/fg4yq8i8wd08xg3fy58l6q73cjy8hjr2-glibc-2.27/lib/ld-linux-x86-64.so.2 => /nix/store/fg4yq8i8wd08xg3fy58l6q73cjy8hjr2-glibc-2.27/lib64/ld-linux-x86-64.so.2 (0x00007fe295824000)\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "# notice that built result is isolated\n", 277 | "! tree /nix/store/bskfav26x2xify79w2kc824k3fiwyika-tree-1.7.0\n", 278 | "\n", 279 | "# shared libraries are packaged within nix dependency tree\n", 280 | "! ldd /nix/store/bskfav26x2xify79w2kc824k3fiwyika-tree-1.7.0/bin/tree" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "Packages tend to have many dependencies and `/nix/store/*` will contain all packages along with different configurations. I have around 45,000 packages on my machine. Note for me this includes npm packages, python packages, etc." 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 11, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "name": "stdout", 297 | "output_type": "stream", 298 | "text": [ 299 | " 44307 38180 1678771\n" 300 | ] 301 | } 302 | ], 303 | "source": [ 304 | "! ls /nix/store/* | wc" 305 | ] 306 | }, 307 | { 308 | "cell_type": "markdown", 309 | "metadata": {}, 310 | "source": [ 311 | "# nix-shell\n", 312 | "\n", 313 | "`nix-shell` or `nix run` is a tool that allows you to create true \"virtual environments\". Since `nixpkgs` keeps track of all dependencies we can modify the environment variables along with using symlinks to produce a virutual environment for a given program. **Using `nix-shell` we can produce development environments that are identical and reproducible between developers!**\n", 314 | "\n", 315 | "```bash\n", 316 | "nix-shell -p htop python37 nodejs emacs\n", 317 | "```\n", 318 | "\n", 319 | "We will produce a virtual environment with all of the following packages available. We can execute a different command if we wish. The `--pure` option tells `nix-shell` to do its best to completely issolate the environment (clear `PATH` etc.). All of this can be specified in a nix derivation if you like.\n", 320 | "\n", 321 | "`shell.nix`\n", 322 | "```nix\n", 323 | "{ pkgs ? import { } }:\n", 324 | "\n", 325 | "pkgs.mkShell {\n", 326 | " buildInputs = [\n", 327 | " pkgs.htop\n", 328 | " pkgs.python37\n", 329 | " pkgs.nodejs\n", 330 | " pkgs.emacs\n", 331 | " ];\n", 332 | " \n", 333 | " shellHook = ''\n", 334 | " emacs --version\n", 335 | " node --version\n", 336 | " python --version\n", 337 | " '';\n", 338 | "}\n", 339 | "```\n", 340 | "\n", 341 | "The command bellow is eqivalent to `nix-shell shell.nix` above." 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 15, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "name": "stdout", 351 | "output_type": "stream", 352 | "text": [ 353 | "GNU Emacs 26.1\n", 354 | "Copyright (C) 2018 Free Software Foundation, Inc.\n", 355 | "GNU Emacs comes with ABSOLUTELY NO WARRANTY.\n", 356 | "You may redistribute copies of GNU Emacs\n", 357 | "under the terms of the GNU General Public License.\n", 358 | "For more information about these matters, see the file named COPYING.\n", 359 | "v8.11.4\n", 360 | "Python 3.7.0 (default, Jun 27 2018, 06:06:01) \n", 361 | "[GCC 7.3.0]\n" 362 | ] 363 | } 364 | ], 365 | "source": [ 366 | "! nix-shell -p htop python37 nodejs emacs --run \"emacs --version; node --version; python --version\" --pure" 367 | ] 368 | }, 369 | { 370 | "cell_type": "markdown", 371 | "metadata": {}, 372 | "source": [ 373 | "[`nix-shell` magic with `shebangs`](http://chriswarbo.net/projects/nixos/nix_shell_shebangs.html). Okay want to see something really really cool?" 374 | ] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "execution_count": 24, 379 | "metadata": {}, 380 | "outputs": [ 381 | { 382 | "name": "stdout", 383 | "output_type": "stream", 384 | "text": [ 385 | "Overwriting /tmp/script.py\n" 386 | ] 387 | } 388 | ], 389 | "source": [ 390 | "%%writefile /tmp/script.py\n", 391 | "#! /usr/bin/env nix-shell\n", 392 | "#! nix-shell -i python -p python36Packages.flask\n", 393 | "import flask\n", 394 | "print('Flask version', flask.__version__)" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": 26, 400 | "metadata": {}, 401 | "outputs": [ 402 | { 403 | "name": "stdout", 404 | "output_type": "stream", 405 | "text": [ 406 | "Traceback (most recent call last):\n", 407 | " File \"\", line 1, in \n", 408 | "ModuleNotFoundError: No module named 'flask'\n", 409 | "Do some action in both phases\n", 410 | "Flask version 1.0.2\n" 411 | ] 412 | } 413 | ], 414 | "source": [ 415 | "# notice we dont have flask installed\n", 416 | "! python -c \"import flask\"\n", 417 | "\n", 418 | "# Now lets execute script (woah we have flask!)\n", 419 | "! chmod +x /tmp/script.py && /tmp/script.py" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "# Docker Images without Docker with optimized Cache Layers\n", 427 | "\n", 428 | "`htop-container.nix`\n", 429 | "\n", 430 | "```nix\n", 431 | "{ pkgs ? import {}; }:\n", 432 | "\n", 433 | "pkgs.dockerTools.buildImage { \n", 434 | " name = \"nix-htop\"; \n", 435 | " contents = [ pkgs.htop pkgs.emacs pkgs.nodejs ]; \n", 436 | " config = { \n", 437 | " Cmd = [ \"/bin/htop\" ]; \n", 438 | " };\n", 439 | "};\n", 440 | "```\n", 441 | "\n", 442 | "```shell\n", 443 | "nix-build htop-container.nix\n", 444 | "docker load --input \n", 445 | "```\n", 446 | "\n", 447 | "Seriously it is this easy to build docker containers without root and without docker. `nix-build` will print the path to the docker build. It can then be imported with docker." 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "metadata": {}, 453 | "source": [ 454 | "# NixOS\n", 455 | "\n", 456 | "NixOS is a distribution of Linux that realizes `nixpkgs` + configuration. There are similar ideas that [manage dotfiles](https://github.com/rycee/home-manager) and [kubernetes](https://github.com/xtruder/kubenix). The idea is if we have all the packages installed via `nixpkgs` and use configuration to control systemd services we can fully specify an OS. We now have a declarative way of specifying operating systems.\n", 457 | "\n", 458 | "What does NixOS have to offer?\n", 459 | " - **atomic upgrades and rollbacks**. Seriously if your machine does not boot or there is an error just boot into a previous version using grub.\n", 460 | " - fully declarative specification of all packages and running processes on machine\n", 461 | " - version controllable OS\n", 462 | "\n", 463 | "```nix\n", 464 | "{ config, pkgs, ... }: \n", 465 | "\n", 466 | "{\n", 467 | " networking = {\n", 468 | " hostName = \"foobar\";\n", 469 | " extraHosts =\n", 470 | " ''\n", 471 | " 10.0.0.1 bizbaz\n", 472 | " '';\n", 473 | " firewall.allowedTCPPorts = [ 8000 ];\n", 474 | " };\n", 475 | "\n", 476 | " services.emacs.enable = true;\n", 477 | "\n", 478 | " environment.systemPackages = [\n", 479 | " pkgs.firefox pkgs.htop pkgs.python37\n", 480 | " ];\n", 481 | "}\n", 482 | "```" 483 | ] 484 | }, 485 | { 486 | "cell_type": "markdown", 487 | "metadata": {}, 488 | "source": [ 489 | "# Nixops\n", 490 | "\n", 491 | "Deploy NixOS configuration to cloud providers, VMs, and remote NixOS deployments. Nixops allows you to declaratively specify the configuration of every machine in your cluster." 492 | ] 493 | }, 494 | { 495 | "cell_type": "code", 496 | "execution_count": null, 497 | "metadata": {}, 498 | "outputs": [], 499 | "source": [] 500 | } 501 | ], 502 | "metadata": { 503 | "kernelspec": { 504 | "display_name": "Python 3", 505 | "language": "python", 506 | "name": "python3" 507 | }, 508 | "language_info": { 509 | "codemirror_mode": { 510 | "name": "ipython", 511 | "version": 3 512 | }, 513 | "file_extension": ".py", 514 | "mimetype": "text/x-python", 515 | "name": "python", 516 | "nbconvert_exporter": "python", 517 | "pygments_lexer": "ipython3", 518 | "version": "3.6.6" 519 | } 520 | }, 521 | "nbformat": 4, 522 | "nbformat_minor": 2 523 | } 524 | --------------------------------------------------------------------------------