├── docker ├── .env ├── bin │ ├── tensorboard │ ├── nbdiff_checkpoint │ ├── rm_empty_subdirs │ └── nbclean_checkpoints ├── bashrc.bash ├── Makefile ├── docker-compose.yml ├── jupyter_notebook_config.py ├── README.md └── Dockerfile ├── images ├── ann │ └── README ├── cnn │ ├── README │ └── test_image.png ├── deep │ └── README ├── nlp │ └── README ├── rl │ ├── README │ └── breakout.gif ├── rnn │ └── README ├── svm │ └── README ├── distributed │ └── README ├── ensembles │ └── README ├── tensorflow │ └── README ├── autoencoders │ └── README ├── classification │ └── README ├── decision_trees │ └── README ├── fundamentals │ └── README ├── end_to_end_project │ ├── README │ └── california.png ├── unsupervised_learning │ ├── README │ └── ladybug.png └── training_linear_models │ └── README ├── book_equations.pdf ├── datasets ├── housing │ ├── housing.tgz │ └── README.md ├── lifesat │ ├── gdp_per_capita.csv │ └── README.md ├── jsb_chorales │ ├── jsb_chorales.tgz │ └── README.md └── inception │ └── imagenet_class_names.txt ├── apt.txt ├── .gitignore ├── environment.yml ├── environment-windows.yml ├── requirements.txt ├── README.md ├── index.ipynb ├── INSTALL.md ├── changes_in_2nd_edition.md ├── LICENSE ├── extra_autodiff.ipynb └── 19_training_and_deploying_at_scale.ipynb /docker/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=handson-ml2 2 | -------------------------------------------------------------------------------- /images/ann/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/cnn/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/deep/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/nlp/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/rl/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/rnn/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/svm/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/distributed/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/ensembles/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/tensorflow/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/autoencoders/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/classification/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/decision_trees/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/fundamentals/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/end_to_end_project/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /images/unsupervised_learning/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /docker/bin/tensorboard: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python -m tensorboard.main "$@" 3 | -------------------------------------------------------------------------------- /images/training_linear_models/README: -------------------------------------------------------------------------------- 1 | Images generated by the notebooks 2 | -------------------------------------------------------------------------------- /book_equations.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/book_equations.pdf -------------------------------------------------------------------------------- /images/rl/breakout.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/images/rl/breakout.gif -------------------------------------------------------------------------------- /images/cnn/test_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/images/cnn/test_image.png -------------------------------------------------------------------------------- /datasets/housing/housing.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/datasets/housing/housing.tgz -------------------------------------------------------------------------------- /docker/bashrc.bash: -------------------------------------------------------------------------------- 1 | alias ll="ls -alF" 2 | alias nbd="nbdiff_checkpoint" 3 | alias tb="tensorboard --logdir=tf_logs" 4 | -------------------------------------------------------------------------------- /datasets/lifesat/gdp_per_capita.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/datasets/lifesat/gdp_per_capita.csv -------------------------------------------------------------------------------- /datasets/jsb_chorales/jsb_chorales.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/datasets/jsb_chorales/jsb_chorales.tgz -------------------------------------------------------------------------------- /images/end_to_end_project/california.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/images/end_to_end_project/california.png -------------------------------------------------------------------------------- /images/unsupervised_learning/ladybug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leriomaggio/handson-ml2/master/images/unsupervised_learning/ladybug.png -------------------------------------------------------------------------------- /apt.txt: -------------------------------------------------------------------------------- 1 | build-essential 2 | cmake 3 | ffmpeg 4 | git 5 | libboost-all-dev 6 | libjpeg-dev 7 | libpq-dev 8 | libsdl2-dev 9 | sudo 10 | swig 11 | unzip 12 | xorg-dev 13 | xvfb 14 | zip 15 | zlib1g-dev 16 | -------------------------------------------------------------------------------- /docker/Makefile: -------------------------------------------------------------------------------- 1 | 2 | help: 3 | cat Makefile 4 | run: 5 | docker-compose up 6 | exec: 7 | docker-compose exec handson-ml2 bash 8 | build: stop .FORCE 9 | docker-compose build 10 | rebuild: stop .FORCE 11 | docker-compose build --force-rm 12 | stop: 13 | docker stop handson-ml2 || true; docker rm handson-ml2 || true; 14 | .FORCE: 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.bak 2 | *.bak.* 3 | *.ckpt 4 | *.old 5 | *.pyc 6 | .DS_Store 7 | .ipynb_checkpoints 8 | checkpoint 9 | logs/* 10 | tf_logs/* 11 | images/**/*.png 12 | images/**/*.dot 13 | my_* 14 | person.proto 15 | person.desc 16 | person_pb2.py 17 | datasets/flowers 18 | datasets/lifesat/lifesat.csv 19 | datasets/spam 20 | datasets/titanic 21 | datasets/words 22 | datasets/jsb_chorales 23 | 24 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | handson-ml2: 4 | build: 5 | context: ../ 6 | dockerfile: ./docker/Dockerfile 7 | args: 8 | - username=devel 9 | - userid=1000 10 | container_name: handson-ml2 11 | image: handson-ml2 12 | restart: unless-stopped 13 | logging: 14 | driver: json-file 15 | options: 16 | max-size: 50m 17 | ports: 18 | - "8888:8888" 19 | - "6006:6006" 20 | volumes: 21 | - ../:/home/devel/handson-ml2 22 | command: /opt/conda/envs/tf2/bin/jupyter notebook --ip='0.0.0.0' --port=8888 --no-browser 23 | -------------------------------------------------------------------------------- /docker/bin/nbdiff_checkpoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [[ "$#" -lt 1 || "$1" =~ ^((-h)|(--help))$ ]] ; then 3 | echo "usage: nbdiff_checkpoint NOTEBOOK.ipynb" 4 | echo 5 | echo "Show differences between given jupyter notebook and its checkpointed version (in .ipynb_checkpoints subdirectory)" 6 | exit 7 | fi 8 | 9 | DIRNAME=$(dirname "$1") 10 | BASENAME=$(basename "$1" .ipynb) 11 | shift 12 | 13 | WORKING_COPY=$DIRNAME/$BASENAME.ipynb 14 | CHECKPOINT_COPY=$DIRNAME/.ipynb_checkpoints/$BASENAME-checkpoint.ipynb 15 | 16 | echo "----- Analysing how to change $CHECKPOINT_COPY into $WORKING_COPY -----" 17 | nbdiff "$CHECKPOINT_COPY" "$WORKING_COPY" --ignore-details "$@" 18 | -------------------------------------------------------------------------------- /docker/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | def export_script_and_view(model, os_path, contents_manager): 5 | if model["type"] != "notebook": 6 | return 7 | dir_name, file_name = os.path.split(os_path) 8 | file_base, file_ext = os.path.splitext(file_name) 9 | if file_base.startswith("Untitled"): 10 | return 11 | export_name = file_base if file_ext == ".ipynb" else file_name 12 | subprocess.check_call(["jupyter", "nbconvert", "--to", "script", file_name, "--output", export_name + "_script"], cwd=dir_name) 13 | subprocess.check_call(["jupyter", "nbconvert", "--to", "html", file_name, "--output", export_name + "_view"], cwd=dir_name) 14 | 15 | c.FileContentsManager.post_save_hook = export_script_and_view 16 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: tf2 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - graphviz 7 | - imageio=2.6 8 | - ipython=7.12 9 | - ipywidgets=7.5 10 | - joblib=0.14 11 | - jupyter=1.0 12 | - matplotlib=3.1 13 | - nbdime=2.0 14 | - nltk=3.4 15 | - numexpr=2.7 16 | - numpy=1.18 17 | - pandas=1.0 18 | - pillow=7.0 19 | - pip 20 | - psutil=5.7 21 | - py-xgboost=0.90 22 | - pydot=1.4 23 | - pyglet=1.5 24 | - pyopengl=3.1 25 | - python=3.7 26 | - python-graphviz 27 | #- pyvirtualdisplay=0.2 # add if on headless server 28 | - requests=2.22 29 | - scikit-image=0.16 30 | - scikit-learn=0.22 31 | - scipy=1.4 32 | - tqdm=4.43 33 | - wheel 34 | - widgetsnbextension=3.5 35 | - pip: 36 | - atari-py==0.2.6 37 | - ftfy==5.7 38 | - gast==0.2.2 39 | - gym==0.17.1 40 | - opencv-python==4.2.0.32 41 | - spacy==2.2.4 42 | - tensorboard==2.1.1 43 | - tensorflow-addons==0.8.3 44 | - tensorflow-data-validation==0.21.5 45 | - tensorflow-datasets==2.1.0 46 | - tensorflow-estimator==2.1.0 47 | - tensorflow-hub==0.7.0 48 | - tensorflow-metadata==0.21.1 49 | - tensorflow-model-analysis==0.21.6 50 | - tensorflow-probability==0.9.0 51 | - tensorflow-serving-api==2.1.0 # or tensorflow-serving-api-gpu if gpu 52 | - tensorflow-transform==0.21.2 53 | - tensorflow==2.1.0 # or tensorflow-gpu if gpu 54 | - tf-agents==0.3.0 55 | - tfx==0.21.2 56 | - transformers==2.8.0 57 | - urlextract==0.14.0 58 | -------------------------------------------------------------------------------- /environment-windows.yml: -------------------------------------------------------------------------------- 1 | name: tf2 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - graphviz 7 | - imageio=2.6.1 8 | - ipython=7.10.1 9 | - ipywidgets=7.5.1 10 | - joblib=0.14.0 11 | - jupyter=1.0.0 12 | - matplotlib=3.1.2 13 | - nbdime=1.1.0 14 | - nltk=3.4.5 15 | - numexpr=2.7.0 16 | - numpy=1.17.3 17 | - pandas=0.25.3 18 | - pillow=6.2.1 19 | - pip 20 | - py-xgboost=0.90 21 | - pydot=1.4.1 22 | - pyopengl=3.1.3b2 23 | - python=3.7 24 | - python-graphviz 25 | - requests=2.22.0 26 | - scikit-image=0.16.2 27 | - scikit-learn=0.22 28 | - scipy=1.3.1 29 | - tqdm=4.40.0 30 | - wheel 31 | - widgetsnbextension=3.5.1 32 | - pip: 33 | #- atari-py==0.2.6 # NOT ON WINDOWS YET 34 | - ftfy==5.7 35 | - gym==0.15.4 36 | - opencv-python==4.1.2.30 37 | - psutil==5.6.7 38 | - pyglet==1.3.2 39 | - spacy==2.2.4 40 | - tensorboard==2.1.1 41 | #- tensorflow-addons==0.8.3 # NOT ON WINDOWS YET 42 | #- tensorflow-data-validation==0.21.5 # NOT ON WINDOWS YET 43 | - tensorflow-datasets==2.1.0 44 | - tensorflow-estimator==2.1.0 45 | - tensorflow-hub==0.7.0 46 | #- tensorflow-metadata==0.21.1 # NOT ON WINDOWS YET 47 | #- tensorflow-model-analysis==0.21.6 # NOT ON WINDOWS YET 48 | - tensorflow-probability==0.9.0 49 | - tensorflow-serving-api==2.1.0 # or tensorflow-serving-api-gpu if gpu 50 | #- tensorflow-transform==0.21.2 # NOT ON WINDOWS YET 51 | - tensorflow==2.1.0 # or tensorflow-gpu if gpu 52 | - tf-agents==0.3.0 53 | #- tfx==0.21.2 # NOT ON WINDOWS YET 54 | - transformers==2.8.0 55 | - urlextract==0.13.0 56 | #- pyvirtualdisplay # add if on headless server 57 | -------------------------------------------------------------------------------- /datasets/jsb_chorales/README.md: -------------------------------------------------------------------------------- 1 | # Johann Sebastian Bach Chorales Dataset 2 | 3 | ## Source 4 | This dataset contains 382 chorales by Johann Sebastian Bach (in the public domain), where each chorale is composed of 100 to 640 chords with a temporal resolution of 1/16th. Each chord is composed of 4 integers, each indicating the index of a note on a piano, except for the value 0 which means "no note played". 5 | 6 | This dataset is based on [czhuang's JSB-Chorales-dataset](https://github.com/czhuang/JSB-Chorales-dataset/blob/master/README.md) (`Jsb16thSeparated.npz`) which used the train, validation, test split from Boulanger-Lewandowski (2012). 7 | 8 | Motivation: I thought it would be nice to have a version of this dataset in CSV format. 9 | 10 | ## Reference 11 | Boulanger-Lewandowski, N., Vincent, P., & Bengio, Y. (2012). Modeling Temporal Dependencies in High-Dimensional Sequences: Application to Polyphonic Music Generation and Transcription. Proceedings of the 29th International Conference on Machine Learning (ICML-12), 1159–1166. 12 | 13 | ## Usage 14 | Download `jsb_chorales.tgz` and untar it: 15 | 16 | ```bash 17 | $ tar xvzf jsb_chorales.tgz 18 | ``` 19 | 20 | ## Data structure 21 | The dataset is split in three (train, valid, test), with a total of 382 CSV files: 22 | 23 | ``` 24 | $ tree 25 | . 26 | ├── train 27 | │   ├── chorale_000.csv 28 | │   ├── chorale_001.csv 29 | │   ├── chorale_002.csv 30 | │ │ ... 31 | │   ├── chorale_227.csv 32 | │   └── chorale_228.csv 33 | ├── valid 34 | │ ├── chorale_229.csv 35 | │ ├── chorale_230.csv 36 | │ ├── chorale_231.csv 37 | │ │ ... 38 | │   ├── chorale_303.csv 39 | │   └── chorale_304.csv 40 | └── test 41 |    ├── chorale_305.csv 42 |    ├── chorale_306.csv 43 |    ├── chorale_307.csv 44 | │ ... 45 |    ├── chorale_380.csv 46 |    └── chorale_381.csv 47 | ``` 48 | 49 | ## Data sample 50 | 51 | ``` 52 | $ head train/chorale_000.csv 53 | note0,note1,note2,note3 54 | 74,70,65,58 55 | 74,70,65,58 56 | 74,70,65,58 57 | 74,70,65,58 58 | 75,70,58,55 59 | 75,70,58,55 60 | 75,70,60,55 61 | 75,70,60,55 62 | 77,69,62,50 63 | ``` 64 | 65 | Enjoy! 66 | -------------------------------------------------------------------------------- /docker/bin/rm_empty_subdirs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | def remove_empty_directories(initial_dir, 6 | allow_initial_delete=False, ignore_nonexistant_initial=False, 7 | dry_run=False, quiet=False): 8 | 9 | FORBIDDEN_SUBDIRS = set([".git"]) 10 | 11 | if not os.path.isdir(initial_dir) and not ignore_nonexistant_initial: 12 | raise RuntimeError("Initial directory '{}' not found!".format(initial_dir)) 13 | 14 | message = "removed" 15 | if dry_run: 16 | message = "to be " + message 17 | 18 | deleted = set() 19 | 20 | for (directory, subdirs, files) in os.walk(initial_dir, topdown=False): 21 | forbidden = False 22 | parent = directory 23 | while parent: 24 | parent, dirname = os.path.split(parent) 25 | if dirname in FORBIDDEN_SUBDIRS: 26 | forbidden = True 27 | break 28 | if forbidden: 29 | continue 30 | 31 | is_empty = len(files) < 1 and len(set([os.path.join(directory, s) for s in subdirs]) - deleted) < 1 32 | 33 | if is_empty and (initial_dir != directory or allow_initial_delete): 34 | if not quiet: 35 | print("{}: {}".format(message, directory)) 36 | deleted.add(directory) 37 | if not dry_run: 38 | os.rmdir(directory) 39 | 40 | def main(): 41 | import argparse 42 | parser = argparse.ArgumentParser(description="Remove empty directories recursively in subtree.") 43 | parser.add_argument("dir", metavar="DIR", type=str, nargs="+", help="directory to be searched") 44 | parser.add_argument("-r", "--allow-dir-removal", action="store_true", help="allow deletion of DIR itself") 45 | parser.add_argument("-i", "--ignore-nonexistent-dir", action="store_true", help="don't throw an error if DIR doesn't exist") 46 | parser.add_argument("-d", "--dry-run", action="store_true", help="only print messages, don't perform any removals") 47 | parser.add_argument("-q", "--quiet", action="store_true", help="don't print names of directories being removed") 48 | args = parser.parse_args() 49 | for directory in args.dir: 50 | remove_empty_directories(directory, args.allow_dir_removal, args.ignore_nonexistent_dir, 51 | args.dry_run, args.quiet) 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # TensorFlow is much easier to install using Anaconda, especially 2 | # on Windows or when using a GPU. Please see the installation 3 | # instructions in INSTALL.md 4 | 5 | 6 | ##### Core scientific packages 7 | jupyter==1.0.0 8 | matplotlib==3.3.2 9 | numpy==1.18.5 10 | pandas==1.1.3 11 | scipy==1.5.3 12 | 13 | ##### Machine Learning packages 14 | scikit-learn==0.23.2 15 | 16 | # Optional: the XGBoost library is only used in chapter 7 17 | xgboost==1.2.1 18 | 19 | # Optional: the transformers library is only using in chapter 16 20 | transformers==3.3.1 21 | 22 | ##### TensorFlow-related packages 23 | 24 | # If you have a TF-compatible GPU and you want to enable GPU support, then 25 | # replace tensorflow-serving-api with tensorflow-serving-api-gpu. 26 | # Your GPU must have CUDA Compute Capability 3.5 or higher support, and 27 | # you must install CUDA, cuDNN and more: see tensorflow.org for the detailed 28 | # installation instructions. 29 | 30 | tensorflow==2.3.1 31 | # Optional: the TF Serving API library is just needed for chapter 19. 32 | tensorflow-serving-api==2.3.0 # or tensorflow-serving-api-gpu if gpu 33 | 34 | tensorboard==2.3.0 35 | tensorboard-plugin-profile==2.3.0 36 | tensorflow-datasets==4.0.1 37 | tensorflow-hub==0.9.0 38 | tensorflow-probability==0.11.1 39 | 40 | # Optional: only used in chapter 13. 41 | # NOT AVAILABLE ON WINDOWS 42 | tfx==0.24.1 43 | 44 | # Optional: only used in chapter 16. 45 | # NOT AVAILABLE ON WINDOWS 46 | tensorflow-addons==0.11.2 47 | 48 | ##### Reinforcement Learning library (chapter 18) 49 | 50 | # There are a few dependencies you need to install first, check out: 51 | # https://github.com/openai/gym#installing-everything 52 | gym[atari]==0.17.3 53 | # On Windows, install atari_py using: 54 | # pip install --no-index -f https://github.com/Kojoley/atari-py/releases atari_py 55 | 56 | tf-agents==0.6.0 57 | 58 | ##### Image manipulation 59 | Pillow==8.0.0 60 | graphviz==0.14.2 61 | opencv-python==4.4.0.44 62 | pyglet==1.4.11 63 | 64 | #pyvirtualdisplay # needed in chapter 16, if on a headless server 65 | # (i.e., without screen, e.g., Colab or VM) 66 | 67 | 68 | ##### Additional utilities 69 | 70 | # Efficient jobs (caching, parallelism, persistence) 71 | joblib==0.14.1 72 | 73 | # Easy http requests 74 | requests==2.24.0 75 | 76 | # Nice utility to diff Jupyter Notebooks. 77 | nbdime==2.1.0 78 | 79 | # May be useful with Pandas for complex "where" clauses (e.g., Pandas 80 | # tutorial). 81 | numexpr==2.7.1 82 | 83 | # Optional: these libraries can be useful in the classification chapter, 84 | # exercise 4. 85 | nltk==3.5 86 | urlextract==1.1.0 87 | 88 | # Optional: these libraries are only used in chapter 16 89 | ftfy==5.8 90 | 91 | # Optional: tqdm displays nice progress bars, ipywidgets for tqdm's notebook support 92 | tqdm==4.50.2 93 | ipywidgets==7.5.1 94 | 95 | 96 | 97 | # Specific lib versions to avoid conflicts 98 | attrs==19.3.0 99 | cloudpickle==1.3.0 100 | dill==0.3.1.1 101 | gast==0.3.3 102 | httplib2==0.17.4 103 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Hands-on Machine Learning in Docker 3 | 4 | This is the Docker configuration which allows you to run and tweak the book's notebooks without installing any dependencies on your machine!
5 | OK, any except `docker`. With `docker-compose`. Well, you may also want `make` (but it is only used as thin layer to call a few simple `docker-compose` commands). 6 | 7 | ## Prerequisites 8 | 9 | As stated, the two things you need is `docker` and `docker-compose`. 10 | 11 | Follow the instructions on [Install Docker](https://docs.docker.com/engine/installation/) and [Install Docker Compose](https://docs.docker.com/compose/install/) for your environment if you haven't got `docker` already. 12 | 13 | Some general knowledge about `docker` infrastructure might be useful (that's an interesting topic on its own) but is not strictly *required* to just run the notebooks. 14 | 15 | ## Usage 16 | 17 | ### Prepare the image (once) 18 | 19 | Switch to `docker` directory here and run `make build` (or `docker-compose build`) to build your docker image. That may take some time but is only required once. Or perhaps a few times after you tweak something in a `Dockerfile`. 20 | 21 | After the process is finished you have a `handson-ml2` image, that will be the base for your experiments. You can confirm that looking on results of `docker images` command. 22 | 23 | ### Run the notebooks 24 | 25 | Run `make run` (or just `docker-compose up`) to start the jupyter server inside the container (also named `handson-ml2`, same as image). Just point your browser to the URL printed on the screen (or just if you enabled password authentication) and you're ready to play with the book's code! 26 | 27 | The server runs in the directory containing the notebooks, and the changes you make from the browser will be persisted there. 28 | 29 | You can close the server just by pressing `Ctrl-C` in terminal window. 30 | 31 | ### Run additional commands in container 32 | 33 | Run `make exec` (or `docker-compose exec handson-ml2 bash`) while the server is running to run an additional `bash` shell inside the `handson-ml2` container. Now you're inside the environment prepared within the image. 34 | 35 | One of the useful things that can be done there would be starting TensorBoard (for example with simple `tb` command, see bashrc file). 36 | 37 | Another one may be comparing versions of the notebooks using the `nbdiff` command if you haven't got `nbdime` installed locally (it is **way** better than plain `diff` for notebooks). See [Tools for diffing and merging of Jupyter notebooks](https://github.com/jupyter/nbdime) for more details. 38 | 39 | You can see changes you made relative to the version in git using `git diff` which is integrated with `nbdiff`. 40 | 41 | You may also try `nbd NOTEBOOK_NAME.ipynb` command (custom, see bashrc file) to compare one of your notebooks with its `checkpointed` version.
42 | To be precise, the output will tell you *what modifications should be re-played on the **manually saved** version of the notebook (located in `.ipynb_checkpoints` subdirectory) to update it to the **current** i.e. **auto-saved** version (given as command's argument - located in working directory)*. 43 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3:latest 2 | 3 | RUN apt-get update && apt-get upgrade -y \ 4 | && apt-get install -y \ 5 | libpq-dev \ 6 | build-essential \ 7 | git \ 8 | sudo \ 9 | cmake zlib1g-dev libjpeg-dev xvfb ffmpeg xorg-dev libboost-all-dev libsdl2-dev swig \ 10 | unzip zip \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | COPY environment.yml /tmp/ 14 | RUN conda update -y -n base conda \ 15 | && echo ' - pyvirtualdisplay' >> /tmp/environment.yml \ 16 | && conda env create -f /tmp/environment.yml \ 17 | && conda clean -y -t \ 18 | && rm /tmp/environment.yml 19 | 20 | ARG username 21 | ARG userid 22 | 23 | ARG home=/home/${username} 24 | ARG workdir=${home}/handson-ml2 25 | 26 | RUN adduser ${username} --uid ${userid} --gecos '' --disabled-password \ 27 | && echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} \ 28 | && chmod 0440 /etc/sudoers.d/${username} 29 | 30 | WORKDIR ${workdir} 31 | RUN chown ${username}:${username} ${workdir} 32 | 33 | USER ${username} 34 | WORKDIR ${workdir} 35 | 36 | ENV PATH /opt/conda/envs/tf2/bin:$PATH 37 | 38 | # The config below enables diffing notebooks with nbdiff (and nbdiff support 39 | # in git diff command) after connecting to the container by "make exec" (or 40 | # "docker-compose exec handson-ml2 bash") 41 | # You may also try running: 42 | # nbdiff NOTEBOOK_NAME.ipynb 43 | # to get nbdiff between checkpointed version and current version of the 44 | # given notebook. 45 | 46 | RUN git-nbdiffdriver config --enable --global 47 | 48 | # INFO: Optionally uncomment any (one) of the following RUN commands below to ignore either 49 | # metadata or details in nbdiff within git diff 50 | #RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-metadata' 51 | RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-details' 52 | 53 | 54 | COPY docker/bashrc.bash /tmp/ 55 | RUN cat /tmp/bashrc.bash >> ${home}/.bashrc \ 56 | && echo "export PATH=\"${workdir}/docker/bin:$PATH\"" >> ${home}/.bashrc \ 57 | && sudo rm /tmp/bashrc.bash 58 | 59 | 60 | # INFO: Uncomment lines below to enable automatic save of python-only and html-only 61 | # exports alongside the notebook 62 | #COPY docker/jupyter_notebook_config.py /tmp/ 63 | #RUN cat /tmp/jupyter_notebook_config.py >> ${home}/.jupyter/jupyter_notebook_config.py 64 | #RUN sudo rm /tmp/jupyter_notebook_config.py 65 | 66 | 67 | # INFO: Uncomment the RUN command below to disable git diff paging 68 | #RUN git config --global core.pager '' 69 | 70 | 71 | # INFO: Uncomment the RUN command below for easy and constant notebook URL (just localhost:8888) 72 | # That will switch Jupyter to using empty password instead of a token. 73 | # To avoid making a security hole you SHOULD in fact not only uncomment but 74 | # regenerate the hash for your own non-empty password and replace the hash below. 75 | # You can compute a password hash in any notebook, just run the code: 76 | # from notebook.auth import passwd 77 | # passwd() 78 | # and take the hash from the output 79 | #RUN mkdir -p ${home}/.jupyter && \ 80 | # echo 'c.NotebookApp.password = u"sha1:c6bbcba2d04b:f969e403db876dcfbe26f47affe41909bd53392e"' \ 81 | # >> ${home}/.jupyter/jupyter_notebook_config.py 82 | -------------------------------------------------------------------------------- /datasets/housing/README.md: -------------------------------------------------------------------------------- 1 | # California Housing 2 | 3 | ## Source 4 | This dataset is a modified version of the California Housing dataset available from [Luís Torgo's page](http://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html) (University of Porto). Luís Torgo obtained it from the StatLib repository (which is closed now). The dataset may also be downloaded from StatLib mirrors. 5 | 6 | This dataset appeared in a 1997 paper titled *Sparse Spatial Autoregressions* by Pace, R. Kelley and Ronald Barry, published in the *Statistics and Probability Letters* journal. They built it using the 1990 California census data. It contains one row per census block group. A block group is the smallest geographical unit for which the U.S. Census Bureau publishes sample data (a block group typically has a population of 600 to 3,000 people). 7 | 8 | ## Tweaks 9 | The dataset in this directory is almost identical to the original, with two differences: 10 | 11 | * 207 values were randomly removed from the `total_bedrooms` column, so we can discuss what to do with missing data. 12 | * An additional categorical attribute called `ocean_proximity` was added, indicating (very roughly) whether each block group is near the ocean, near the Bay area, inland or on an island. This allows discussing what to do with categorical data. 13 | 14 | Note that the block groups are called "districts" in the Jupyter notebooks, simply because in some contexts the name "block group" was confusing. 15 | 16 | ## Data description 17 | 18 | >>> housing.info() 19 | 20 | RangeIndex: 20640 entries, 0 to 20639 21 | Data columns (total 10 columns): 22 | longitude 20640 non-null float64 23 | latitude 20640 non-null float64 24 | housing_median_age 20640 non-null float64 25 | total_rooms 20640 non-null float64 26 | total_bedrooms 20433 non-null float64 27 | population 20640 non-null float64 28 | households 20640 non-null float64 29 | median_income 20640 non-null float64 30 | median_house_value 20640 non-null float64 31 | ocean_proximity 20640 non-null object 32 | dtypes: float64(9), object(1) 33 | memory usage: 1.6+ MB 34 | 35 | >>> housing["ocean_proximity"].value_counts() 36 | <1H OCEAN 9136 37 | INLAND 6551 38 | NEAR OCEAN 2658 39 | NEAR BAY 2290 40 | ISLAND 5 41 | Name: ocean_proximity, dtype: int64 42 | 43 | >>> housing.describe() 44 | longitude latitude housing_median_age total_rooms \ 45 | count 16513.000000 16513.000000 16513.000000 16513.000000 46 | mean -119.575972 35.639693 28.652335 2622.347605 47 | std 2.002048 2.138279 12.576306 2138.559393 48 | min -124.350000 32.540000 1.000000 6.000000 49 | 25% -121.800000 33.940000 18.000000 1442.000000 50 | 50% -118.510000 34.260000 29.000000 2119.000000 51 | 75% -118.010000 37.720000 37.000000 3141.000000 52 | max -114.310000 41.950000 52.000000 39320.000000 53 | 54 | total_bedrooms population households median_income 55 | count 16355.000000 16513.000000 16513.000000 16513.000000 56 | mean 534.885112 1419.525465 496.975050 3.875651 57 | std 412.716467 1115.715084 375.737945 1.905088 58 | min 2.000000 3.000000 2.000000 0.499900 59 | 25% 295.000000 784.000000 278.000000 2.566800 60 | 50% 433.000000 1164.000000 408.000000 3.541400 61 | 75% 644.000000 1718.000000 602.000000 4.745000 62 | max 6210.000000 35682.000000 5358.000000 15.000100 63 | 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Machine Learning Notebooks 2 | ========================== 3 | 4 | This project aims at teaching you the fundamentals of Machine Learning in 5 | python. It contains the example code and solutions to the exercises in the second edition of my O'Reilly book [Hands-on Machine Learning with Scikit-Learn, Keras and TensorFlow](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/): 6 | 7 | 8 | 9 | **Note**: If you are looking for the first edition notebooks, check out [ageron/handson-ml](https://github.com/ageron/handson-ml). 10 | 11 | ## Quick Start 12 | 13 | ### Want to play with these notebooks online without having to install anything? 14 | Use any of the following services. 15 | 16 | **WARNING**: Please be aware that these services provide temporary environments: anything you do will be deleted after a while, so make sure you download any data you care about. 17 | 18 | * **Recommended**: open this repository in [Colaboratory](https://colab.research.google.com/github/ageron/handson-ml2/blob/master/): 19 | 20 | 21 | * Or open it in [Binder](https://mybinder.org/v2/gh/ageron/handson-ml2/master): 22 | 23 | 24 | * _Note_: Most of the time, Binder starts up quickly and works great, but when handson-ml2 is updated, Binder creates a new environment from scratch, and this can take quite some time. 25 | 26 | * Or open it in [Deepnote](https://beta.deepnote.com/launch?template=data-science&url=https%3A//github.com/ageron/handson-ml2/blob/master/index.ipynb): 27 | 28 | 29 | ### Just want to quickly look at some notebooks, without executing any code? 30 | 31 | Browse this repository using [jupyter.org's notebook viewer](https://nbviewer.jupyter.org/github/ageron/handson-ml2/blob/master/index.ipynb): 32 | 33 | 34 | _Note_: [github.com's notebook viewer](index.ipynb) also works but it is slower and the math equations are not always displayed correctly. 35 | 36 | ### Want to run this project using a Docker image? 37 | Read the [Docker instructions](https://github.com/ageron/handson-ml2/tree/master/docker). 38 | 39 | ### Want to install this project on your own machine? 40 | 41 | Start by installing [Anaconda](https://www.anaconda.com/distribution/) (or [Miniconda](https://docs.conda.io/en/latest/miniconda.html)), [git](https://git-scm.com/downloads), and if you have a TensorFlow-compatible GPU, install the [GPU driver](https://www.nvidia.com/Download/index.aspx). 42 | 43 | Next, clone this project by opening a terminal and typing the following commands (do not type the first `$` signs on each line, they just indicate that these are terminal commands): 44 | 45 | $ git clone https://github.com/ageron/handson-ml2.git 46 | $ cd handson-ml2 47 | 48 | If you want to use a GPU, then edit `environment.yml` (or `environment-windows.yml` on Windows) and replace `tensorflow=2.0.0` with `tensorflow-gpu=2.0.0`. Also replace `tensorflow-serving-api==2.0.0` with `tensorflow-serving-api-gpu==2.0.0`. 49 | 50 | Next, run the following commands: 51 | 52 | $ conda env create -f environment.yml # or environment-windows.yml on Windows 53 | $ conda activate tf2 54 | $ python -m ipykernel install --user --name=python3 55 | 56 | Then if you're on Windows, run the following command: 57 | 58 | $ pip install --no-index -f https://github.com/Kojoley/atari-py/releases atari_py 59 | 60 | Finally, start Jupyter: 61 | 62 | $ jupyter notebook 63 | 64 | If you need further instructions, read the [detailed installation instructions](INSTALL.md). 65 | 66 | ## Contributors 67 | I would like to thank everyone who contributed to this project, either by providing useful feedback, filing issues or submitting Pull Requests. Special thanks go to Haesun Park who helped on some of the exercise solutions, and to Steven Bunkley and Ziembla who created the `docker` directory. Thanks as well to github user SuperYorio for helping out on the coding exercise solutions. -------------------------------------------------------------------------------- /datasets/lifesat/README.md: -------------------------------------------------------------------------------- 1 | # Life satisfaction and GDP per capita 2 | ## Life satisfaction 3 | ### Source 4 | This dataset was obtained from the OECD's website at: http://stats.oecd.org/index.aspx?DataSetCode=BLI 5 | 6 | ### Data description 7 | 8 | Int64Index: 3292 entries, 0 to 3291 9 | Data columns (total 17 columns): 10 | "LOCATION" 3292 non-null object 11 | Country 3292 non-null object 12 | INDICATOR 3292 non-null object 13 | Indicator 3292 non-null object 14 | MEASURE 3292 non-null object 15 | Measure 3292 non-null object 16 | INEQUALITY 3292 non-null object 17 | Inequality 3292 non-null object 18 | Unit Code 3292 non-null object 19 | Unit 3292 non-null object 20 | PowerCode Code 3292 non-null int64 21 | PowerCode 3292 non-null object 22 | Reference Period Code 0 non-null float64 23 | Reference Period 0 non-null float64 24 | Value 3292 non-null float64 25 | Flag Codes 1120 non-null object 26 | Flags 1120 non-null object 27 | dtypes: float64(3), int64(1), object(13) 28 | memory usage: 462.9+ KB 29 | 30 | ### Example usage using python Pandas 31 | 32 | >>> life_sat = pd.read_csv("oecd_bli_2015.csv", thousands=',') 33 | 34 | >>> life_sat_total = life_sat[life_sat["INEQUALITY"]=="TOT"] 35 | 36 | >>> life_sat_total = life_sat_total.pivot(index="Country", columns="Indicator", values="Value") 37 | 38 | >>> life_sat_total.info() 39 | 40 | Index: 37 entries, Australia to United States 41 | Data columns (total 24 columns): 42 | Air pollution 37 non-null float64 43 | Assault rate 37 non-null float64 44 | Consultation on rule-making 37 non-null float64 45 | Dwellings without basic facilities 37 non-null float64 46 | Educational attainment 37 non-null float64 47 | Employees working very long hours 37 non-null float64 48 | Employment rate 37 non-null float64 49 | Homicide rate 37 non-null float64 50 | Household net adjusted disposable income 37 non-null float64 51 | Household net financial wealth 37 non-null float64 52 | Housing expenditure 37 non-null float64 53 | Job security 37 non-null float64 54 | Life expectancy 37 non-null float64 55 | Life satisfaction 37 non-null float64 56 | Long-term unemployment rate 37 non-null float64 57 | Personal earnings 37 non-null float64 58 | Quality of support network 37 non-null float64 59 | Rooms per person 37 non-null float64 60 | Self-reported health 37 non-null float64 61 | Student skills 37 non-null float64 62 | Time devoted to leisure and personal care 37 non-null float64 63 | Voter turnout 37 non-null float64 64 | Water quality 37 non-null float64 65 | Years in education 37 non-null float64 66 | dtypes: float64(24) 67 | memory usage: 7.2+ KB 68 | 69 | ## GDP per capita 70 | ### Source 71 | Dataset obtained from the IMF's website at: http://goo.gl/j1MSKe 72 | 73 | ### Data description 74 | 75 | Int64Index: 190 entries, 0 to 189 76 | Data columns (total 7 columns): 77 | Country 190 non-null object 78 | Subject Descriptor 189 non-null object 79 | Units 189 non-null object 80 | Scale 189 non-null object 81 | Country/Series-specific Notes 188 non-null object 82 | 2015 187 non-null float64 83 | Estimates Start After 188 non-null float64 84 | dtypes: float64(2), object(5) 85 | memory usage: 11.9+ KB 86 | 87 | ### Example usage using python Pandas 88 | 89 | >>> gdp_per_capita = pd.read_csv( 90 | ... datapath+"gdp_per_capita.csv", thousands=',', delimiter='\t', 91 | ... encoding='latin1', na_values="n/a", index_col="Country") 92 | ... 93 | >>> gdp_per_capita.rename(columns={"2015": "GDP per capita"}, inplace=True) 94 | 95 | -------------------------------------------------------------------------------- /index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Machine Learning Notebooks\n", 8 | "\n", 9 | "*Welcome to the Machine Learning Notebooks!*\n", 10 | "\n", 11 | "[Prerequisites](#Prerequisites) (see below)\n", 12 | "\n", 13 | "## Notebooks\n", 14 | "1. [The Machine Learning landscape](01_the_machine_learning_landscape.ipynb)\n", 15 | "2. [End-to-end Machine Learning project](02_end_to_end_machine_learning_project.ipynb)\n", 16 | "3. [Classification](03_classification.ipynb)\n", 17 | "4. [Training Models](04_training_linear_models.ipynb)\n", 18 | "5. [Support Vector Machines](05_support_vector_machines.ipynb)\n", 19 | "6. [Decision Trees](06_decision_trees.ipynb)\n", 20 | "7. [Ensemble Learning and Random Forests](07_ensemble_learning_and_random_forests.ipynb)\n", 21 | "8. [Dimensionality Reduction](08_dimensionality_reduction.ipynb)\n", 22 | "9. [Unsupervised Learning Techniques](09_unsupervised_learning.ipynb)\n", 23 | "10. [Artificial Neural Nets with Keras](10_neural_nets_with_keras.ipynb)\n", 24 | "11. [Training Deep Neural Networks](11_training_deep_neural_networks.ipynb)\n", 25 | "12. [Custom Models and Training with TensorFlow](12_custom_models_and_training_with_tensorflow.ipynb)\n", 26 | "13. [Loading and Preprocessing Data](13_loading_and_preprocessing_data.ipynb)\n", 27 | "14. [Deep Computer Vision Using Convolutional Neural Networks](14_deep_computer_vision_with_cnns.ipynb)\n", 28 | "15. [Processing Sequences Using RNNs and CNNs](15_processing_sequences_using_rnns_and_cnns.ipynb)\n", 29 | "16. [Natural Language Processing with RNNs and Attention](16_nlp_with_rnns_and_attention.ipynb)\n", 30 | "17. [Representation Learning Using Autoencoders](17_autoencoders.ipynb)\n", 31 | "18. [Reinforcement Learning](18_reinforcement_learning.ipynb)\n", 32 | "19. [Training and Deploying TensorFlow Models at Scale](19_training_and_deploying_at_scale.ipynb)\n", 33 | "\n", 34 | "## Scientific Python tutorials\n", 35 | "* [NumPy](tools_numpy.ipynb)\n", 36 | "* [Matplotlib](tools_matplotlib.ipynb)\n", 37 | "* [Pandas](tools_pandas.ipynb)\n", 38 | "\n", 39 | "## Math Tutorials\n", 40 | "* [Linear Algebra](math_linear_algebra.ipynb)\n", 41 | "* [Differential Calculus](math_differential_calculus.ipynb)\n", 42 | "\n", 43 | "## Extra Material\n", 44 | "* [Auto-differentiation](extra_autodiff.ipynb)\n", 45 | "\n", 46 | "## Misc.\n", 47 | "* [Equations](book_equations.pdf) (list of equations in the book)\n" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "## Prerequisites\n", 55 | "### To understand\n", 56 | "* **Python** – you don't need to be an expert python programmer, but you do need to know the basics. If you don't, the official [Python tutorial](https://docs.python.org/3/tutorial/) is a good place to start.\n", 57 | "* **Scientific Python** – We will be using a few popular python libraries, in particular NumPy, matplotlib and pandas. If you are not familiar with these libraries, you should probably start by going through the tutorials in the Tools section (especially NumPy).\n", 58 | "* **Math** – We will also use some notions of Linear Algebra, Calculus, Statistics and Probability theory. You should be able to follow along if you learned these in the past as it won't be very advanced, but if you don't know about these topics or you need a refresher then go through the appropriate introduction in the Math section.\n", 59 | "\n", 60 | "### To run the examples\n", 61 | "* **Jupyter** – These notebooks are based on Jupyter. You can run these notebooks in just one click using a hosted platform such as Binder, Deepnote or Colaboratory (no installation required), or you can just view them using Jupyter.org's viewer, or you can install everything on your machine, as you prefer. Check out the [home page](https://github.com/ageron/handson-ml2/) for more details." 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [] 70 | } 71 | ], 72 | "metadata": { 73 | "kernelspec": { 74 | "display_name": "Python 3", 75 | "language": "python", 76 | "name": "python3" 77 | }, 78 | "language_info": { 79 | "codemirror_mode": { 80 | "name": "ipython", 81 | "version": 3 82 | }, 83 | "file_extension": ".py", 84 | "mimetype": "text/x-python", 85 | "name": "python", 86 | "nbconvert_exporter": "python", 87 | "pygments_lexer": "ipython3", 88 | "version": "3.7.6" 89 | }, 90 | "nav_menu": {}, 91 | "toc": { 92 | "navigate_menu": true, 93 | "number_sections": true, 94 | "sideBar": true, 95 | "threshold": 6, 96 | "toc_cell": false, 97 | "toc_section_display": "block", 98 | "toc_window_display": false 99 | } 100 | }, 101 | "nbformat": 4, 102 | "nbformat_minor": 4 103 | } 104 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Download this repository 4 | To install this repository and run the Jupyter notebooks on your machine, you will first need git, which you may already have. Open a terminal and type `git` to check. If you do not have git, you can download it from [git-scm.com](https://git-scm.com/). 5 | 6 | Next, clone this repository by opening a terminal and typing the following commands: 7 | 8 | $ cd $HOME # or any other development directory you prefer 9 | $ git clone https://github.com/ageron/handson-ml2.git 10 | $ cd handson-ml2 11 | 12 | If you do not want to install git, you can instead download [master.zip](https://github.com/ageron/handson-ml2/archive/master.zip), unzip it, rename the resulting directory to `handson-ml2` and move it to your development directory. 13 | 14 | ## Install Anaconda 15 | Next, you will need Python 3.6 or 3.7 and a bunch of Python libraries. The simplest way to install these is to use Anaconda, which is a great cross-platform Python distribution for scientific computing. It comes bundled with many scientific libraries, including NumPy, Pandas, Matplotlib, Scikit-Learn and much more, so it's quite a large installation. If you choose to [download and install Anaconda](https://www.anaconda.com/distribution/), just make sure to install the Python 3 version. If you prefer a lighter weight Anaconda distribution, you can [install Miniconda](https://docs.conda.io/en/latest/miniconda.html), which contains the bare minimum to run the `conda` packaging tool. 16 | 17 | Once Anaconda or miniconda is installed, then run the following command to update the conda packaging tool to the latest version: 18 | 19 | $ conda update -n base -c defaults conda 20 | 21 | > **Note**: if you don't like Anaconda for some reason, then you can install Python 3 and use pip to install the required libraries manually (this is not recommended, unless you know what you are doing). 22 | 23 | 24 | ## Install the GPU Driver and Libraries 25 | If you have a TensorFlow-compatible GPU card (NVidia card with Compute Capability ≥ 3.5), and you want TensorFlow to use it, then you should download the latest driver for your card from [nvidia.com](https://www.nvidia.com/Download/index.aspx?lang=en-us) and install it. You will also need NVidia's CUDA and cuDNN libraries, but the good news is that they will be installed automatically when you install the tensorflow-gpu package from Anaconda. However, if you don't use Anaconda, you will have to install them manually. If you hit any roadblock, see TensorFlow's [GPU installation instructions](https://tensorflow.org/install/gpu) for more details. 26 | 27 | If you want to use a GPU then you should also edit environment.yml (or environment-windows.yml if you're on Windows), located at the root of the handson-ml2 project, replace tensorflow=2.0.0 with tensorflow-gpu=2.0.0, and replace tensorflow-serving-api==2.0.0 with tensorflow-serving-api-gpu==2.0.0. This will not be needed anymore when TensorFlow 2.1 is released. 28 | 29 | ## Create the tf2 Environment 30 | Next, make sure you're in the handson-ml2 directory and run the following command. It will create a new `conda` environment containing every library you will need to run all the notebooks (by default, the environment will be named `tf2`, but you can choose another name using the `-n` option): 31 | 32 | $ conda env create -f environment.yml # or environment-windows.yml on Windows 33 | 34 | Next, activate the new environment: 35 | 36 | $ conda activate tf2 37 | 38 | ## Windows 39 | If you're on Windows, and you want to go through chapter 18 on Reinforcement Learning, then you will also need to run the following command. It installs a Windows-compatible fork of the atari-py library. 40 | 41 | $ pip install --no-index -f https://github.com/Kojoley/atari-py/releases atari_py 42 | 43 | 44 | > **Warning**: TensorFlow Transform (used in chapter 13) and TensorFlow-AddOns (used in chapter 16) are not yet available on Windows, but the TensorFlow team is working on it. 45 | 46 | 47 | ## Start Jupyter 48 | You're almost there! You just need to register the `tf2` conda environment to Jupyter. The notebooks in this project will default to the environment named `python3`, so it's best to register this environment using the name `python3` (if you prefer to use another name, you will have to select it in the "Kernel > Change kernel..." menu in Jupyter every time you open a notebook): 49 | 50 | $ python3 -m ipykernel install --user --name=python3 51 | 52 | And that's it! You can now start Jupyter like this: 53 | 54 | $ jupyter notebook 55 | 56 | This should open up your browser, and you should see Jupyter's tree view, with the contents of the current directory. If your browser does not open automatically, visit [localhost:8888](http://localhost:8888/tree). Click on `index.ipynb` to get started. 57 | 58 | Congrats! You are ready to learn Machine Learning, hands on! 59 | 60 | When you're done with Jupyter, you can close it by typing Ctrl-C in the Terminal window where you started it. Every time you want to work on this project, you will need to open a Terminal, and run: 61 | 62 | $ cd $HOME # or whatever development directory you chose earlier 63 | $ cd handson-ml2 64 | $ conda activate tf2 65 | $ jupyter notebook 66 | -------------------------------------------------------------------------------- /docker/bin/nbclean_checkpoints: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import collections 4 | import glob 5 | import hashlib 6 | import os 7 | import subprocess 8 | 9 | 10 | class NotebookAnalyser: 11 | 12 | def __init__(self, dry_run=False, verbose=False, colorful=False): 13 | self._dry_run = dry_run 14 | self._verbose = verbose 15 | self._colors = collections.defaultdict(lambda: "") 16 | if colorful: 17 | for color in [ 18 | NotebookAnalyser.COLOR_WHITE, 19 | NotebookAnalyser.COLOR_RED, 20 | NotebookAnalyser.COLOR_GREEN, 21 | NotebookAnalyser.COLOR_YELLOW, 22 | ]: 23 | self._colors[color] = "\033[{}m".format(color) 24 | 25 | NOTEBOOK_SUFFIX = ".ipynb" 26 | CHECKPOINT_DIR = NOTEBOOK_SUFFIX + "_checkpoints" 27 | CHECKPOINT_MASK = "*-checkpoint" + NOTEBOOK_SUFFIX 28 | CHECKPOINT_MASK_LEN = len(CHECKPOINT_MASK) - 1 29 | 30 | @staticmethod 31 | def get_hash(file_path): 32 | with open(file_path, "rb") as input: 33 | hash = hashlib.md5() 34 | for chunk in iter(lambda: input.read(4096), b""): 35 | hash.update(chunk) 36 | return hash.hexdigest() 37 | 38 | MESSAGE_ORPHANED = "missing " 39 | MESSAGE_MODIFIED = "modified" 40 | MESSAGE_DELETED = "DELETING" 41 | 42 | COLOR_WHITE = "0" 43 | COLOR_RED = "31" 44 | COLOR_GREEN = "32" 45 | COLOR_YELLOW = "33" 46 | 47 | def log(self, message, file, color=COLOR_WHITE): 48 | color_on = self._colors[color] 49 | color_off = self._colors[NotebookAnalyser.COLOR_WHITE] 50 | print("{}{}{}: {}".format(color_on, message, color_off, file)) 51 | 52 | def clean_checkpoints(self, directory): 53 | for checkpoint_path in sorted(glob.glob(os.path.join(directory, NotebookAnalyser.CHECKPOINT_MASK))): 54 | 55 | workfile_dir = os.path.dirname(os.path.dirname(checkpoint_path)) 56 | workfile_name = os.path.basename(checkpoint_path)[:-NotebookAnalyser.CHECKPOINT_MASK_LEN] + NotebookAnalyser.NOTEBOOK_SUFFIX 57 | workfile_path = os.path.join(workfile_dir, workfile_name) 58 | 59 | status = "" 60 | if not os.path.isfile(workfile_path): 61 | if self._verbose: 62 | self.log(NotebookAnalyser.MESSAGE_ORPHANED, workfile_path, NotebookAnalyser.COLOR_RED) 63 | else: 64 | checkpoint_stat = os.stat(checkpoint_path) 65 | workfile_stat = os.stat(workfile_path) 66 | 67 | modified = workfile_stat.st_size != checkpoint_stat.st_size 68 | 69 | if not modified: 70 | checkpoint_hash = NotebookAnalyser.get_hash(checkpoint_path) 71 | workfile_hash = NotebookAnalyser.get_hash(workfile_path) 72 | modified = checkpoint_hash != workfile_hash 73 | 74 | if modified: 75 | if self._verbose: 76 | self.log(NotebookAnalyser.MESSAGE_MODIFIED, workfile_path, NotebookAnalyser.COLOR_YELLOW) 77 | else: 78 | self.log(NotebookAnalyser.MESSAGE_DELETED, checkpoint_path, NotebookAnalyser.COLOR_GREEN) 79 | if not self._dry_run: 80 | os.remove(checkpoint_path) 81 | 82 | if not self._dry_run and not os.listdir(directory): 83 | self.log(NotebookAnalyser.MESSAGE_DELETED, directory, NotebookAnalyser.COLOR_GREEN) 84 | os.rmdir(directory) 85 | 86 | def clean_checkpoints_recursively(self, directory): 87 | for (root, subdirs, files) in os.walk(directory): 88 | subdirs.sort() # INFO: traverse alphabetically 89 | if NotebookAnalyser.CHECKPOINT_DIR in subdirs: 90 | subdirs.remove(NotebookAnalyser.CHECKPOINT_DIR) # INFO: don't recurse there 91 | self.clean_checkpoints(os.path.join(root, NotebookAnalyser.CHECKPOINT_DIR)) 92 | 93 | 94 | def main(): 95 | import argparse 96 | parser = argparse.ArgumentParser(description="Remove checkpointed versions of those jupyter notebooks that are identical to their working copies.", 97 | epilog="""Notebooks will be reported as either 98 | "DELETED" if the working copy and checkpointed version are identical 99 | (checkpoint will be deleted), 100 | "missing" if there is a checkpoint but no corresponding working file can be found 101 | or "modified" if notebook and the checkpoint are not byte-to-byte identical. 102 | If removal of checkpoints results in empty ".ipynb_checkpoints" directory 103 | that directory is also deleted. 104 | """) #, formatter_class=argparse.RawDescriptionHelpFormatter) 105 | parser.add_argument("dirs", metavar="DIR", type=str, nargs="*", default=".", help="directories to search") 106 | parser.add_argument("-d", "--dry-run", action="store_true", help="only print messages, don't perform any removals") 107 | parser.add_argument("-v", "--verbose", action="store_true", help="verbose mode") 108 | parser.add_argument("-c", "--color", action="store_true", help="colorful mode") 109 | args = parser.parse_args() 110 | 111 | analyser = NotebookAnalyser(args.dry_run, args.verbose, args.color) 112 | for directory in args.dirs: 113 | analyser.clean_checkpoints_recursively(directory) 114 | 115 | if __name__ == "__main__": 116 | main() 117 | -------------------------------------------------------------------------------- /changes_in_2nd_edition.md: -------------------------------------------------------------------------------- 1 | # Changes in the Second Edition 2 | The second edition has six main objectives: 3 | 4 | 1. Cover additional ML topics: additional unsupervised learning techniques (including clustering, anomaly detection, density estimation and mixture models), additional techniques for training deep nets (including self-normalized networks), additional computer vision techniques (including Xception, SENet, object detection with YOLO, and semantic segmentation using R-CNN), handling sequences using CNNs (including WaveNet), natural language processing using RNNs, CNNs and Transformers, generative adversarial networks 5 | 2. Cover additional libraries and APIs: Keras, the Data API, TF-Agents for Reinforcement Learning, training and deploying TF models at scale using the Distribution Strategies API, TF-Serving and Google Cloud AI Platform. Also briefly introduce TF Transform, TFLite, TF Addons/Seq2Seq, TensorFlow.js and more. 6 | 3. Mention some of the latest important results from Deep Learning research. 7 | 4. Migrate all TensorFlow chapters to TensorFlow 2, and use TensorFlow's implementation of the Keras API (called tf.keras) whenever possible, to simplify the code examples. 8 | 5. Update the code examples to use the latest version of Scikit-Learn, NumPy, Pandas, Matplotlib and other libraries. 9 | 6. Clarify some sections and fix some errors, thanks to plenty of great feedback from readers. 10 | 11 | Some chapters were added, others were rewritten and a few were reordered. The following table shows the mapping between the 1st edition chapters and the 2nd edition chapters: 12 | 13 | |1st ed. chapter | 2nd ed. chapter | %Changes | 2nd ed. title 14 | |--|--|--|--| 15 | |1|1|<10%|The Machine Learning Landscape 16 | |2|2|<10%|End-to-End Machine Learning Project 17 | |3|3|<10%|Classification 18 | |4|4|<10%|Training Models 19 | |5|5|<10%|Support Vector Machines 20 | |6|6|<10%|Decision Trees 21 | |7|7|<10%|Ensemble Learning and Random Forests 22 | |8|8|<10%|Dimensionality Reduction 23 | |N/A|9|100% new|Unsupervised Learning Techniques 24 | |10|10|~75%|Introduction to Artificial Neural Networks with Keras 25 | |11|11|~50%|Training Deep Neural Networks 26 | |9|12|100% rewritten|Custom Models and Training with TensorFlow 27 | |Part of 12|13|100% rewritten|Loading and Preprocessing Data with TensorFlow 28 | |13|14|~50%|Deep Computer Vision Using Convolutional Neural Networks 29 | |Part of 14|15|~75%|Processing Sequences Using RNNs and CNNs 30 | |Part of 14|16|~90%|Natural Language Processing with RNNs and Attention 31 | |15|17|~75%|Autoencoders and GANs 32 | |16|18|~75%|Reinforcement Learning 33 | |Part of 12|19|~75% new|Productionizing TensorFlow Models 34 | 35 | More specifically, here are the main changes for the 2nd edition (other than clarifications, corrections and code updates): 36 | 37 | * Chapter 1 – The Machine Learning Landscape 38 | * Added more examples of ML applications and the corresponding algorithms. 39 | * Added a section on handling mismatch between the training set and the validation & test sets. 40 | * Chapter 2 – End-to-End Machine Learning Project 41 | * Added how to compute a confidence interval. 42 | * Improved the installation instructions (e.g., for Windows). 43 | * Introduced the upgraded `OneHotEncoder` and the new `ColumnTransformer`. 44 | * Added more details on deployment, monitoring and maintenance. 45 | * Chapter 4 – Training Models 46 | * Explained the need for training instances to be Independent and Identically Distributed (IID). 47 | * Chapter 7 – Ensemble Learning and Random Forests 48 | * Added a short section about XGBoost. 49 | * Chapter 9 – Unsupervised Learning Techniques (new chapter) 50 | * Clustering with K-Means, how to choose the number of clusters, how to use it for dimensionality reduction, semi-supervised learning, image segmentation, and more. 51 | * The DBSCAN clustering algorithm and an overview of other clustering algorithms available in Scikit-Learn. 52 | * Gaussian mixture models, the Expectation-Maximization (EM) algorithm, Bayesian variational inference, and how mixture models can be used for clustering, density estimation, anomaly detection and novelty detection. 53 | * Overview of other anomaly detection and novelty detection algorithms. 54 | * Chapter 10 – Introduction to Artificial Neural Networks with Keras (mostly new) 55 | * Added an introduction to the Keras API, including all its APIs (Sequential, Functional and Subclassing), persistence and callbacks (including the `TensorBoard` callback). 56 | * Chapter 11 – Training Deep Neural Networks (many changes) 57 | * Introduced self-normalizing nets, the SELU activation function and Alpha Dropout. 58 | * Introduced self-supervised learning. 59 | * Added Nadam optimization. 60 | * Added Monte-Carlo Dropout. 61 | * Added a note about the risks of adaptive optimization methods. 62 | * Updated the practical guidelines. 63 | * Chapter 12 – Custom Models and Training with TensorFlow (completely rewritten) 64 | * A tour of TensorFlow 2. 65 | * TensorFlow's lower-level Python API. 66 | * Writing custom loss functions, metrics, layers, models. 67 | * Using auto-differentiation and creating custom training algorithms. 68 | * TensorFlow Functions and graphs (including tracing and autograph). 69 | * Chapter 13 – Loading and Preprocessing Data with TensorFlow (new chapter) 70 | * The Data API 71 | * Loading/Storing data efficiently using TFRecords. 72 | * Writing custom preprocessing layers, using Keras preprocessing layers, encoding categorical features and text, using one-hot vectors, bag-of-words, TF-IDF or embeddings. 73 | * An overview of TF Transform and TF Datasets. 74 | * Moved the low-level implementation of the neural network to the exercises. 75 | * Removed details about queues and readers that are now superseded by the Data API. 76 | * Chapter 14 – Deep Computer Vision Using Convolutional Neural Networks 77 | * Added Xception and SENet architectures. 78 | * Added a Keras implementation of ResNet-34. 79 | * Showed how to use pretrained models using Keras. 80 | * Added an end-to-end transfer learning example. 81 | * Added classification and localization. 82 | * Introduced Fully Convolutional Networks (FCNs). 83 | * Introduced object detection using the YOLO architecture. 84 | * Introduced semantic segmentation using R-CNN. 85 | * Chapter 15 – Processing Sequences Using RNNs and CNNs 86 | * Added an introduction to Wavenet. 87 | * Moved the Encoder–Decoder architecture and Bidirectional RNNs to Chapter 16. 88 | * Chapter 16 – Natural Language Processing with RNNs and Attention (new chapter) 89 | * Explained how to use the Data API to handle sequential data. 90 | * Showed an end-to-end example of text generation using a Character RNN, using both a stateless and a stateful RNN. 91 | * Showed an end-to-end example of sentiment analysis using an LSTM. 92 | * Explained masking in Keras. 93 | * Showed how to reuse pretrained embeddings using TF Hub. 94 | * Showed how to build an Encoder–Decoder for Neural Machine Translation using TensorFlow Addons/seq2seq. 95 | * Introduced beam search. 96 | * Explained attention mechanisms. 97 | * Added a short overview of visual attention and a note on explainability. 98 | * Introduced the fully attention-based Transformer architecture, including positional embeddings and multi-head attention. 99 | * Added an overview of recent language models (2018). 100 | * Chapters 17 – Representation Learning and Generative Learning Using Autoencoders and GANs 101 | * Added convolutional autoencoders and recurrent autoencoders. 102 | * Add Generative Adversarial Networks (GANs), including basic GANs, deep convolutional GANs (DCGANs), Progressively Growing GANs (ProGANs) and StyleGANs. 103 | * Chapter 18 – Reinforcement Learning 104 | * Added Double DQN, Dueling DQN and Prioritized Experience Replay. 105 | * Introduced TF Agents. 106 | * Chapter 19 – Training and Deploying TensorFlow Models at Scale (mostly new chapter) 107 | * Serving a TensorFlow model using TF Serving and Google Cloud AI Platform. 108 | * Deploying a Model to a Mobile or Embedded Device using TFLite. 109 | * Using GPUs to Speed Up Computations. 110 | * Training models across multiple devices using the Distribution Strategies API. 111 | 112 | ## Migrating from TF 1 to TF 2 113 | Migrating from TensorFlow 1.x to 2.0 is similar to migrating from Python 2 to 3. The first thing to do is... breathe. Don't rush. TensorFlow 1.x will be around for a while, you have time. 114 | 115 | * You should start by upgrading to the last TensorFlow 1.x version (it will probably be 1.15 by the time you read this). 116 | * Migrate as much of your code as possible to using high level APIs, either tf.keras or the Estimators API. The Estimators API will still work in TF 2.0, but you should prefer using Keras from now on, as the TF team announced that Keras is preferred and it's likely that they will put more effort into improving the Keras API. Also prefer using Keras preprocessing layers (see chapter 13) rather than `tf.feature_columns`. 117 | * If your code uses only high-level APIs, then it will be easy to migrate, as it should work the same way in the latest TF 1.x release and in TF 2.0. 118 | * Get rid of any usage of `tf.contrib` as it won't exist in TF 2.0. Some parts of it were moved to the core API, others were moved to separate projects, and others were not maintained so they were just deleted. If needed, you must install the appropriate libraries or copy some legacy `tf.contrib` code into your own project (as a last resort). 119 | * Write as many tests as you can, it will make the migration easier and safer. 120 | * You can run TF 1.x code in TF 2.0 by starting your program with `import tensorflow.compat.v1 as tf` and `tf.disable_v2_behavior()`. 121 | * Once you are ready to make the jump, you can run the `tf_upgrade_v2` [upgrade script](https://www.tensorflow.org/beta/guide/upgrade). 122 | 123 | For more details on migration, checkout TensorFlow's [Migration Guide](https://www.tensorflow.org/beta/guide/migration_guide). 124 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | 179 | -------------------------------------------------------------------------------- /datasets/inception/imagenet_class_names.txt: -------------------------------------------------------------------------------- 1 | n01440764 tench, Tinca tinca 2 | n01443537 goldfish, Carassius auratus 3 | n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias 4 | n01491361 tiger shark, Galeocerdo cuvieri 5 | n01494475 hammerhead, hammerhead shark 6 | n01496331 electric ray, crampfish, numbfish, torpedo 7 | n01498041 stingray 8 | n01514668 cock 9 | n01514859 hen 10 | n01518878 ostrich, Struthio camelus 11 | n01530575 brambling, Fringilla montifringilla 12 | n01531178 goldfinch, Carduelis carduelis 13 | n01532829 house finch, linnet, Carpodacus mexicanus 14 | n01534433 junco, snowbird 15 | n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea 16 | n01558993 robin, American robin, Turdus migratorius 17 | n01560419 bulbul 18 | n01580077 jay 19 | n01582220 magpie 20 | n01592084 chickadee 21 | n01601694 water ouzel, dipper 22 | n01608432 kite 23 | n01614925 bald eagle, American eagle, Haliaeetus leucocephalus 24 | n01616318 vulture 25 | n01622779 great grey owl, great gray owl, Strix nebulosa 26 | n01629819 European fire salamander, Salamandra salamandra 27 | n01630670 common newt, Triturus vulgaris 28 | n01631663 eft 29 | n01632458 spotted salamander, Ambystoma maculatum 30 | n01632777 axolotl, mud puppy, Ambystoma mexicanum 31 | n01641577 bullfrog, Rana catesbeiana 32 | n01644373 tree frog, tree-frog 33 | n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui 34 | n01664065 loggerhead, loggerhead turtle, Caretta caretta 35 | n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea 36 | n01667114 mud turtle 37 | n01667778 terrapin 38 | n01669191 box turtle, box tortoise 39 | n01675722 banded gecko 40 | n01677366 common iguana, iguana, Iguana iguana 41 | n01682714 American chameleon, anole, Anolis carolinensis 42 | n01685808 whiptail, whiptail lizard 43 | n01687978 agama 44 | n01688243 frilled lizard, Chlamydosaurus kingi 45 | n01689811 alligator lizard 46 | n01692333 Gila monster, Heloderma suspectum 47 | n01693334 green lizard, Lacerta viridis 48 | n01694178 African chameleon, Chamaeleo chamaeleon 49 | n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis 50 | n01697457 African crocodile, Nile crocodile, Crocodylus niloticus 51 | n01698640 American alligator, Alligator mississipiensis 52 | n01704323 triceratops 53 | n01728572 thunder snake, worm snake, Carphophis amoenus 54 | n01728920 ringneck snake, ring-necked snake, ring snake 55 | n01729322 hognose snake, puff adder, sand viper 56 | n01729977 green snake, grass snake 57 | n01734418 king snake, kingsnake 58 | n01735189 garter snake, grass snake 59 | n01737021 water snake 60 | n01739381 vine snake 61 | n01740131 night snake, Hypsiglena torquata 62 | n01742172 boa constrictor, Constrictor constrictor 63 | n01744401 rock python, rock snake, Python sebae 64 | n01748264 Indian cobra, Naja naja 65 | n01749939 green mamba 66 | n01751748 sea snake 67 | n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus 68 | n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus 69 | n01756291 sidewinder, horned rattlesnake, Crotalus cerastes 70 | n01768244 trilobite 71 | n01770081 harvestman, daddy longlegs, Phalangium opilio 72 | n01770393 scorpion 73 | n01773157 black and gold garden spider, Argiope aurantia 74 | n01773549 barn spider, Araneus cavaticus 75 | n01773797 garden spider, Aranea diademata 76 | n01774384 black widow, Latrodectus mactans 77 | n01774750 tarantula 78 | n01775062 wolf spider, hunting spider 79 | n01776313 tick 80 | n01784675 centipede 81 | n01795545 black grouse 82 | n01796340 ptarmigan 83 | n01797886 ruffed grouse, partridge, Bonasa umbellus 84 | n01798484 prairie chicken, prairie grouse, prairie fowl 85 | n01806143 peacock 86 | n01806567 quail 87 | n01807496 partridge 88 | n01817953 African grey, African gray, Psittacus erithacus 89 | n01818515 macaw 90 | n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita 91 | n01820546 lorikeet 92 | n01824575 coucal 93 | n01828970 bee eater 94 | n01829413 hornbill 95 | n01833805 hummingbird 96 | n01843065 jacamar 97 | n01843383 toucan 98 | n01847000 drake 99 | n01855032 red-breasted merganser, Mergus serrator 100 | n01855672 goose 101 | n01860187 black swan, Cygnus atratus 102 | n01871265 tusker 103 | n01872401 echidna, spiny anteater, anteater 104 | n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus 105 | n01877812 wallaby, brush kangaroo 106 | n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus 107 | n01883070 wombat 108 | n01910747 jellyfish 109 | n01914609 sea anemone, anemone 110 | n01917289 brain coral 111 | n01924916 flatworm, platyhelminth 112 | n01930112 nematode, nematode worm, roundworm 113 | n01943899 conch 114 | n01944390 snail 115 | n01945685 slug 116 | n01950731 sea slug, nudibranch 117 | n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore 118 | n01968897 chambered nautilus, pearly nautilus, nautilus 119 | n01978287 Dungeness crab, Cancer magister 120 | n01978455 rock crab, Cancer irroratus 121 | n01980166 fiddler crab 122 | n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica 123 | n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus 124 | n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish 125 | n01985128 crayfish, crawfish, crawdad, crawdaddy 126 | n01986214 hermit crab 127 | n01990800 isopod 128 | n02002556 white stork, Ciconia ciconia 129 | n02002724 black stork, Ciconia nigra 130 | n02006656 spoonbill 131 | n02007558 flamingo 132 | n02009229 little blue heron, Egretta caerulea 133 | n02009912 American egret, great white heron, Egretta albus 134 | n02011460 bittern 135 | n02012849 crane 136 | n02013706 limpkin, Aramus pictus 137 | n02017213 European gallinule, Porphyrio porphyrio 138 | n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana 139 | n02018795 bustard 140 | n02025239 ruddy turnstone, Arenaria interpres 141 | n02027492 red-backed sandpiper, dunlin, Erolia alpina 142 | n02028035 redshank, Tringa totanus 143 | n02033041 dowitcher 144 | n02037110 oystercatcher, oyster catcher 145 | n02051845 pelican 146 | n02056570 king penguin, Aptenodytes patagonica 147 | n02058221 albatross, mollymawk 148 | n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus 149 | n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca 150 | n02074367 dugong, Dugong dugon 151 | n02077923 sea lion 152 | n02085620 Chihuahua 153 | n02085782 Japanese spaniel 154 | n02085936 Maltese dog, Maltese terrier, Maltese 155 | n02086079 Pekinese, Pekingese, Peke 156 | n02086240 Shih-Tzu 157 | n02086646 Blenheim spaniel 158 | n02086910 papillon 159 | n02087046 toy terrier 160 | n02087394 Rhodesian ridgeback 161 | n02088094 Afghan hound, Afghan 162 | n02088238 basset, basset hound 163 | n02088364 beagle 164 | n02088466 bloodhound, sleuthhound 165 | n02088632 bluetick 166 | n02089078 black-and-tan coonhound 167 | n02089867 Walker hound, Walker foxhound 168 | n02089973 English foxhound 169 | n02090379 redbone 170 | n02090622 borzoi, Russian wolfhound 171 | n02090721 Irish wolfhound 172 | n02091032 Italian greyhound 173 | n02091134 whippet 174 | n02091244 Ibizan hound, Ibizan Podenco 175 | n02091467 Norwegian elkhound, elkhound 176 | n02091635 otterhound, otter hound 177 | n02091831 Saluki, gazelle hound 178 | n02092002 Scottish deerhound, deerhound 179 | n02092339 Weimaraner 180 | n02093256 Staffordshire bullterrier, Staffordshire bull terrier 181 | n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier 182 | n02093647 Bedlington terrier 183 | n02093754 Border terrier 184 | n02093859 Kerry blue terrier 185 | n02093991 Irish terrier 186 | n02094114 Norfolk terrier 187 | n02094258 Norwich terrier 188 | n02094433 Yorkshire terrier 189 | n02095314 wire-haired fox terrier 190 | n02095570 Lakeland terrier 191 | n02095889 Sealyham terrier, Sealyham 192 | n02096051 Airedale, Airedale terrier 193 | n02096177 cairn, cairn terrier 194 | n02096294 Australian terrier 195 | n02096437 Dandie Dinmont, Dandie Dinmont terrier 196 | n02096585 Boston bull, Boston terrier 197 | n02097047 miniature schnauzer 198 | n02097130 giant schnauzer 199 | n02097209 standard schnauzer 200 | n02097298 Scotch terrier, Scottish terrier, Scottie 201 | n02097474 Tibetan terrier, chrysanthemum dog 202 | n02097658 silky terrier, Sydney silky 203 | n02098105 soft-coated wheaten terrier 204 | n02098286 West Highland white terrier 205 | n02098413 Lhasa, Lhasa apso 206 | n02099267 flat-coated retriever 207 | n02099429 curly-coated retriever 208 | n02099601 golden retriever 209 | n02099712 Labrador retriever 210 | n02099849 Chesapeake Bay retriever 211 | n02100236 German short-haired pointer 212 | n02100583 vizsla, Hungarian pointer 213 | n02100735 English setter 214 | n02100877 Irish setter, red setter 215 | n02101006 Gordon setter 216 | n02101388 Brittany spaniel 217 | n02101556 clumber, clumber spaniel 218 | n02102040 English springer, English springer spaniel 219 | n02102177 Welsh springer spaniel 220 | n02102318 cocker spaniel, English cocker spaniel, cocker 221 | n02102480 Sussex spaniel 222 | n02102973 Irish water spaniel 223 | n02104029 kuvasz 224 | n02104365 schipperke 225 | n02105056 groenendael 226 | n02105162 malinois 227 | n02105251 briard 228 | n02105412 kelpie 229 | n02105505 komondor 230 | n02105641 Old English sheepdog, bobtail 231 | n02105855 Shetland sheepdog, Shetland sheep dog, Shetland 232 | n02106030 collie 233 | n02106166 Border collie 234 | n02106382 Bouvier des Flandres, Bouviers des Flandres 235 | n02106550 Rottweiler 236 | n02106662 German shepherd, German shepherd dog, German police dog, alsatian 237 | n02107142 Doberman, Doberman pinscher 238 | n02107312 miniature pinscher 239 | n02107574 Greater Swiss Mountain dog 240 | n02107683 Bernese mountain dog 241 | n02107908 Appenzeller 242 | n02108000 EntleBucher 243 | n02108089 boxer 244 | n02108422 bull mastiff 245 | n02108551 Tibetan mastiff 246 | n02108915 French bulldog 247 | n02109047 Great Dane 248 | n02109525 Saint Bernard, St Bernard 249 | n02109961 Eskimo dog, husky 250 | n02110063 malamute, malemute, Alaskan malamute 251 | n02110185 Siberian husky 252 | n02110341 dalmatian, coach dog, carriage dog 253 | n02110627 affenpinscher, monkey pinscher, monkey dog 254 | n02110806 basenji 255 | n02110958 pug, pug-dog 256 | n02111129 Leonberg 257 | n02111277 Newfoundland, Newfoundland dog 258 | n02111500 Great Pyrenees 259 | n02111889 Samoyed, Samoyede 260 | n02112018 Pomeranian 261 | n02112137 chow, chow chow 262 | n02112350 keeshond 263 | n02112706 Brabancon griffon 264 | n02113023 Pembroke, Pembroke Welsh corgi 265 | n02113186 Cardigan, Cardigan Welsh corgi 266 | n02113624 toy poodle 267 | n02113712 miniature poodle 268 | n02113799 standard poodle 269 | n02113978 Mexican hairless 270 | n02114367 timber wolf, grey wolf, gray wolf, Canis lupus 271 | n02114548 white wolf, Arctic wolf, Canis lupus tundrarum 272 | n02114712 red wolf, maned wolf, Canis rufus, Canis niger 273 | n02114855 coyote, prairie wolf, brush wolf, Canis latrans 274 | n02115641 dingo, warrigal, warragal, Canis dingo 275 | n02115913 dhole, Cuon alpinus 276 | n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus 277 | n02117135 hyena, hyaena 278 | n02119022 red fox, Vulpes vulpes 279 | n02119789 kit fox, Vulpes macrotis 280 | n02120079 Arctic fox, white fox, Alopex lagopus 281 | n02120505 grey fox, gray fox, Urocyon cinereoargenteus 282 | n02123045 tabby, tabby cat 283 | n02123159 tiger cat 284 | n02123394 Persian cat 285 | n02123597 Siamese cat, Siamese 286 | n02124075 Egyptian cat 287 | n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor 288 | n02127052 lynx, catamount 289 | n02128385 leopard, Panthera pardus 290 | n02128757 snow leopard, ounce, Panthera uncia 291 | n02128925 jaguar, panther, Panthera onca, Felis onca 292 | n02129165 lion, king of beasts, Panthera leo 293 | n02129604 tiger, Panthera tigris 294 | n02130308 cheetah, chetah, Acinonyx jubatus 295 | n02132136 brown bear, bruin, Ursus arctos 296 | n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus 297 | n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus 298 | n02134418 sloth bear, Melursus ursinus, Ursus ursinus 299 | n02137549 mongoose 300 | n02138441 meerkat, mierkat 301 | n02165105 tiger beetle 302 | n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle 303 | n02167151 ground beetle, carabid beetle 304 | n02168699 long-horned beetle, longicorn, longicorn beetle 305 | n02169497 leaf beetle, chrysomelid 306 | n02172182 dung beetle 307 | n02174001 rhinoceros beetle 308 | n02177972 weevil 309 | n02190166 fly 310 | n02206856 bee 311 | n02219486 ant, emmet, pismire 312 | n02226429 grasshopper, hopper 313 | n02229544 cricket 314 | n02231487 walking stick, walkingstick, stick insect 315 | n02233338 cockroach, roach 316 | n02236044 mantis, mantid 317 | n02256656 cicada, cicala 318 | n02259212 leafhopper 319 | n02264363 lacewing, lacewing fly 320 | n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk 321 | n02268853 damselfly 322 | n02276258 admiral 323 | n02277742 ringlet, ringlet butterfly 324 | n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus 325 | n02280649 cabbage butterfly 326 | n02281406 sulphur butterfly, sulfur butterfly 327 | n02281787 lycaenid, lycaenid butterfly 328 | n02317335 starfish, sea star 329 | n02319095 sea urchin 330 | n02321529 sea cucumber, holothurian 331 | n02325366 wood rabbit, cottontail, cottontail rabbit 332 | n02326432 hare 333 | n02328150 Angora, Angora rabbit 334 | n02342885 hamster 335 | n02346627 porcupine, hedgehog 336 | n02356798 fox squirrel, eastern fox squirrel, Sciurus niger 337 | n02361337 marmot 338 | n02363005 beaver 339 | n02364673 guinea pig, Cavia cobaya 340 | n02389026 sorrel 341 | n02391049 zebra 342 | n02395406 hog, pig, grunter, squealer, Sus scrofa 343 | n02396427 wild boar, boar, Sus scrofa 344 | n02397096 warthog 345 | n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius 346 | n02403003 ox 347 | n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis 348 | n02410509 bison 349 | n02412080 ram, tup 350 | n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis 351 | n02417914 ibex, Capra ibex 352 | n02422106 hartebeest 353 | n02422699 impala, Aepyceros melampus 354 | n02423022 gazelle 355 | n02437312 Arabian camel, dromedary, Camelus dromedarius 356 | n02437616 llama 357 | n02441942 weasel 358 | n02442845 mink 359 | n02443114 polecat, fitch, foulmart, foumart, Mustela putorius 360 | n02443484 black-footed ferret, ferret, Mustela nigripes 361 | n02444819 otter 362 | n02445715 skunk, polecat, wood pussy 363 | n02447366 badger 364 | n02454379 armadillo 365 | n02457408 three-toed sloth, ai, Bradypus tridactylus 366 | n02480495 orangutan, orang, orangutang, Pongo pygmaeus 367 | n02480855 gorilla, Gorilla gorilla 368 | n02481823 chimpanzee, chimp, Pan troglodytes 369 | n02483362 gibbon, Hylobates lar 370 | n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus 371 | n02484975 guenon, guenon monkey 372 | n02486261 patas, hussar monkey, Erythrocebus patas 373 | n02486410 baboon 374 | n02487347 macaque 375 | n02488291 langur 376 | n02488702 colobus, colobus monkey 377 | n02489166 proboscis monkey, Nasalis larvatus 378 | n02490219 marmoset 379 | n02492035 capuchin, ringtail, Cebus capucinus 380 | n02492660 howler monkey, howler 381 | n02493509 titi, titi monkey 382 | n02493793 spider monkey, Ateles geoffroyi 383 | n02494079 squirrel monkey, Saimiri sciureus 384 | n02497673 Madagascar cat, ring-tailed lemur, Lemur catta 385 | n02500267 indri, indris, Indri indri, Indri brevicaudatus 386 | n02504013 Indian elephant, Elephas maximus 387 | n02504458 African elephant, Loxodonta africana 388 | n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens 389 | n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca 390 | n02514041 barracouta, snoek 391 | n02526121 eel 392 | n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch 393 | n02606052 rock beauty, Holocanthus tricolor 394 | n02607072 anemone fish 395 | n02640242 sturgeon 396 | n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus 397 | n02643566 lionfish 398 | n02655020 puffer, pufferfish, blowfish, globefish 399 | n02666196 abacus 400 | n02667093 abaya 401 | n02669723 academic gown, academic robe, judge's robe 402 | n02672831 accordion, piano accordion, squeeze box 403 | n02676566 acoustic guitar 404 | n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier 405 | n02690373 airliner 406 | n02692877 airship, dirigible 407 | n02699494 altar 408 | n02701002 ambulance 409 | n02704792 amphibian, amphibious vehicle 410 | n02708093 analog clock 411 | n02727426 apiary, bee house 412 | n02730930 apron 413 | n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin 414 | n02749479 assault rifle, assault gun 415 | n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack 416 | n02776631 bakery, bakeshop, bakehouse 417 | n02777292 balance beam, beam 418 | n02782093 balloon 419 | n02783161 ballpoint, ballpoint pen, ballpen, Biro 420 | n02786058 Band Aid 421 | n02787622 banjo 422 | n02788148 bannister, banister, balustrade, balusters, handrail 423 | n02790996 barbell 424 | n02791124 barber chair 425 | n02791270 barbershop 426 | n02793495 barn 427 | n02794156 barometer 428 | n02795169 barrel, cask 429 | n02797295 barrow, garden cart, lawn cart, wheelbarrow 430 | n02799071 baseball 431 | n02802426 basketball 432 | n02804414 bassinet 433 | n02804610 bassoon 434 | n02807133 bathing cap, swimming cap 435 | n02808304 bath towel 436 | n02808440 bathtub, bathing tub, bath, tub 437 | n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon 438 | n02814860 beacon, lighthouse, beacon light, pharos 439 | n02815834 beaker 440 | n02817516 bearskin, busby, shako 441 | n02823428 beer bottle 442 | n02823750 beer glass 443 | n02825657 bell cote, bell cot 444 | n02834397 bib 445 | n02835271 bicycle-built-for-two, tandem bicycle, tandem 446 | n02837789 bikini, two-piece 447 | n02840245 binder, ring-binder 448 | n02841315 binoculars, field glasses, opera glasses 449 | n02843684 birdhouse 450 | n02859443 boathouse 451 | n02860847 bobsled, bobsleigh, bob 452 | n02865351 bolo tie, bolo, bola tie, bola 453 | n02869837 bonnet, poke bonnet 454 | n02870880 bookcase 455 | n02871525 bookshop, bookstore, bookstall 456 | n02877765 bottlecap 457 | n02879718 bow 458 | n02883205 bow tie, bow-tie, bowtie 459 | n02892201 brass, memorial tablet, plaque 460 | n02892767 brassiere, bra, bandeau 461 | n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty 462 | n02895154 breastplate, aegis, egis 463 | n02906734 broom 464 | n02909870 bucket, pail 465 | n02910353 buckle 466 | n02916936 bulletproof vest 467 | n02917067 bullet train, bullet 468 | n02927161 butcher shop, meat market 469 | n02930766 cab, hack, taxi, taxicab 470 | n02939185 caldron, cauldron 471 | n02948072 candle, taper, wax light 472 | n02950826 cannon 473 | n02951358 canoe 474 | n02951585 can opener, tin opener 475 | n02963159 cardigan 476 | n02965783 car mirror 477 | n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig 478 | n02966687 carpenter's kit, tool kit 479 | n02971356 carton 480 | n02974003 car wheel 481 | n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM 482 | n02978881 cassette 483 | n02979186 cassette player 484 | n02980441 castle 485 | n02981792 catamaran 486 | n02988304 CD player 487 | n02992211 cello, violoncello 488 | n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone 489 | n02999410 chain 490 | n03000134 chainlink fence 491 | n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour 492 | n03000684 chain saw, chainsaw 493 | n03014705 chest 494 | n03016953 chiffonier, commode 495 | n03017168 chime, bell, gong 496 | n03018349 china cabinet, china closet 497 | n03026506 Christmas stocking 498 | n03028079 church, church building 499 | n03032252 cinema, movie theater, movie theatre, movie house, picture palace 500 | n03041632 cleaver, meat cleaver, chopper 501 | n03042490 cliff dwelling 502 | n03045698 cloak 503 | n03047690 clog, geta, patten, sabot 504 | n03062245 cocktail shaker 505 | n03063599 coffee mug 506 | n03063689 coffeepot 507 | n03065424 coil, spiral, volute, whorl, helix 508 | n03075370 combination lock 509 | n03085013 computer keyboard, keypad 510 | n03089624 confectionery, confectionary, candy store 511 | n03095699 container ship, containership, container vessel 512 | n03100240 convertible 513 | n03109150 corkscrew, bottle screw 514 | n03110669 cornet, horn, trumpet, trump 515 | n03124043 cowboy boot 516 | n03124170 cowboy hat, ten-gallon hat 517 | n03125729 cradle 518 | n03126707 crane 519 | n03127747 crash helmet 520 | n03127925 crate 521 | n03131574 crib, cot 522 | n03133878 Crock Pot 523 | n03134739 croquet ball 524 | n03141823 crutch 525 | n03146219 cuirass 526 | n03160309 dam, dike, dyke 527 | n03179701 desk 528 | n03180011 desktop computer 529 | n03187595 dial telephone, dial phone 530 | n03188531 diaper, nappy, napkin 531 | n03196217 digital clock 532 | n03197337 digital watch 533 | n03201208 dining table, board 534 | n03207743 dishrag, dishcloth 535 | n03207941 dishwasher, dish washer, dishwashing machine 536 | n03208938 disk brake, disc brake 537 | n03216828 dock, dockage, docking facility 538 | n03218198 dogsled, dog sled, dog sleigh 539 | n03220513 dome 540 | n03223299 doormat, welcome mat 541 | n03240683 drilling platform, offshore rig 542 | n03249569 drum, membranophone, tympan 543 | n03250847 drumstick 544 | n03255030 dumbbell 545 | n03259280 Dutch oven 546 | n03271574 electric fan, blower 547 | n03272010 electric guitar 548 | n03272562 electric locomotive 549 | n03290653 entertainment center 550 | n03291819 envelope 551 | n03297495 espresso maker 552 | n03314780 face powder 553 | n03325584 feather boa, boa 554 | n03337140 file, file cabinet, filing cabinet 555 | n03344393 fireboat 556 | n03345487 fire engine, fire truck 557 | n03347037 fire screen, fireguard 558 | n03355925 flagpole, flagstaff 559 | n03372029 flute, transverse flute 560 | n03376595 folding chair 561 | n03379051 football helmet 562 | n03384352 forklift 563 | n03388043 fountain 564 | n03388183 fountain pen 565 | n03388549 four-poster 566 | n03393912 freight car 567 | n03394916 French horn, horn 568 | n03400231 frying pan, frypan, skillet 569 | n03404251 fur coat 570 | n03417042 garbage truck, dustcart 571 | n03424325 gasmask, respirator, gas helmet 572 | n03425413 gas pump, gasoline pump, petrol pump, island dispenser 573 | n03443371 goblet 574 | n03444034 go-kart 575 | n03445777 golf ball 576 | n03445924 golfcart, golf cart 577 | n03447447 gondola 578 | n03447721 gong, tam-tam 579 | n03450230 gown 580 | n03452741 grand piano, grand 581 | n03457902 greenhouse, nursery, glasshouse 582 | n03459775 grille, radiator grille 583 | n03461385 grocery store, grocery, food market, market 584 | n03467068 guillotine 585 | n03476684 hair slide 586 | n03476991 hair spray 587 | n03478589 half track 588 | n03481172 hammer 589 | n03482405 hamper 590 | n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier 591 | n03485407 hand-held computer, hand-held microcomputer 592 | n03485794 handkerchief, hankie, hanky, hankey 593 | n03492542 hard disc, hard disk, fixed disk 594 | n03494278 harmonica, mouth organ, harp, mouth harp 595 | n03495258 harp 596 | n03496892 harvester, reaper 597 | n03498962 hatchet 598 | n03527444 holster 599 | n03529860 home theater, home theatre 600 | n03530642 honeycomb 601 | n03532672 hook, claw 602 | n03534580 hoopskirt, crinoline 603 | n03535780 horizontal bar, high bar 604 | n03538406 horse cart, horse-cart 605 | n03544143 hourglass 606 | n03584254 iPod 607 | n03584829 iron, smoothing iron 608 | n03590841 jack-o'-lantern 609 | n03594734 jean, blue jean, denim 610 | n03594945 jeep, landrover 611 | n03595614 jersey, T-shirt, tee shirt 612 | n03598930 jigsaw puzzle 613 | n03599486 jinrikisha, ricksha, rickshaw 614 | n03602883 joystick 615 | n03617480 kimono 616 | n03623198 knee pad 617 | n03627232 knot 618 | n03630383 lab coat, laboratory coat 619 | n03633091 ladle 620 | n03637318 lampshade, lamp shade 621 | n03642806 laptop, laptop computer 622 | n03649909 lawn mower, mower 623 | n03657121 lens cap, lens cover 624 | n03658185 letter opener, paper knife, paperknife 625 | n03661043 library 626 | n03662601 lifeboat 627 | n03666591 lighter, light, igniter, ignitor 628 | n03670208 limousine, limo 629 | n03673027 liner, ocean liner 630 | n03676483 lipstick, lip rouge 631 | n03680355 Loafer 632 | n03690938 lotion 633 | n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system 634 | n03692522 loupe, jeweler's loupe 635 | n03697007 lumbermill, sawmill 636 | n03706229 magnetic compass 637 | n03709823 mailbag, postbag 638 | n03710193 mailbox, letter box 639 | n03710637 maillot 640 | n03710721 maillot, tank suit 641 | n03717622 manhole cover 642 | n03720891 maraca 643 | n03721384 marimba, xylophone 644 | n03724870 mask 645 | n03729826 matchstick 646 | n03733131 maypole 647 | n03733281 maze, labyrinth 648 | n03733805 measuring cup 649 | n03742115 medicine chest, medicine cabinet 650 | n03743016 megalith, megalithic structure 651 | n03759954 microphone, mike 652 | n03761084 microwave, microwave oven 653 | n03763968 military uniform 654 | n03764736 milk can 655 | n03769881 minibus 656 | n03770439 miniskirt, mini 657 | n03770679 minivan 658 | n03773504 missile 659 | n03775071 mitten 660 | n03775546 mixing bowl 661 | n03776460 mobile home, manufactured home 662 | n03777568 Model T 663 | n03777754 modem 664 | n03781244 monastery 665 | n03782006 monitor 666 | n03785016 moped 667 | n03786901 mortar 668 | n03787032 mortarboard 669 | n03788195 mosque 670 | n03788365 mosquito net 671 | n03791053 motor scooter, scooter 672 | n03792782 mountain bike, all-terrain bike, off-roader 673 | n03792972 mountain tent 674 | n03793489 mouse, computer mouse 675 | n03794056 mousetrap 676 | n03796401 moving van 677 | n03803284 muzzle 678 | n03804744 nail 679 | n03814639 neck brace 680 | n03814906 necklace 681 | n03825788 nipple 682 | n03832673 notebook, notebook computer 683 | n03837869 obelisk 684 | n03838899 oboe, hautboy, hautbois 685 | n03840681 ocarina, sweet potato 686 | n03841143 odometer, hodometer, mileometer, milometer 687 | n03843555 oil filter 688 | n03854065 organ, pipe organ 689 | n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO 690 | n03866082 overskirt 691 | n03868242 oxcart 692 | n03868863 oxygen mask 693 | n03871628 packet 694 | n03873416 paddle, boat paddle 695 | n03874293 paddlewheel, paddle wheel 696 | n03874599 padlock 697 | n03876231 paintbrush 698 | n03877472 pajama, pyjama, pj's, jammies 699 | n03877845 palace 700 | n03884397 panpipe, pandean pipe, syrinx 701 | n03887697 paper towel 702 | n03888257 parachute, chute 703 | n03888605 parallel bars, bars 704 | n03891251 park bench 705 | n03891332 parking meter 706 | n03895866 passenger car, coach, carriage 707 | n03899768 patio, terrace 708 | n03902125 pay-phone, pay-station 709 | n03903868 pedestal, plinth, footstall 710 | n03908618 pencil box, pencil case 711 | n03908714 pencil sharpener 712 | n03916031 perfume, essence 713 | n03920288 Petri dish 714 | n03924679 photocopier 715 | n03929660 pick, plectrum, plectron 716 | n03929855 pickelhaube 717 | n03930313 picket fence, paling 718 | n03930630 pickup, pickup truck 719 | n03933933 pier 720 | n03935335 piggy bank, penny bank 721 | n03937543 pill bottle 722 | n03938244 pillow 723 | n03942813 ping-pong ball 724 | n03944341 pinwheel 725 | n03947888 pirate, pirate ship 726 | n03950228 pitcher, ewer 727 | n03954731 plane, carpenter's plane, woodworking plane 728 | n03956157 planetarium 729 | n03958227 plastic bag 730 | n03961711 plate rack 731 | n03967562 plow, plough 732 | n03970156 plunger, plumber's helper 733 | n03976467 Polaroid camera, Polaroid Land camera 734 | n03976657 pole 735 | n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria 736 | n03980874 poncho 737 | n03982430 pool table, billiard table, snooker table 738 | n03983396 pop bottle, soda bottle 739 | n03991062 pot, flowerpot 740 | n03992509 potter's wheel 741 | n03995372 power drill 742 | n03998194 prayer rug, prayer mat 743 | n04004767 printer 744 | n04005630 prison, prison house 745 | n04008634 projectile, missile 746 | n04009552 projector 747 | n04019541 puck, hockey puck 748 | n04023962 punching bag, punch bag, punching ball, punchball 749 | n04026417 purse 750 | n04033901 quill, quill pen 751 | n04033995 quilt, comforter, comfort, puff 752 | n04037443 racer, race car, racing car 753 | n04039381 racket, racquet 754 | n04040759 radiator 755 | n04041544 radio, wireless 756 | n04044716 radio telescope, radio reflector 757 | n04049303 rain barrel 758 | n04065272 recreational vehicle, RV, R.V. 759 | n04067472 reel 760 | n04069434 reflex camera 761 | n04070727 refrigerator, icebox 762 | n04074963 remote control, remote 763 | n04081281 restaurant, eating house, eating place, eatery 764 | n04086273 revolver, six-gun, six-shooter 765 | n04090263 rifle 766 | n04099969 rocking chair, rocker 767 | n04111531 rotisserie 768 | n04116512 rubber eraser, rubber, pencil eraser 769 | n04118538 rugby ball 770 | n04118776 rule, ruler 771 | n04120489 running shoe 772 | n04125021 safe 773 | n04127249 safety pin 774 | n04131690 saltshaker, salt shaker 775 | n04133789 sandal 776 | n04136333 sarong 777 | n04141076 sax, saxophone 778 | n04141327 scabbard 779 | n04141975 scale, weighing machine 780 | n04146614 school bus 781 | n04147183 schooner 782 | n04149813 scoreboard 783 | n04152593 screen, CRT screen 784 | n04153751 screw 785 | n04154565 screwdriver 786 | n04162706 seat belt, seatbelt 787 | n04179913 sewing machine 788 | n04192698 shield, buckler 789 | n04200800 shoe shop, shoe-shop, shoe store 790 | n04201297 shoji 791 | n04204238 shopping basket 792 | n04204347 shopping cart 793 | n04208210 shovel 794 | n04209133 shower cap 795 | n04209239 shower curtain 796 | n04228054 ski 797 | n04229816 ski mask 798 | n04235860 sleeping bag 799 | n04238763 slide rule, slipstick 800 | n04239074 sliding door 801 | n04243546 slot, one-armed bandit 802 | n04251144 snorkel 803 | n04252077 snowmobile 804 | n04252225 snowplow, snowplough 805 | n04254120 soap dispenser 806 | n04254680 soccer ball 807 | n04254777 sock 808 | n04258138 solar dish, solar collector, solar furnace 809 | n04259630 sombrero 810 | n04263257 soup bowl 811 | n04264628 space bar 812 | n04265275 space heater 813 | n04266014 space shuttle 814 | n04270147 spatula 815 | n04273569 speedboat 816 | n04275548 spider web, spider's web 817 | n04277352 spindle 818 | n04285008 sports car, sport car 819 | n04286575 spotlight, spot 820 | n04296562 stage 821 | n04310018 steam locomotive 822 | n04311004 steel arch bridge 823 | n04311174 steel drum 824 | n04317175 stethoscope 825 | n04325704 stole 826 | n04326547 stone wall 827 | n04328186 stopwatch, stop watch 828 | n04330267 stove 829 | n04332243 strainer 830 | n04335435 streetcar, tram, tramcar, trolley, trolley car 831 | n04336792 stretcher 832 | n04344873 studio couch, day bed 833 | n04346328 stupa, tope 834 | n04347754 submarine, pigboat, sub, U-boat 835 | n04350905 suit, suit of clothes 836 | n04355338 sundial 837 | n04355933 sunglass 838 | n04356056 sunglasses, dark glasses, shades 839 | n04357314 sunscreen, sunblock, sun blocker 840 | n04366367 suspension bridge 841 | n04367480 swab, swob, mop 842 | n04370456 sweatshirt 843 | n04371430 swimming trunks, bathing trunks 844 | n04371774 swing 845 | n04372370 switch, electric switch, electrical switch 846 | n04376876 syringe 847 | n04380533 table lamp 848 | n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle 849 | n04392985 tape player 850 | n04398044 teapot 851 | n04399382 teddy, teddy bear 852 | n04404412 television, television system 853 | n04409515 tennis ball 854 | n04417672 thatch, thatched roof 855 | n04418357 theater curtain, theatre curtain 856 | n04423845 thimble 857 | n04428191 thresher, thrasher, threshing machine 858 | n04429376 throne 859 | n04435653 tile roof 860 | n04442312 toaster 861 | n04443257 tobacco shop, tobacconist shop, tobacconist 862 | n04447861 toilet seat 863 | n04456115 torch 864 | n04458633 totem pole 865 | n04461696 tow truck, tow car, wrecker 866 | n04462240 toyshop 867 | n04465501 tractor 868 | n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi 869 | n04476259 tray 870 | n04479046 trench coat 871 | n04482393 tricycle, trike, velocipede 872 | n04483307 trimaran 873 | n04485082 tripod 874 | n04486054 triumphal arch 875 | n04487081 trolleybus, trolley coach, trackless trolley 876 | n04487394 trombone 877 | n04493381 tub, vat 878 | n04501370 turnstile 879 | n04505470 typewriter keyboard 880 | n04507155 umbrella 881 | n04509417 unicycle, monocycle 882 | n04515003 upright, upright piano 883 | n04517823 vacuum, vacuum cleaner 884 | n04522168 vase 885 | n04523525 vault 886 | n04525038 velvet 887 | n04525305 vending machine 888 | n04532106 vestment 889 | n04532670 viaduct 890 | n04536866 violin, fiddle 891 | n04540053 volleyball 892 | n04542943 waffle iron 893 | n04548280 wall clock 894 | n04548362 wallet, billfold, notecase, pocketbook 895 | n04550184 wardrobe, closet, press 896 | n04552348 warplane, military plane 897 | n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin 898 | n04554684 washer, automatic washer, washing machine 899 | n04557648 water bottle 900 | n04560804 water jug 901 | n04562935 water tower 902 | n04579145 whiskey jug 903 | n04579432 whistle 904 | n04584207 wig 905 | n04589890 window screen 906 | n04590129 window shade 907 | n04591157 Windsor tie 908 | n04591713 wine bottle 909 | n04592741 wing 910 | n04596742 wok 911 | n04597913 wooden spoon 912 | n04599235 wool, woolen, woollen 913 | n04604644 worm fence, snake fence, snake-rail fence, Virginia fence 914 | n04606251 wreck 915 | n04612504 yawl 916 | n04613696 yurt 917 | n06359193 web site, website, internet site, site 918 | n06596364 comic book 919 | n06785654 crossword puzzle, crossword 920 | n06794110 street sign 921 | n06874185 traffic light, traffic signal, stoplight 922 | n07248320 book jacket, dust cover, dust jacket, dust wrapper 923 | n07565083 menu 924 | n07579787 plate 925 | n07583066 guacamole 926 | n07584110 consomme 927 | n07590611 hot pot, hotpot 928 | n07613480 trifle 929 | n07614500 ice cream, icecream 930 | n07615774 ice lolly, lolly, lollipop, popsicle 931 | n07684084 French loaf 932 | n07693725 bagel, beigel 933 | n07695742 pretzel 934 | n07697313 cheeseburger 935 | n07697537 hotdog, hot dog, red hot 936 | n07711569 mashed potato 937 | n07714571 head cabbage 938 | n07714990 broccoli 939 | n07715103 cauliflower 940 | n07716358 zucchini, courgette 941 | n07716906 spaghetti squash 942 | n07717410 acorn squash 943 | n07717556 butternut squash 944 | n07718472 cucumber, cuke 945 | n07718747 artichoke, globe artichoke 946 | n07720875 bell pepper 947 | n07730033 cardoon 948 | n07734744 mushroom 949 | n07742313 Granny Smith 950 | n07745940 strawberry 951 | n07747607 orange 952 | n07749582 lemon 953 | n07753113 fig 954 | n07753275 pineapple, ananas 955 | n07753592 banana 956 | n07754684 jackfruit, jak, jack 957 | n07760859 custard apple 958 | n07768694 pomegranate 959 | n07802026 hay 960 | n07831146 carbonara 961 | n07836838 chocolate sauce, chocolate syrup 962 | n07860988 dough 963 | n07871810 meat loaf, meatloaf 964 | n07873807 pizza, pizza pie 965 | n07875152 potpie 966 | n07880968 burrito 967 | n07892512 red wine 968 | n07920052 espresso 969 | n07930864 cup 970 | n07932039 eggnog 971 | n09193705 alp 972 | n09229709 bubble 973 | n09246464 cliff, drop, drop-off 974 | n09256479 coral reef 975 | n09288635 geyser 976 | n09332890 lakeside, lakeshore 977 | n09399592 promontory, headland, head, foreland 978 | n09421951 sandbar, sand bar 979 | n09428293 seashore, coast, seacoast, sea-coast 980 | n09468604 valley, vale 981 | n09472597 volcano 982 | n09835506 ballplayer, baseball player 983 | n10148035 groom, bridegroom 984 | n10565667 scuba diver 985 | n11879895 rapeseed 986 | n11939491 daisy 987 | n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum 988 | n12144580 corn 989 | n12267677 acorn 990 | n12620546 hip, rose hip, rosehip 991 | n12768682 buckeye, horse chestnut, conker 992 | n12985857 coral fungus 993 | n12998815 agaric 994 | n13037406 gyromitra 995 | n13040303 stinkhorn, carrion fungus 996 | n13044778 earthstar 997 | n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa 998 | n13054560 bolete 999 | n13133613 ear, spike, capitulum 1000 | n15075141 toilet tissue, toilet paper, bathroom tissue -------------------------------------------------------------------------------- /extra_autodiff.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "**Appendix D – Autodiff**" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "_This notebook contains toy implementations of various autodiff techniques, to explain how they works._" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "# Setup" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "# Introduction" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "Suppose we want to compute the gradients of the function $f(x,y)=x^2y + y + 2$ with regards to the parameters x and y:" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 1, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "def f(x,y):\n", 45 | " return x*x*y + y + 2" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "One approach is to solve this analytically:\n", 53 | "\n", 54 | "$\\dfrac{\\partial f}{\\partial x} = 2xy$\n", 55 | "\n", 56 | "$\\dfrac{\\partial f}{\\partial y} = x^2 + 1$" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 2, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "def df(x,y):\n", 66 | " return 2*x*y, x*x + 1" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "So for example $\\dfrac{\\partial f}{\\partial x}(3,4) = 24$ and $\\dfrac{\\partial f}{\\partial y}(3,4) = 10$." 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 3, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "text/plain": [ 84 | "(24, 10)" 85 | ] 86 | }, 87 | "execution_count": 3, 88 | "metadata": {}, 89 | "output_type": "execute_result" 90 | } 91 | ], 92 | "source": [ 93 | "df(3, 4)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "Perfect! We can also find the equations for the second order derivatives (also called Hessians):\n", 101 | "\n", 102 | "$\\dfrac{\\partial^2 f}{\\partial x \\partial x} = \\dfrac{\\partial (2xy)}{\\partial x} = 2y$\n", 103 | "\n", 104 | "$\\dfrac{\\partial^2 f}{\\partial x \\partial y} = \\dfrac{\\partial (2xy)}{\\partial y} = 2x$\n", 105 | "\n", 106 | "$\\dfrac{\\partial^2 f}{\\partial y \\partial x} = \\dfrac{\\partial (x^2 + 1)}{\\partial x} = 2x$\n", 107 | "\n", 108 | "$\\dfrac{\\partial^2 f}{\\partial y \\partial y} = \\dfrac{\\partial (x^2 + 1)}{\\partial y} = 0$" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "At x=3 and y=4, these Hessians are respectively 8, 6, 6, 0. Let's use the equations above to compute them:" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 4, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "def d2f(x, y):\n", 125 | " return [2*y, 2*x], [2*x, 0]" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 5, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "data": { 135 | "text/plain": [ 136 | "([8, 6], [6, 0])" 137 | ] 138 | }, 139 | "execution_count": 5, 140 | "metadata": {}, 141 | "output_type": "execute_result" 142 | } 143 | ], 144 | "source": [ 145 | "d2f(3, 4)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "Perfect, but this requires some mathematical work. It is not too hard in this case, but for a deep neural network, it is pratically impossible to compute the derivatives this way. So let's look at various ways to automate this!" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": {}, 158 | "source": [ 159 | "# Numeric differentiation" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "Here, we compute an approxiation of the gradients using the equation: $\\dfrac{\\partial f}{\\partial x} = \\displaystyle{\\lim_{\\epsilon \\to 0}}\\dfrac{f(x+\\epsilon, y) - f(x, y)}{\\epsilon}$ (and there is a similar definition for $\\dfrac{\\partial f}{\\partial y}$)." 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 6, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "def gradients(func, vars_list, eps=0.0001):\n", 176 | " partial_derivatives = []\n", 177 | " base_func_eval = func(*vars_list)\n", 178 | " for idx in range(len(vars_list)):\n", 179 | " tweaked_vars = vars_list[:]\n", 180 | " tweaked_vars[idx] += eps\n", 181 | " tweaked_func_eval = func(*tweaked_vars)\n", 182 | " derivative = (tweaked_func_eval - base_func_eval) / eps\n", 183 | " partial_derivatives.append(derivative)\n", 184 | " return partial_derivatives" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 7, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "def df(x, y):\n", 194 | " return gradients(f, [x, y])" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 8, 200 | "metadata": {}, 201 | "outputs": [ 202 | { 203 | "data": { 204 | "text/plain": [ 205 | "[24.000400000048216, 10.000000000047748]" 206 | ] 207 | }, 208 | "execution_count": 8, 209 | "metadata": {}, 210 | "output_type": "execute_result" 211 | } 212 | ], 213 | "source": [ 214 | "df(3, 4)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "markdown", 219 | "metadata": {}, 220 | "source": [ 221 | "It works well!" 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": {}, 227 | "source": [ 228 | "The good news is that it is pretty easy to compute the Hessians. First let's create functions that compute the first order partial derivatives (also called Jacobians):" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": 9, 234 | "metadata": {}, 235 | "outputs": [ 236 | { 237 | "data": { 238 | "text/plain": [ 239 | "(24.000400000048216, 10.000000000047748)" 240 | ] 241 | }, 242 | "execution_count": 9, 243 | "metadata": {}, 244 | "output_type": "execute_result" 245 | } 246 | ], 247 | "source": [ 248 | "def dfdx(x, y):\n", 249 | " return gradients(f, [x,y])[0]\n", 250 | "\n", 251 | "def dfdy(x, y):\n", 252 | " return gradients(f, [x,y])[1]\n", 253 | "\n", 254 | "dfdx(3., 4.), dfdy(3., 4.)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "Now we can simply apply the `gradients()` function to these functions:" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 10, 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "def d2f(x, y):\n", 271 | " return [gradients(dfdx, [3., 4.]), gradients(dfdy, [3., 4.])]" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 11, 277 | "metadata": {}, 278 | "outputs": [ 279 | { 280 | "data": { 281 | "text/plain": [ 282 | "[[7.999999951380232, 6.000099261882497],\n", 283 | " [6.000099261882497, -1.4210854715202004e-06]]" 284 | ] 285 | }, 286 | "execution_count": 11, 287 | "metadata": {}, 288 | "output_type": "execute_result" 289 | } 290 | ], 291 | "source": [ 292 | "d2f(3, 4)" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "So everything works well, but the result is approximate, and computing the gradients of a function with regards to $n$ variables requires calling that function $n$ times. In deep neural nets, there are often thousands of parameters to tweak using gradient descent (which requires computing the gradients of the loss function with regards to each of these parameters), so this approach would be much too slow." 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "## Implementing a Toy Computation Graph" 307 | ] 308 | }, 309 | { 310 | "cell_type": "markdown", 311 | "metadata": {}, 312 | "source": [ 313 | "Rather than this numerical approach, let's implement some symbolic autodiff techniques. For this, we will need to define classes to represent constants, variables and operations." 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 12, 319 | "metadata": {}, 320 | "outputs": [], 321 | "source": [ 322 | "class Const(object):\n", 323 | " def __init__(self, value):\n", 324 | " self.value = value\n", 325 | " def evaluate(self):\n", 326 | " return self.value\n", 327 | " def __str__(self):\n", 328 | " return str(self.value)\n", 329 | "\n", 330 | "class Var(object):\n", 331 | " def __init__(self, name, init_value=0):\n", 332 | " self.value = init_value\n", 333 | " self.name = name\n", 334 | " def evaluate(self):\n", 335 | " return self.value\n", 336 | " def __str__(self):\n", 337 | " return self.name\n", 338 | "\n", 339 | "class BinaryOperator(object):\n", 340 | " def __init__(self, a, b):\n", 341 | " self.a = a\n", 342 | " self.b = b\n", 343 | "\n", 344 | "class Add(BinaryOperator):\n", 345 | " def evaluate(self):\n", 346 | " return self.a.evaluate() + self.b.evaluate()\n", 347 | " def __str__(self):\n", 348 | " return \"{} + {}\".format(self.a, self.b)\n", 349 | "\n", 350 | "class Mul(BinaryOperator):\n", 351 | " def evaluate(self):\n", 352 | " return self.a.evaluate() * self.b.evaluate()\n", 353 | " def __str__(self):\n", 354 | " return \"({}) * ({})\".format(self.a, self.b)" 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "Good, now we can build a computation graph to represent the function $f$:" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": 13, 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "x = Var(\"x\")\n", 371 | "y = Var(\"y\")\n", 372 | "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2" 373 | ] 374 | }, 375 | { 376 | "cell_type": "markdown", 377 | "metadata": {}, 378 | "source": [ 379 | "And we can run this graph to compute $f$ at any point, for example $f(3, 4)$." 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": 14, 385 | "metadata": {}, 386 | "outputs": [ 387 | { 388 | "data": { 389 | "text/plain": [ 390 | "42" 391 | ] 392 | }, 393 | "execution_count": 14, 394 | "metadata": {}, 395 | "output_type": "execute_result" 396 | } 397 | ], 398 | "source": [ 399 | "x.value = 3\n", 400 | "y.value = 4\n", 401 | "f.evaluate()" 402 | ] 403 | }, 404 | { 405 | "cell_type": "markdown", 406 | "metadata": {}, 407 | "source": [ 408 | "Perfect, it found the ultimate answer." 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": {}, 414 | "source": [ 415 | "## Computing gradients" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "metadata": {}, 421 | "source": [ 422 | "The autodiff methods we will present below are all based on the *chain rule*." 423 | ] 424 | }, 425 | { 426 | "cell_type": "markdown", 427 | "metadata": {}, 428 | "source": [ 429 | "Suppose we have two functions $u$ and $v$, and we apply them sequentially to some input $x$, and we get the result $z$. So we have $z = v(u(x))$, which we can rewrite as $z = v(s)$ and $s = u(x)$. Now we can apply the chain rule to get the partial derivative of the output $z$ with regards to the input $x$:\n", 430 | "\n", 431 | "$ \\dfrac{\\partial z}{\\partial x} = \\dfrac{\\partial s}{\\partial x} \\cdot \\dfrac{\\partial z}{\\partial s}$" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "Now if $z$ is the output of a sequence of functions which have intermediate outputs $s_1, s_2, ..., s_n$, the chain rule still applies:\n", 439 | "\n", 440 | "$ \\dfrac{\\partial z}{\\partial x} = \\dfrac{\\partial s_1}{\\partial x} \\cdot \\dfrac{\\partial s_2}{\\partial s_1} \\cdot \\dfrac{\\partial s_3}{\\partial s_2} \\cdot \\dots \\cdot \\dfrac{\\partial s_{n-1}}{\\partial s_{n-2}} \\cdot \\dfrac{\\partial s_n}{\\partial s_{n-1}} \\cdot \\dfrac{\\partial z}{\\partial s_n}$" 441 | ] 442 | }, 443 | { 444 | "cell_type": "markdown", 445 | "metadata": {}, 446 | "source": [ 447 | "In forward mode autodiff, the algorithm computes these terms \"forward\" (i.e., in the same order as the computations required to compute the output $z$), that is from left to right: first $\\dfrac{\\partial s_1}{\\partial x}$, then $\\dfrac{\\partial s_2}{\\partial s_1}$, and so on. In reverse mode autodiff, the algorithm computes these terms \"backwards\", from right to left: first $\\dfrac{\\partial z}{\\partial s_n}$, then $\\dfrac{\\partial s_n}{\\partial s_{n-1}}$, and so on.\n", 448 | "\n", 449 | "For example, suppose you want to compute the derivative of the function $z(x)=\\sin(x^2)$ at x=3, using forward mode autodiff. The algorithm would first compute the partial derivative $\\dfrac{\\partial s_1}{\\partial x}=\\dfrac{\\partial x^2}{\\partial x}=2x=6$. Next, it would compute $\\dfrac{\\partial z}{\\partial x}=\\dfrac{\\partial s_1}{\\partial x}\\cdot\\dfrac{\\partial z}{\\partial s_1}= 6 \\cdot \\dfrac{\\partial \\sin(s_1)}{\\partial s_1}=6 \\cdot \\cos(s_1)=6 \\cdot \\cos(3^2)\\approx-5.46$." 450 | ] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": {}, 455 | "source": [ 456 | "Let's verify this result using the `gradients()` function defined earlier:" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": 15, 462 | "metadata": {}, 463 | "outputs": [ 464 | { 465 | "data": { 466 | "text/plain": [ 467 | "[-5.46761419430053]" 468 | ] 469 | }, 470 | "execution_count": 15, 471 | "metadata": {}, 472 | "output_type": "execute_result" 473 | } 474 | ], 475 | "source": [ 476 | "from math import sin\n", 477 | "\n", 478 | "def z(x):\n", 479 | " return sin(x**2)\n", 480 | "\n", 481 | "gradients(z, [3])" 482 | ] 483 | }, 484 | { 485 | "cell_type": "markdown", 486 | "metadata": {}, 487 | "source": [ 488 | "Look good. Now let's do the same thing using reverse mode autodiff. This time the algorithm would start from the right hand side so it would compute $\\dfrac{\\partial z}{\\partial s_1} = \\dfrac{\\partial \\sin(s_1)}{\\partial s_1}=\\cos(s_1)=\\cos(3^2)\\approx -0.91$. Next it would compute $\\dfrac{\\partial z}{\\partial x}=\\dfrac{\\partial s_1}{\\partial x}\\cdot\\dfrac{\\partial z}{\\partial s_1} \\approx \\dfrac{\\partial s_1}{\\partial x} \\cdot -0.91 = \\dfrac{\\partial x^2}{\\partial x} \\cdot -0.91=2x \\cdot -0.91 = 6\\cdot-0.91=-5.46$." 489 | ] 490 | }, 491 | { 492 | "cell_type": "markdown", 493 | "metadata": {}, 494 | "source": [ 495 | "Of course both approaches give the same result (except for rounding errors), and with a single input and output they involve the same number of computations. But when there are several inputs or outputs, they can have very different performance. Indeed, if there are many inputs, the right-most terms will be needed to compute the partial derivatives with regards to each input, so it is a good idea to compute these right-most terms first. That means using reverse-mode autodiff. This way, the right-most terms can be computed just once and used to compute all the partial derivatives. Conversely, if there are many outputs, forward-mode is generally preferable because the left-most terms can be computed just once to compute the partial derivatives of the different outputs. In Deep Learning, there are typically thousands of model parameters, meaning there are lots of inputs, but few outputs. In fact, there is generally just one output during training: the loss. This is why reverse mode autodiff is used in TensorFlow and all major Deep Learning libraries." 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": {}, 501 | "source": [ 502 | "There's one additional complexity in reverse mode autodiff: the value of $s_i$ is generally required when computing $\\dfrac{\\partial s_{i+1}}{\\partial s_i}$, and computing $s_i$ requires first computing $s_{i-1}$, which requires computing $s_{i-2}$, and so on. So basically, a first pass forward through the network is required to compute $s_1$, $s_2$, $s_3$, $\\dots$, $s_{n-1}$ and $s_n$, and then the algorithm can compute the partial derivatives from right to left. Storing all the intermediate values $s_i$ in RAM is sometimes a problem, especially when handling images, and when using GPUs which often have limited RAM: to limit this problem, one can reduce the number of layers in the neural network, or configure TensorFlow to make it swap these values from GPU RAM to CPU RAM. Another approach is to only cache every other intermediate value, $s_1$, $s_3$, $s_5$, $\\dots$, $s_{n-4}$, $s_{n-2}$ and $s_n$. This means that when the algorithm computes the partial derivatives, if an intermediate value $s_i$ is missing, it will need to recompute it based on the previous intermediate value $s_{i-1}$. This trades off CPU for RAM (if you are interested, check out [this paper](https://pdfs.semanticscholar.org/f61e/9fd5a4878e1493f7a6b03774a61c17b7e9a4.pdf))." 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "metadata": {}, 508 | "source": [ 509 | "### Forward mode autodiff" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 16, 515 | "metadata": {}, 516 | "outputs": [], 517 | "source": [ 518 | "Const.gradient = lambda self, var: Const(0)\n", 519 | "Var.gradient = lambda self, var: Const(1) if self is var else Const(0)\n", 520 | "Add.gradient = lambda self, var: Add(self.a.gradient(var), self.b.gradient(var))\n", 521 | "Mul.gradient = lambda self, var: Add(Mul(self.a, self.b.gradient(var)), Mul(self.a.gradient(var), self.b))\n", 522 | "\n", 523 | "x = Var(name=\"x\", init_value=3.)\n", 524 | "y = Var(name=\"y\", init_value=4.)\n", 525 | "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", 526 | "\n", 527 | "dfdx = f.gradient(x) # 2xy\n", 528 | "dfdy = f.gradient(y) # x² + 1" 529 | ] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": 17, 534 | "metadata": {}, 535 | "outputs": [ 536 | { 537 | "data": { 538 | "text/plain": [ 539 | "(24.0, 10.0)" 540 | ] 541 | }, 542 | "execution_count": 17, 543 | "metadata": {}, 544 | "output_type": "execute_result" 545 | } 546 | ], 547 | "source": [ 548 | "dfdx.evaluate(), dfdy.evaluate()" 549 | ] 550 | }, 551 | { 552 | "cell_type": "markdown", 553 | "metadata": {}, 554 | "source": [ 555 | "Since the output of the `gradient()` method is fully symbolic, we are not limited to the first order derivatives, we can also compute second order derivatives, and so on:" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 18, 561 | "metadata": {}, 562 | "outputs": [], 563 | "source": [ 564 | "d2fdxdx = dfdx.gradient(x) # 2y\n", 565 | "d2fdxdy = dfdx.gradient(y) # 2x\n", 566 | "d2fdydx = dfdy.gradient(x) # 2x\n", 567 | "d2fdydy = dfdy.gradient(y) # 0" 568 | ] 569 | }, 570 | { 571 | "cell_type": "code", 572 | "execution_count": 19, 573 | "metadata": {}, 574 | "outputs": [ 575 | { 576 | "data": { 577 | "text/plain": [ 578 | "[[8.0, 6.0], [6.0, 0.0]]" 579 | ] 580 | }, 581 | "execution_count": 19, 582 | "metadata": {}, 583 | "output_type": "execute_result" 584 | } 585 | ], 586 | "source": [ 587 | "[[d2fdxdx.evaluate(), d2fdxdy.evaluate()],\n", 588 | " [d2fdydx.evaluate(), d2fdydy.evaluate()]]" 589 | ] 590 | }, 591 | { 592 | "cell_type": "markdown", 593 | "metadata": {}, 594 | "source": [ 595 | "Note that the result is now exact, not an approximation (up to the limit of the machine's float precision, of course)." 596 | ] 597 | }, 598 | { 599 | "cell_type": "markdown", 600 | "metadata": {}, 601 | "source": [ 602 | "### Forward mode autodiff using dual numbers" 603 | ] 604 | }, 605 | { 606 | "cell_type": "markdown", 607 | "metadata": {}, 608 | "source": [ 609 | "A nice way to apply forward mode autodiff is to use [dual numbers](https://en.wikipedia.org/wiki/Dual_number). In short, a dual number $z$ has the form $z = a + b\\epsilon$, where $a$ and $b$ are real numbers, and $\\epsilon$ is an infinitesimal number, positive but smaller than all real numbers, and such that $\\epsilon^2=0$.\n", 610 | "It can be shown that $f(x + \\epsilon) = f(x) + \\dfrac{\\partial f}{\\partial x}\\epsilon$, so simply by computing $f(x + \\epsilon)$ we get both the value of $f(x)$ and the partial derivative of $f$ with regards to $x$. " 611 | ] 612 | }, 613 | { 614 | "cell_type": "markdown", 615 | "metadata": {}, 616 | "source": [ 617 | "Dual numbers have their own arithmetic rules, which are generally quite natural. For example:\n", 618 | "\n", 619 | "**Addition**\n", 620 | "\n", 621 | "$(a_1 + b_1\\epsilon) + (a_2 + b_2\\epsilon) = (a_1 + a_2) + (b_1 + b_2)\\epsilon$\n", 622 | "\n", 623 | "**Subtraction**\n", 624 | "\n", 625 | "$(a_1 + b_1\\epsilon) - (a_2 + b_2\\epsilon) = (a_1 - a_2) + (b_1 - b_2)\\epsilon$\n", 626 | "\n", 627 | "**Multiplication**\n", 628 | "\n", 629 | "$(a_1 + b_1\\epsilon) \\times (a_2 + b_2\\epsilon) = (a_1 a_2) + (a_1 b_2 + a_2 b_1)\\epsilon + b_1 b_2\\epsilon^2 = (a_1 a_2) + (a_1b_2 + a_2b_1)\\epsilon$\n", 630 | "\n", 631 | "**Division**\n", 632 | "\n", 633 | "$\\dfrac{a_1 + b_1\\epsilon}{a_2 + b_2\\epsilon} = \\dfrac{a_1 + b_1\\epsilon}{a_2 + b_2\\epsilon} \\cdot \\dfrac{a_2 - b_2\\epsilon}{a_2 - b_2\\epsilon} = \\dfrac{a_1 a_2 + (b_1 a_2 - a_1 b_2)\\epsilon - b_1 b_2\\epsilon^2}{{a_2}^2 + (a_2 b_2 - a_2 b_2)\\epsilon - {b_2}^2\\epsilon} = \\dfrac{a_1}{a_2} + \\dfrac{a_1 b_2 - b_1 a_2}{{a_2}^2}\\epsilon$\n", 634 | "\n", 635 | "**Power**\n", 636 | "\n", 637 | "$(a + b\\epsilon)^n = a^n + (n a^{n-1}b)\\epsilon$\n", 638 | "\n", 639 | "etc." 640 | ] 641 | }, 642 | { 643 | "cell_type": "markdown", 644 | "metadata": {}, 645 | "source": [ 646 | "Let's create a class to represent dual numbers, and implement a few operations (addition and multiplication). You can try adding some more if you want." 647 | ] 648 | }, 649 | { 650 | "cell_type": "code", 651 | "execution_count": 20, 652 | "metadata": {}, 653 | "outputs": [], 654 | "source": [ 655 | "class DualNumber(object):\n", 656 | " def __init__(self, value=0.0, eps=0.0):\n", 657 | " self.value = value\n", 658 | " self.eps = eps\n", 659 | " def __add__(self, b):\n", 660 | " return DualNumber(self.value + self.to_dual(b).value,\n", 661 | " self.eps + self.to_dual(b).eps)\n", 662 | " def __radd__(self, a):\n", 663 | " return self.to_dual(a).__add__(self)\n", 664 | " def __mul__(self, b):\n", 665 | " return DualNumber(self.value * self.to_dual(b).value,\n", 666 | " self.eps * self.to_dual(b).value + self.value * self.to_dual(b).eps)\n", 667 | " def __rmul__(self, a):\n", 668 | " return self.to_dual(a).__mul__(self)\n", 669 | " def __str__(self):\n", 670 | " if self.eps:\n", 671 | " return \"{:.1f} + {:.1f}ε\".format(self.value, self.eps)\n", 672 | " else:\n", 673 | " return \"{:.1f}\".format(self.value)\n", 674 | " def __repr__(self):\n", 675 | " return str(self)\n", 676 | " @classmethod\n", 677 | " def to_dual(cls, n):\n", 678 | " if hasattr(n, \"value\"):\n", 679 | " return n\n", 680 | " else:\n", 681 | " return cls(n)" 682 | ] 683 | }, 684 | { 685 | "cell_type": "markdown", 686 | "metadata": {}, 687 | "source": [ 688 | "$3 + (3 + 4 \\epsilon) = 6 + 4\\epsilon$" 689 | ] 690 | }, 691 | { 692 | "cell_type": "code", 693 | "execution_count": 21, 694 | "metadata": {}, 695 | "outputs": [ 696 | { 697 | "data": { 698 | "text/plain": [ 699 | "6.0 + 4.0ε" 700 | ] 701 | }, 702 | "execution_count": 21, 703 | "metadata": {}, 704 | "output_type": "execute_result" 705 | } 706 | ], 707 | "source": [ 708 | "3 + DualNumber(3, 4)" 709 | ] 710 | }, 711 | { 712 | "cell_type": "markdown", 713 | "metadata": {}, 714 | "source": [ 715 | "$(3 + 4ε)\\times(5 + 7ε)$ = $3 \\times 5 + 3 \\times 7ε + 4ε \\times 5 + 4ε \\times 7ε$ = $15 + 21ε + 20ε + 28ε^2$ = $15 + 41ε + 28 \\times 0$ = $15 + 41ε$" 716 | ] 717 | }, 718 | { 719 | "cell_type": "code", 720 | "execution_count": 22, 721 | "metadata": {}, 722 | "outputs": [ 723 | { 724 | "data": { 725 | "text/plain": [ 726 | "15.0 + 41.0ε" 727 | ] 728 | }, 729 | "execution_count": 22, 730 | "metadata": {}, 731 | "output_type": "execute_result" 732 | } 733 | ], 734 | "source": [ 735 | "DualNumber(3, 4) * DualNumber(5, 7)" 736 | ] 737 | }, 738 | { 739 | "cell_type": "markdown", 740 | "metadata": {}, 741 | "source": [ 742 | "Now let's see if the dual numbers work with our toy computation framework:" 743 | ] 744 | }, 745 | { 746 | "cell_type": "code", 747 | "execution_count": 23, 748 | "metadata": {}, 749 | "outputs": [ 750 | { 751 | "data": { 752 | "text/plain": [ 753 | "42.0" 754 | ] 755 | }, 756 | "execution_count": 23, 757 | "metadata": {}, 758 | "output_type": "execute_result" 759 | } 760 | ], 761 | "source": [ 762 | "x.value = DualNumber(3.0)\n", 763 | "y.value = DualNumber(4.0)\n", 764 | "\n", 765 | "f.evaluate()" 766 | ] 767 | }, 768 | { 769 | "cell_type": "markdown", 770 | "metadata": {}, 771 | "source": [ 772 | "Yep, sure works. Now let's use this to compute the partial derivatives of $f$ with regards to $x$ and $y$ at x=3 and y=4:" 773 | ] 774 | }, 775 | { 776 | "cell_type": "code", 777 | "execution_count": 24, 778 | "metadata": {}, 779 | "outputs": [], 780 | "source": [ 781 | "x.value = DualNumber(3.0, 1.0) # 3 + ε\n", 782 | "y.value = DualNumber(4.0) # 4\n", 783 | "\n", 784 | "dfdx = f.evaluate().eps\n", 785 | "\n", 786 | "x.value = DualNumber(3.0) # 3\n", 787 | "y.value = DualNumber(4.0, 1.0) # 4 + ε\n", 788 | "\n", 789 | "dfdy = f.evaluate().eps" 790 | ] 791 | }, 792 | { 793 | "cell_type": "code", 794 | "execution_count": 25, 795 | "metadata": {}, 796 | "outputs": [ 797 | { 798 | "data": { 799 | "text/plain": [ 800 | "24.0" 801 | ] 802 | }, 803 | "execution_count": 25, 804 | "metadata": {}, 805 | "output_type": "execute_result" 806 | } 807 | ], 808 | "source": [ 809 | "dfdx" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 26, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "data": { 819 | "text/plain": [ 820 | "10.0" 821 | ] 822 | }, 823 | "execution_count": 26, 824 | "metadata": {}, 825 | "output_type": "execute_result" 826 | } 827 | ], 828 | "source": [ 829 | "dfdy" 830 | ] 831 | }, 832 | { 833 | "cell_type": "markdown", 834 | "metadata": {}, 835 | "source": [ 836 | "Great! However, in this implementation we are limited to first order derivatives.\n", 837 | "Now let's look at reverse mode." 838 | ] 839 | }, 840 | { 841 | "cell_type": "markdown", 842 | "metadata": {}, 843 | "source": [ 844 | "### Reverse mode autodiff" 845 | ] 846 | }, 847 | { 848 | "cell_type": "markdown", 849 | "metadata": {}, 850 | "source": [ 851 | "Let's rewrite our toy framework to add reverse mode autodiff:" 852 | ] 853 | }, 854 | { 855 | "cell_type": "code", 856 | "execution_count": 27, 857 | "metadata": {}, 858 | "outputs": [], 859 | "source": [ 860 | "class Const(object):\n", 861 | " def __init__(self, value):\n", 862 | " self.value = value\n", 863 | " def evaluate(self):\n", 864 | " return self.value\n", 865 | " def backpropagate(self, gradient):\n", 866 | " pass\n", 867 | " def __str__(self):\n", 868 | " return str(self.value)\n", 869 | "\n", 870 | "class Var(object):\n", 871 | " def __init__(self, name, init_value=0):\n", 872 | " self.value = init_value\n", 873 | " self.name = name\n", 874 | " self.gradient = 0\n", 875 | " def evaluate(self):\n", 876 | " return self.value\n", 877 | " def backpropagate(self, gradient):\n", 878 | " self.gradient += gradient\n", 879 | " def __str__(self):\n", 880 | " return self.name\n", 881 | "\n", 882 | "class BinaryOperator(object):\n", 883 | " def __init__(self, a, b):\n", 884 | " self.a = a\n", 885 | " self.b = b\n", 886 | "\n", 887 | "class Add(BinaryOperator):\n", 888 | " def evaluate(self):\n", 889 | " self.value = self.a.evaluate() + self.b.evaluate()\n", 890 | " return self.value\n", 891 | " def backpropagate(self, gradient):\n", 892 | " self.a.backpropagate(gradient)\n", 893 | " self.b.backpropagate(gradient)\n", 894 | " def __str__(self):\n", 895 | " return \"{} + {}\".format(self.a, self.b)\n", 896 | "\n", 897 | "class Mul(BinaryOperator):\n", 898 | " def evaluate(self):\n", 899 | " self.value = self.a.evaluate() * self.b.evaluate()\n", 900 | " return self.value\n", 901 | " def backpropagate(self, gradient):\n", 902 | " self.a.backpropagate(gradient * self.b.value)\n", 903 | " self.b.backpropagate(gradient * self.a.value)\n", 904 | " def __str__(self):\n", 905 | " return \"({}) * ({})\".format(self.a, self.b)" 906 | ] 907 | }, 908 | { 909 | "cell_type": "code", 910 | "execution_count": 28, 911 | "metadata": {}, 912 | "outputs": [], 913 | "source": [ 914 | "x = Var(\"x\", init_value=3)\n", 915 | "y = Var(\"y\", init_value=4)\n", 916 | "f = Add(Mul(Mul(x, x), y), Add(y, Const(2))) # f(x,y) = x²y + y + 2\n", 917 | "\n", 918 | "result = f.evaluate()\n", 919 | "f.backpropagate(1.0)" 920 | ] 921 | }, 922 | { 923 | "cell_type": "code", 924 | "execution_count": 29, 925 | "metadata": {}, 926 | "outputs": [ 927 | { 928 | "name": "stdout", 929 | "output_type": "stream", 930 | "text": [ 931 | "((x) * (x)) * (y) + y + 2\n" 932 | ] 933 | } 934 | ], 935 | "source": [ 936 | "print(f)" 937 | ] 938 | }, 939 | { 940 | "cell_type": "code", 941 | "execution_count": 30, 942 | "metadata": {}, 943 | "outputs": [ 944 | { 945 | "data": { 946 | "text/plain": [ 947 | "42" 948 | ] 949 | }, 950 | "execution_count": 30, 951 | "metadata": {}, 952 | "output_type": "execute_result" 953 | } 954 | ], 955 | "source": [ 956 | "result" 957 | ] 958 | }, 959 | { 960 | "cell_type": "code", 961 | "execution_count": 31, 962 | "metadata": {}, 963 | "outputs": [ 964 | { 965 | "data": { 966 | "text/plain": [ 967 | "24.0" 968 | ] 969 | }, 970 | "execution_count": 31, 971 | "metadata": {}, 972 | "output_type": "execute_result" 973 | } 974 | ], 975 | "source": [ 976 | "x.gradient" 977 | ] 978 | }, 979 | { 980 | "cell_type": "code", 981 | "execution_count": 32, 982 | "metadata": {}, 983 | "outputs": [ 984 | { 985 | "data": { 986 | "text/plain": [ 987 | "10.0" 988 | ] 989 | }, 990 | "execution_count": 32, 991 | "metadata": {}, 992 | "output_type": "execute_result" 993 | } 994 | ], 995 | "source": [ 996 | "y.gradient" 997 | ] 998 | }, 999 | { 1000 | "cell_type": "markdown", 1001 | "metadata": {}, 1002 | "source": [ 1003 | "Again, in this implementation the outputs are just numbers, not symbolic expressions, so we are limited to first order derivatives. However, we could have made the `backpropagate()` methods return symbolic expressions rather than values (e.g., return `Add(2,3)` rather than 5). This would make it possible to compute second order gradients (and beyond). This is what TensorFlow does, as do all the major libraries that implement autodiff." 1004 | ] 1005 | }, 1006 | { 1007 | "cell_type": "markdown", 1008 | "metadata": {}, 1009 | "source": [ 1010 | "### Reverse mode autodiff using TensorFlow" 1011 | ] 1012 | }, 1013 | { 1014 | "cell_type": "code", 1015 | "execution_count": 33, 1016 | "metadata": {}, 1017 | "outputs": [], 1018 | "source": [ 1019 | "import tensorflow as tf" 1020 | ] 1021 | }, 1022 | { 1023 | "cell_type": "code", 1024 | "execution_count": 34, 1025 | "metadata": {}, 1026 | "outputs": [ 1027 | { 1028 | "data": { 1029 | "text/plain": [ 1030 | "[,\n", 1031 | " ]" 1032 | ] 1033 | }, 1034 | "execution_count": 34, 1035 | "metadata": {}, 1036 | "output_type": "execute_result" 1037 | } 1038 | ], 1039 | "source": [ 1040 | "x = tf.Variable(3.)\n", 1041 | "y = tf.Variable(4.)\n", 1042 | "\n", 1043 | "with tf.GradientTape() as tape:\n", 1044 | " f = x*x*y + y + 2\n", 1045 | "\n", 1046 | "jacobians = tape.gradient(f, [x, y])\n", 1047 | "jacobians" 1048 | ] 1049 | }, 1050 | { 1051 | "cell_type": "markdown", 1052 | "metadata": {}, 1053 | "source": [ 1054 | "Since everything is symbolic, we can compute second order derivatives, and beyond:" 1055 | ] 1056 | }, 1057 | { 1058 | "cell_type": "code", 1059 | "execution_count": 35, 1060 | "metadata": {}, 1061 | "outputs": [ 1062 | { 1063 | "name": "stdout", 1064 | "output_type": "stream", 1065 | "text": [ 1066 | "WARNING:tensorflow:Calling GradientTape.gradient on a persistent tape inside its context is significantly less efficient than calling it outside the context (it causes the gradient ops to be recorded on the tape, leading to increased CPU and memory usage). Only call GradientTape.gradient inside the context if you actually want to trace the gradient in order to compute higher order derivatives.\n" 1067 | ] 1068 | }, 1069 | { 1070 | "data": { 1071 | "text/plain": [ 1072 | "[[,\n", 1073 | " ],\n", 1074 | " [, None]]" 1075 | ] 1076 | }, 1077 | "execution_count": 35, 1078 | "metadata": {}, 1079 | "output_type": "execute_result" 1080 | } 1081 | ], 1082 | "source": [ 1083 | "x = tf.Variable(3.)\n", 1084 | "y = tf.Variable(4.)\n", 1085 | "\n", 1086 | "with tf.GradientTape(persistent=True) as tape:\n", 1087 | " f = x*x*y + y + 2\n", 1088 | " df_dx, df_dy = tape.gradient(f, [x, y])\n", 1089 | "\n", 1090 | "d2f_d2x, d2f_dydx = tape.gradient(df_dx, [x, y])\n", 1091 | "d2f_dxdy, d2f_d2y = tape.gradient(df_dy, [x, y])\n", 1092 | "del tape\n", 1093 | "\n", 1094 | "hessians = [[d2f_d2x, d2f_dydx], [d2f_dxdy, d2f_d2y]]\n", 1095 | "hessians" 1096 | ] 1097 | }, 1098 | { 1099 | "cell_type": "markdown", 1100 | "metadata": {}, 1101 | "source": [ 1102 | "Note that when we compute the derivative of a tensor with regards to a variable that it does not depend on, instead of returning 0.0, the `gradient()` function returns `None`." 1103 | ] 1104 | }, 1105 | { 1106 | "cell_type": "markdown", 1107 | "metadata": {}, 1108 | "source": [ 1109 | "And that's all folks! Hope you enjoyed this notebook." 1110 | ] 1111 | } 1112 | ], 1113 | "metadata": { 1114 | "kernelspec": { 1115 | "display_name": "Python 3", 1116 | "language": "python", 1117 | "name": "python3" 1118 | }, 1119 | "language_info": { 1120 | "codemirror_mode": { 1121 | "name": "ipython", 1122 | "version": 3 1123 | }, 1124 | "file_extension": ".py", 1125 | "mimetype": "text/x-python", 1126 | "name": "python", 1127 | "nbconvert_exporter": "python", 1128 | "pygments_lexer": "ipython3", 1129 | "version": "3.7.6" 1130 | }, 1131 | "nav_menu": { 1132 | "height": "603px", 1133 | "width": "616px" 1134 | }, 1135 | "toc": { 1136 | "navigate_menu": true, 1137 | "number_sections": true, 1138 | "sideBar": true, 1139 | "threshold": 6, 1140 | "toc_cell": false, 1141 | "toc_section_display": "block", 1142 | "toc_window_display": true 1143 | } 1144 | }, 1145 | "nbformat": 4, 1146 | "nbformat_minor": 1 1147 | } 1148 | -------------------------------------------------------------------------------- /19_training_and_deploying_at_scale.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "**Chapter 19 – Training and Deploying TensorFlow Models at Scale**" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "_This notebook contains all the sample code in chapter 19._" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "\n", 22 | " \n", 25 | "
\n", 23 | " Run in Google Colab\n", 24 | "
" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "# Setup\n", 33 | "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0.\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 1, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "# Python ≥3.5 is required\n", 43 | "import sys\n", 44 | "assert sys.version_info >= (3, 5)\n", 45 | "\n", 46 | "# Scikit-Learn ≥0.20 is required\n", 47 | "import sklearn\n", 48 | "assert sklearn.__version__ >= \"0.20\"\n", 49 | "\n", 50 | "try:\n", 51 | " # %tensorflow_version only exists in Colab.\n", 52 | " %tensorflow_version 2.x\n", 53 | " !echo \"deb http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal\" > /etc/apt/sources.list.d/tensorflow-serving.list\n", 54 | " !curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | apt-key add -\n", 55 | " !apt update && apt-get install -y tensorflow-model-server\n", 56 | " !pip install -q -U tensorflow-serving-api\n", 57 | " IS_COLAB = True\n", 58 | "except Exception:\n", 59 | " IS_COLAB = False\n", 60 | "\n", 61 | "# TensorFlow ≥2.0 is required\n", 62 | "import tensorflow as tf\n", 63 | "from tensorflow import keras\n", 64 | "assert tf.__version__ >= \"2.0\"\n", 65 | "\n", 66 | "if not tf.config.list_physical_devices('GPU'):\n", 67 | " print(\"No GPU was detected. CNNs can be very slow without a GPU.\")\n", 68 | " if IS_COLAB:\n", 69 | " print(\"Go to Runtime > Change runtime and select a GPU hardware accelerator.\")\n", 70 | "\n", 71 | "# Common imports\n", 72 | "import numpy as np\n", 73 | "import os\n", 74 | "\n", 75 | "# to make this notebook's output stable across runs\n", 76 | "np.random.seed(42)\n", 77 | "tf.random.set_seed(42)\n", 78 | "\n", 79 | "# To plot pretty figures\n", 80 | "%matplotlib inline\n", 81 | "import matplotlib as mpl\n", 82 | "import matplotlib.pyplot as plt\n", 83 | "mpl.rc('axes', labelsize=14)\n", 84 | "mpl.rc('xtick', labelsize=12)\n", 85 | "mpl.rc('ytick', labelsize=12)\n", 86 | "\n", 87 | "# Where to save the figures\n", 88 | "PROJECT_ROOT_DIR = \".\"\n", 89 | "CHAPTER_ID = \"deploy\"\n", 90 | "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n", 91 | "os.makedirs(IMAGES_PATH, exist_ok=True)\n", 92 | "\n", 93 | "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n", 94 | " path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n", 95 | " print(\"Saving figure\", fig_id)\n", 96 | " if tight_layout:\n", 97 | " plt.tight_layout()\n", 98 | " plt.savefig(path, format=fig_extension, dpi=resolution)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "# Deploying TensorFlow models to TensorFlow Serving (TFS)\n", 106 | "We will use the REST API or the gRPC API." 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Save/Load a `SavedModel`" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 2, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()\n", 123 | "X_train_full = X_train_full[..., np.newaxis].astype(np.float32) / 255.\n", 124 | "X_test = X_test[..., np.newaxis].astype(np.float32) / 255.\n", 125 | "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", 126 | "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n", 127 | "X_new = X_test[:3]" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 3, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "name": "stdout", 137 | "output_type": "stream", 138 | "text": [ 139 | "Train on 55000 samples, validate on 5000 samples\n", 140 | "Epoch 1/10\n", 141 | "55000/55000 [==============================] - 2s 40us/sample - loss: 0.7018 - accuracy: 0.8223 - val_loss: 0.3722 - val_accuracy: 0.9022\n", 142 | "Epoch 2/10\n", 143 | "55000/55000 [==============================] - 2s 36us/sample - loss: 0.3528 - accuracy: 0.9021 - val_loss: 0.3000 - val_accuracy: 0.9170\n", 144 | "Epoch 3/10\n", 145 | "55000/55000 [==============================] - 2s 36us/sample - loss: 0.3032 - accuracy: 0.9150 - val_loss: 0.2659 - val_accuracy: 0.9280\n", 146 | "Epoch 4/10\n", 147 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.2730 - accuracy: 0.9233 - val_loss: 0.2442 - val_accuracy: 0.9342\n", 148 | "Epoch 5/10\n", 149 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.2504 - accuracy: 0.9305 - val_loss: 0.2272 - val_accuracy: 0.9346\n", 150 | "Epoch 6/10\n", 151 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.2319 - accuracy: 0.9353 - val_loss: 0.2104 - val_accuracy: 0.9418\n", 152 | "Epoch 7/10\n", 153 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.2156 - accuracy: 0.9395 - val_loss: 0.1987 - val_accuracy: 0.9484\n", 154 | "Epoch 8/10\n", 155 | "55000/55000 [==============================] - 2s 36us/sample - loss: 0.2019 - accuracy: 0.9434 - val_loss: 0.1893 - val_accuracy: 0.9496\n", 156 | "Epoch 9/10\n", 157 | "55000/55000 [==============================] - 2s 41us/sample - loss: 0.1898 - accuracy: 0.9471 - val_loss: 0.1765 - val_accuracy: 0.9526\n", 158 | "Epoch 10/10\n", 159 | "55000/55000 [==============================] - 2s 39us/sample - loss: 0.1791 - accuracy: 0.9495 - val_loss: 0.1691 - val_accuracy: 0.9550\n" 160 | ] 161 | }, 162 | { 163 | "data": { 164 | "text/plain": [ 165 | "" 166 | ] 167 | }, 168 | "execution_count": 3, 169 | "metadata": {}, 170 | "output_type": "execute_result" 171 | } 172 | ], 173 | "source": [ 174 | "np.random.seed(42)\n", 175 | "tf.random.set_seed(42)\n", 176 | "\n", 177 | "model = keras.models.Sequential([\n", 178 | " keras.layers.Flatten(input_shape=[28, 28, 1]),\n", 179 | " keras.layers.Dense(100, activation=\"relu\"),\n", 180 | " keras.layers.Dense(10, activation=\"softmax\")\n", 181 | "])\n", 182 | "model.compile(loss=\"sparse_categorical_crossentropy\",\n", 183 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 184 | " metrics=[\"accuracy\"])\n", 185 | "model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 4, 191 | "metadata": {}, 192 | "outputs": [ 193 | { 194 | "data": { 195 | "text/plain": [ 196 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 197 | " [0. , 0. , 0.99, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 198 | " [0. , 0.96, 0.01, 0. , 0. , 0. , 0. , 0.01, 0.01, 0. ]],\n", 199 | " dtype=float32)" 200 | ] 201 | }, 202 | "execution_count": 4, 203 | "metadata": {}, 204 | "output_type": "execute_result" 205 | } 206 | ], 207 | "source": [ 208 | "np.round(model.predict(X_new), 2)" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 5, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "data": { 218 | "text/plain": [ 219 | "'my_mnist_model/0001'" 220 | ] 221 | }, 222 | "execution_count": 5, 223 | "metadata": {}, 224 | "output_type": "execute_result" 225 | } 226 | ], 227 | "source": [ 228 | "model_version = \"0001\"\n", 229 | "model_name = \"my_mnist_model\"\n", 230 | "model_path = os.path.join(model_name, model_version)\n", 231 | "model_path" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 6, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "!rm -rf {model_name}" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": 7, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "tf.saved_model.save(model, model_path)" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 8, 255 | "metadata": {}, 256 | "outputs": [ 257 | { 258 | "name": "stdout", 259 | "output_type": "stream", 260 | "text": [ 261 | "my_mnist_model/\n", 262 | " 0001/\n", 263 | " saved_model.pb\n", 264 | " variables/\n", 265 | " variables.data-00000-of-00001\n", 266 | " variables.index\n", 267 | " assets/\n" 268 | ] 269 | } 270 | ], 271 | "source": [ 272 | "for root, dirs, files in os.walk(model_name):\n", 273 | " indent = ' ' * root.count(os.sep)\n", 274 | " print('{}{}/'.format(indent, os.path.basename(root)))\n", 275 | " for filename in files:\n", 276 | " print('{}{}'.format(indent + ' ', filename))" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 9, 282 | "metadata": {}, 283 | "outputs": [ 284 | { 285 | "name": "stdout", 286 | "output_type": "stream", 287 | "text": [ 288 | "The given SavedModel contains the following tag-sets:\n", 289 | "serve\n" 290 | ] 291 | } 292 | ], 293 | "source": [ 294 | "!saved_model_cli show --dir {model_path}" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 10, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "The given SavedModel MetaGraphDef contains SignatureDefs with the following keys:\n", 307 | "SignatureDef key: \"__saved_model_init_op\"\n", 308 | "SignatureDef key: \"serving_default\"\n" 309 | ] 310 | } 311 | ], 312 | "source": [ 313 | "!saved_model_cli show --dir {model_path} --tag_set serve" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 11, 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "name": "stdout", 323 | "output_type": "stream", 324 | "text": [ 325 | "The given SavedModel SignatureDef contains the following input(s):\n", 326 | " inputs['flatten_2_input'] tensor_info:\n", 327 | " dtype: DT_FLOAT\n", 328 | " shape: (-1, 28, 28, 1)\n", 329 | " name: serving_default_flatten_2_input:0\n", 330 | "The given SavedModel SignatureDef contains the following output(s):\n", 331 | " outputs['dense_5'] tensor_info:\n", 332 | " dtype: DT_FLOAT\n", 333 | " shape: (-1, 10)\n", 334 | " name: StatefulPartitionedCall:0\n", 335 | "Method name is: tensorflow/serving/predict\n" 336 | ] 337 | } 338 | ], 339 | "source": [ 340 | "!saved_model_cli show --dir {model_path} --tag_set serve \\\n", 341 | " --signature_def serving_default" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 12, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "name": "stdout", 351 | "output_type": "stream", 352 | "text": [ 353 | "\n", 354 | "MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:\n", 355 | "\n", 356 | "signature_def['__saved_model_init_op']:\n", 357 | " The given SavedModel SignatureDef contains the following input(s):\n", 358 | " The given SavedModel SignatureDef contains the following output(s):\n", 359 | " outputs['__saved_model_init_op'] tensor_info:\n", 360 | " dtype: DT_INVALID\n", 361 | " shape: unknown_rank\n", 362 | " name: NoOp\n", 363 | " Method name is: \n", 364 | "\n", 365 | "signature_def['serving_default']:\n", 366 | " The given SavedModel SignatureDef contains the following input(s):\n", 367 | " inputs['flatten_2_input'] tensor_info:\n", 368 | " dtype: DT_FLOAT\n", 369 | " shape: (-1, 28, 28, 1)\n", 370 | " name: serving_default_flatten_2_input:0\n", 371 | " The given SavedModel SignatureDef contains the following output(s):\n", 372 | " outputs['dense_5'] tensor_info:\n", 373 | " dtype: DT_FLOAT\n", 374 | " shape: (-1, 10)\n", 375 | " name: StatefulPartitionedCall:0\n", 376 | " Method name is: tensorflow/serving/predict\n" 377 | ] 378 | } 379 | ], 380 | "source": [ 381 | "!saved_model_cli show --dir {model_path} --all" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": {}, 387 | "source": [ 388 | "Let's write the new instances to a `npy` file so we can pass them easily to our model:" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": 13, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "np.save(\"my_mnist_tests.npy\", X_new)" 398 | ] 399 | }, 400 | { 401 | "cell_type": "code", 402 | "execution_count": 14, 403 | "metadata": {}, 404 | "outputs": [ 405 | { 406 | "data": { 407 | "text/plain": [ 408 | "'flatten_2_input'" 409 | ] 410 | }, 411 | "execution_count": 14, 412 | "metadata": {}, 413 | "output_type": "execute_result" 414 | } 415 | ], 416 | "source": [ 417 | "input_name = model.input_names[0]\n", 418 | "input_name" 419 | ] 420 | }, 421 | { 422 | "cell_type": "markdown", 423 | "metadata": {}, 424 | "source": [ 425 | "And now let's use `saved_model_cli` to make predictions for the instances we just saved:" 426 | ] 427 | }, 428 | { 429 | "cell_type": "code", 430 | "execution_count": 15, 431 | "metadata": {}, 432 | "outputs": [ 433 | { 434 | "name": "stdout", 435 | "output_type": "stream", 436 | "text": [ 437 | "2019-06-10 10:56:43.396851: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", 438 | "WARNING: Logging before flag parsing goes to stderr.\n", 439 | "W0610 10:56:43.397369 140735810999168 deprecation.py:323] From /Users/ageron/miniconda3/envs/tf2/lib/python3.6/site-packages/tensorflow/python/tools/saved_model_cli.py:339: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n", 440 | "Instructions for updating:\n", 441 | "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n", 442 | "W0610 10:56:43.421489 140735810999168 deprecation.py:323] From /Users/ageron/miniconda3/envs/tf2/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", 443 | "Instructions for updating:\n", 444 | "Use standard file APIs to check for files with this prefix.\n", 445 | "Result for output key dense_5:\n", 446 | "[[1.17575204e-04 1.13160660e-07 5.96997386e-04 2.08104262e-03\n", 447 | " 2.57820852e-06 6.44166794e-05 2.77263990e-08 9.96703804e-01\n", 448 | " 3.96052455e-05 3.93810158e-04]\n", 449 | " [1.22226949e-03 2.92685600e-05 9.86054957e-01 9.63000767e-03\n", 450 | " 8.81790996e-08 2.88744748e-04 1.58111588e-03 1.12290488e-09\n", 451 | " 1.19344448e-03 1.09315742e-07]\n", 452 | " [6.40679718e-05 9.63618696e-01 9.04400647e-03 2.98595289e-03\n", 453 | " 5.95759891e-04 3.74212675e-03 2.50709383e-03 1.14931818e-02\n", 454 | " 5.52693009e-03 4.22279176e-04]]\n" 455 | ] 456 | } 457 | ], 458 | "source": [ 459 | "!saved_model_cli run --dir {model_path} --tag_set serve \\\n", 460 | " --signature_def serving_default \\\n", 461 | " --inputs {input_name}=my_mnist_tests.npy" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": 16, 467 | "metadata": {}, 468 | "outputs": [ 469 | { 470 | "data": { 471 | "text/plain": [ 472 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 473 | " [0. , 0. , 0.99, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 474 | " [0. , 0.96, 0.01, 0. , 0. , 0. , 0. , 0.01, 0.01, 0. ]])" 475 | ] 476 | }, 477 | "execution_count": 16, 478 | "metadata": {}, 479 | "output_type": "execute_result" 480 | } 481 | ], 482 | "source": [ 483 | "np.round([[1.1739199e-04, 1.1239604e-07, 6.0210604e-04, 2.0804715e-03, 2.5779348e-06,\n", 484 | " 6.4079795e-05, 2.7411186e-08, 9.9669880e-01, 3.9654213e-05, 3.9471846e-04],\n", 485 | " [1.2294615e-03, 2.9207937e-05, 9.8599273e-01, 9.6755642e-03, 8.8930705e-08,\n", 486 | " 2.9156188e-04, 1.5831805e-03, 1.1311053e-09, 1.1980456e-03, 1.1113169e-07],\n", 487 | " [6.4066830e-05, 9.6359509e-01, 9.0598064e-03, 2.9872139e-03, 5.9552520e-04,\n", 488 | " 3.7478798e-03, 2.5074568e-03, 1.1462728e-02, 5.5553433e-03, 4.2495009e-04]], 2)" 489 | ] 490 | }, 491 | { 492 | "cell_type": "markdown", 493 | "metadata": {}, 494 | "source": [ 495 | "## TensorFlow Serving" 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": {}, 501 | "source": [ 502 | "Install [Docker](https://docs.docker.com/install/) if you don't have it already. Then run:\n", 503 | "\n", 504 | "```bash\n", 505 | "docker pull tensorflow/serving\n", 506 | "\n", 507 | "export ML_PATH=$HOME/ml # or wherever this project is\n", 508 | "docker run -it --rm -p 8500:8500 -p 8501:8501 \\\n", 509 | " -v \"$ML_PATH/my_mnist_model:/models/my_mnist_model\" \\\n", 510 | " -e MODEL_NAME=my_mnist_model \\\n", 511 | " tensorflow/serving\n", 512 | "```\n", 513 | "Once you are finished using it, press Ctrl-C to shut down the server." 514 | ] 515 | }, 516 | { 517 | "cell_type": "markdown", 518 | "metadata": {}, 519 | "source": [ 520 | "Alternatively, if `tensorflow_model_server` is installed (e.g., if you are running this notebook in Colab), then the following 3 cells will start the server:" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": 17, 526 | "metadata": {}, 527 | "outputs": [], 528 | "source": [ 529 | "os.environ[\"MODEL_DIR\"] = os.path.split(os.path.abspath(model_path))[0]" 530 | ] 531 | }, 532 | { 533 | "cell_type": "code", 534 | "execution_count": 18, 535 | "metadata": {}, 536 | "outputs": [], 537 | "source": [ 538 | "%%bash --bg\n", 539 | "nohup tensorflow_model_server \\\n", 540 | " --rest_api_port=8501 \\\n", 541 | " --model_name=my_mnist_model \\\n", 542 | " --model_base_path=\"${MODEL_DIR}\" >server.log 2>&1" 543 | ] 544 | }, 545 | { 546 | "cell_type": "code", 547 | "execution_count": 19, 548 | "metadata": {}, 549 | "outputs": [ 550 | { 551 | "name": "stdout", 552 | "output_type": "stream", 553 | "text": [ 554 | "2019-11-06 13:04:12.267136: I external/org_tensorflow/tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", 555 | "2019-11-06 13:04:12.283035: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:202] Restoring SavedModel bundle.\n", 556 | "2019-11-06 13:04:12.300096: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:151] Running initialization op on SavedModel bundle at path: /content/my_mnist_model/0002\n", 557 | "2019-11-06 13:04:12.304438: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:311] SavedModel load for tags { serve }; Status: success. Took 39993 microseconds.\n", 558 | "2019-11-06 13:04:12.304900: I tensorflow_serving/servables/tensorflow/saved_model_warmup.cc:105] No warmup data file found at /content/my_mnist_model/0002/assets.extra/tf_serving_warmup_requests\n", 559 | "2019-11-06 13:04:12.305057: I tensorflow_serving/core/loader_harness.cc:87] Successfully loaded servable version {name: my_mnist_model version: 2}\n", 560 | "2019-11-06 13:04:12.306462: I tensorflow_serving/model_servers/server.cc:353] Running gRPC ModelServer at 0.0.0.0:8500 ...\n", 561 | "[warn] getaddrinfo: address family for nodename not supported\n", 562 | "2019-11-06 13:04:12.307179: I tensorflow_serving/model_servers/server.cc:373] Exporting HTTP/REST API at:localhost:8501 ...\n", 563 | "[evhttp_server.cc : 238] NET_LOG: Entering the event loop ...\n" 564 | ] 565 | } 566 | ], 567 | "source": [ 568 | "!tail server.log" 569 | ] 570 | }, 571 | { 572 | "cell_type": "code", 573 | "execution_count": 20, 574 | "metadata": {}, 575 | "outputs": [], 576 | "source": [ 577 | "import json\n", 578 | "\n", 579 | "input_data_json = json.dumps({\n", 580 | " \"signature_name\": \"serving_default\",\n", 581 | " \"instances\": X_new.tolist(),\n", 582 | "})" 583 | ] 584 | }, 585 | { 586 | "cell_type": "code", 587 | "execution_count": 21, 588 | "metadata": {}, 589 | "outputs": [ 590 | { 591 | "data": { 592 | "text/plain": [ 593 | "'\\'{\"signature_name\": \"serving_default\", \"instances\": [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3294117748737335, 0.7254902124404907, 0.6235294342041016, 0.5921568870544434, 0.23529411852359772, 0.1411764770746231, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8705882430076599, 0.9960784316062927, 0.9960784316062927, 0.9960784316062927, 0.9960784316062927, 0.9450980424880981, 0.7764706015586853, 0.7764706015586853, 0.7764706015586853, 0.776470...'" 594 | ] 595 | }, 596 | "execution_count": 21, 597 | "metadata": {}, 598 | "output_type": "execute_result" 599 | } 600 | ], 601 | "source": [ 602 | "repr(input_data_json)[:1500] + \"...\"" 603 | ] 604 | }, 605 | { 606 | "cell_type": "markdown", 607 | "metadata": {}, 608 | "source": [ 609 | "Now let's use TensorFlow Serving's REST API to make predictions:" 610 | ] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "execution_count": 22, 615 | "metadata": {}, 616 | "outputs": [], 617 | "source": [ 618 | "import requests\n", 619 | "\n", 620 | "SERVER_URL = 'http://localhost:8501/v1/models/my_mnist_model:predict'\n", 621 | "response = requests.post(SERVER_URL, data=input_data_json)\n", 622 | "response.raise_for_status() # raise an exception in case of error\n", 623 | "response = response.json()" 624 | ] 625 | }, 626 | { 627 | "cell_type": "code", 628 | "execution_count": 23, 629 | "metadata": {}, 630 | "outputs": [ 631 | { 632 | "data": { 633 | "text/plain": [ 634 | "dict_keys(['predictions'])" 635 | ] 636 | }, 637 | "execution_count": 23, 638 | "metadata": {}, 639 | "output_type": "execute_result" 640 | } 641 | ], 642 | "source": [ 643 | "response.keys()" 644 | ] 645 | }, 646 | { 647 | "cell_type": "code", 648 | "execution_count": 24, 649 | "metadata": {}, 650 | "outputs": [ 651 | { 652 | "data": { 653 | "text/plain": [ 654 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 655 | " [0. , 0. , 0.99, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 656 | " [0. , 0.96, 0.01, 0. , 0. , 0. , 0. , 0.01, 0.01, 0. ]])" 657 | ] 658 | }, 659 | "execution_count": 24, 660 | "metadata": {}, 661 | "output_type": "execute_result" 662 | } 663 | ], 664 | "source": [ 665 | "y_proba = np.array(response[\"predictions\"])\n", 666 | "y_proba.round(2)" 667 | ] 668 | }, 669 | { 670 | "cell_type": "markdown", 671 | "metadata": {}, 672 | "source": [ 673 | "### Using the gRPC API" 674 | ] 675 | }, 676 | { 677 | "cell_type": "code", 678 | "execution_count": 25, 679 | "metadata": {}, 680 | "outputs": [], 681 | "source": [ 682 | "from tensorflow_serving.apis.predict_pb2 import PredictRequest\n", 683 | "\n", 684 | "request = PredictRequest()\n", 685 | "request.model_spec.name = model_name\n", 686 | "request.model_spec.signature_name = \"serving_default\"\n", 687 | "input_name = model.input_names[0]\n", 688 | "request.inputs[input_name].CopyFrom(tf.make_tensor_proto(X_new))" 689 | ] 690 | }, 691 | { 692 | "cell_type": "code", 693 | "execution_count": 26, 694 | "metadata": {}, 695 | "outputs": [], 696 | "source": [ 697 | "import grpc\n", 698 | "from tensorflow_serving.apis import prediction_service_pb2_grpc\n", 699 | "\n", 700 | "channel = grpc.insecure_channel('localhost:8500')\n", 701 | "predict_service = prediction_service_pb2_grpc.PredictionServiceStub(channel)\n", 702 | "response = predict_service.Predict(request, timeout=10.0)" 703 | ] 704 | }, 705 | { 706 | "cell_type": "code", 707 | "execution_count": 27, 708 | "metadata": {}, 709 | "outputs": [ 710 | { 711 | "data": { 712 | "text/plain": [ 713 | "outputs {\n", 714 | " key: \"dense_4\"\n", 715 | " value {\n", 716 | " dtype: DT_FLOAT\n", 717 | " tensor_shape {\n", 718 | " dim {\n", 719 | " size: 3\n", 720 | " }\n", 721 | " dim {\n", 722 | " size: 10\n", 723 | " }\n", 724 | " }\n", 725 | " float_val: 2.0824443708988838e-05\n", 726 | " float_val: 1.4913139168015732e-08\n", 727 | " float_val: 0.0004813199338968843\n", 728 | " float_val: 0.001888890634290874\n", 729 | " float_val: 2.682592992186983e-07\n", 730 | " float_val: 8.666840585647151e-06\n", 731 | " float_val: 1.6853943241024183e-10\n", 732 | " float_val: 0.9975269436836243\n", 733 | " float_val: 3.833709342870861e-05\n", 734 | " float_val: 3.4738284739432856e-05\n", 735 | " float_val: 0.00017358684272039682\n", 736 | " float_val: 0.0002858016814570874\n", 737 | " float_val: 0.9816810488700867\n", 738 | " float_val: 0.0157401692122221\n", 739 | " float_val: 1.1949770339914068e-10\n", 740 | " float_val: 0.00023017563216853887\n", 741 | " float_val: 3.078056761296466e-05\n", 742 | " float_val: 5.393230750883049e-09\n", 743 | " float_val: 0.0018584482604637742\n", 744 | " float_val: 1.8884094288296183e-09\n", 745 | " float_val: 3.397366526769474e-05\n", 746 | " float_val: 0.9835277795791626\n", 747 | " float_val: 0.001533020636998117\n", 748 | " float_val: 0.0014515116345137358\n", 749 | " float_val: 0.00018795969663187861\n", 750 | " float_val: 0.0011680654715746641\n", 751 | " float_val: 0.0014667459763586521\n", 752 | " float_val: 0.006120447069406509\n", 753 | " float_val: 0.004315734840929508\n", 754 | " float_val: 0.00019466254161670804\n", 755 | " }\n", 756 | "}\n", 757 | "model_spec {\n", 758 | " name: \"my_mnist_model\"\n", 759 | " version {\n", 760 | " value: 2\n", 761 | " }\n", 762 | " signature_name: \"serving_default\"\n", 763 | "}" 764 | ] 765 | }, 766 | "execution_count": 27, 767 | "metadata": {}, 768 | "output_type": "execute_result" 769 | } 770 | ], 771 | "source": [ 772 | "response" 773 | ] 774 | }, 775 | { 776 | "cell_type": "markdown", 777 | "metadata": {}, 778 | "source": [ 779 | "Convert the response to a tensor:" 780 | ] 781 | }, 782 | { 783 | "cell_type": "code", 784 | "execution_count": 28, 785 | "metadata": { 786 | "scrolled": true 787 | }, 788 | "outputs": [ 789 | { 790 | "data": { 791 | "text/plain": [ 792 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 793 | " [0. , 0. , 0.98, 0.02, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 794 | " [0. , 0.98, 0. , 0. , 0. , 0. , 0. , 0.01, 0. , 0. ]],\n", 795 | " dtype=float32)" 796 | ] 797 | }, 798 | "execution_count": 28, 799 | "metadata": {}, 800 | "output_type": "execute_result" 801 | } 802 | ], 803 | "source": [ 804 | "output_name = model.output_names[0]\n", 805 | "outputs_proto = response.outputs[output_name]\n", 806 | "y_proba = tf.make_ndarray(outputs_proto)\n", 807 | "y_proba.round(2)" 808 | ] 809 | }, 810 | { 811 | "cell_type": "markdown", 812 | "metadata": {}, 813 | "source": [ 814 | "Or to a NumPy array if your client does not include the TensorFlow library:" 815 | ] 816 | }, 817 | { 818 | "cell_type": "code", 819 | "execution_count": 29, 820 | "metadata": {}, 821 | "outputs": [ 822 | { 823 | "data": { 824 | "text/plain": [ 825 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 826 | " [0. , 0. , 0.98, 0.02, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 827 | " [0. , 0.98, 0. , 0. , 0. , 0. , 0. , 0.01, 0. , 0. ]])" 828 | ] 829 | }, 830 | "execution_count": 29, 831 | "metadata": {}, 832 | "output_type": "execute_result" 833 | } 834 | ], 835 | "source": [ 836 | "output_name = model.output_names[0]\n", 837 | "outputs_proto = response.outputs[output_name]\n", 838 | "shape = [dim.size for dim in outputs_proto.tensor_shape.dim]\n", 839 | "y_proba = np.array(outputs_proto.float_val).reshape(shape)\n", 840 | "y_proba.round(2)" 841 | ] 842 | }, 843 | { 844 | "cell_type": "markdown", 845 | "metadata": {}, 846 | "source": [ 847 | "## Deploying a new model version" 848 | ] 849 | }, 850 | { 851 | "cell_type": "code", 852 | "execution_count": 30, 853 | "metadata": { 854 | "scrolled": true 855 | }, 856 | "outputs": [ 857 | { 858 | "name": "stdout", 859 | "output_type": "stream", 860 | "text": [ 861 | "Train on 55000 samples, validate on 5000 samples\n", 862 | "Epoch 1/10\n", 863 | "55000/55000 [==============================] - 2s 39us/sample - loss: 0.7035 - accuracy: 0.8060 - val_loss: 0.3445 - val_accuracy: 0.9032\n", 864 | "Epoch 2/10\n", 865 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.3213 - accuracy: 0.9084 - val_loss: 0.2660 - val_accuracy: 0.9252\n", 866 | "Epoch 3/10\n", 867 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.2663 - accuracy: 0.9236 - val_loss: 0.2304 - val_accuracy: 0.9392\n", 868 | "Epoch 4/10\n", 869 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.2331 - accuracy: 0.9331 - val_loss: 0.2069 - val_accuracy: 0.9430\n", 870 | "Epoch 5/10\n", 871 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.2105 - accuracy: 0.9390 - val_loss: 0.1910 - val_accuracy: 0.9446\n", 872 | "Epoch 6/10\n", 873 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.1924 - accuracy: 0.9442 - val_loss: 0.1732 - val_accuracy: 0.9518\n", 874 | "Epoch 7/10\n", 875 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.1771 - accuracy: 0.9489 - val_loss: 0.1679 - val_accuracy: 0.9526\n", 876 | "Epoch 8/10\n", 877 | "55000/55000 [==============================] - 2s 37us/sample - loss: 0.1650 - accuracy: 0.9527 - val_loss: 0.1574 - val_accuracy: 0.9546\n", 878 | "Epoch 9/10\n", 879 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.1540 - accuracy: 0.9555 - val_loss: 0.1446 - val_accuracy: 0.9590\n", 880 | "Epoch 10/10\n", 881 | "55000/55000 [==============================] - 2s 35us/sample - loss: 0.1448 - accuracy: 0.9583 - val_loss: 0.1414 - val_accuracy: 0.9608\n" 882 | ] 883 | }, 884 | { 885 | "data": { 886 | "text/plain": [ 887 | "" 888 | ] 889 | }, 890 | "execution_count": 30, 891 | "metadata": {}, 892 | "output_type": "execute_result" 893 | } 894 | ], 895 | "source": [ 896 | "np.random.seed(42)\n", 897 | "tf.random.set_seed(42)\n", 898 | "\n", 899 | "model = keras.models.Sequential([\n", 900 | " keras.layers.Flatten(input_shape=[28, 28, 1]),\n", 901 | " keras.layers.Dense(50, activation=\"relu\"),\n", 902 | " keras.layers.Dense(50, activation=\"relu\"),\n", 903 | " keras.layers.Dense(10, activation=\"softmax\")\n", 904 | "])\n", 905 | "model.compile(loss=\"sparse_categorical_crossentropy\",\n", 906 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 907 | " metrics=[\"accuracy\"])\n", 908 | "history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))" 909 | ] 910 | }, 911 | { 912 | "cell_type": "code", 913 | "execution_count": 31, 914 | "metadata": {}, 915 | "outputs": [ 916 | { 917 | "data": { 918 | "text/plain": [ 919 | "'my_mnist_model/0002'" 920 | ] 921 | }, 922 | "execution_count": 31, 923 | "metadata": {}, 924 | "output_type": "execute_result" 925 | } 926 | ], 927 | "source": [ 928 | "model_version = \"0002\"\n", 929 | "model_name = \"my_mnist_model\"\n", 930 | "model_path = os.path.join(model_name, model_version)\n", 931 | "model_path" 932 | ] 933 | }, 934 | { 935 | "cell_type": "code", 936 | "execution_count": 32, 937 | "metadata": {}, 938 | "outputs": [], 939 | "source": [ 940 | "tf.saved_model.save(model, model_path)" 941 | ] 942 | }, 943 | { 944 | "cell_type": "code", 945 | "execution_count": 33, 946 | "metadata": {}, 947 | "outputs": [ 948 | { 949 | "name": "stdout", 950 | "output_type": "stream", 951 | "text": [ 952 | "my_mnist_model/\n", 953 | " 0002/\n", 954 | " saved_model.pb\n", 955 | " variables/\n", 956 | " variables.data-00000-of-00001\n", 957 | " variables.index\n", 958 | " assets/\n", 959 | " 0001/\n", 960 | " saved_model.pb\n", 961 | " variables/\n", 962 | " variables.data-00000-of-00001\n", 963 | " variables.index\n", 964 | " assets/\n" 965 | ] 966 | } 967 | ], 968 | "source": [ 969 | "for root, dirs, files in os.walk(model_name):\n", 970 | " indent = ' ' * root.count(os.sep)\n", 971 | " print('{}{}/'.format(indent, os.path.basename(root)))\n", 972 | " for filename in files:\n", 973 | " print('{}{}'.format(indent + ' ', filename))" 974 | ] 975 | }, 976 | { 977 | "cell_type": "markdown", 978 | "metadata": {}, 979 | "source": [ 980 | "**Warning**: You may need to wait a minute before the new model is loaded by TensorFlow Serving." 981 | ] 982 | }, 983 | { 984 | "cell_type": "code", 985 | "execution_count": 34, 986 | "metadata": {}, 987 | "outputs": [], 988 | "source": [ 989 | "import requests\n", 990 | "\n", 991 | "SERVER_URL = 'http://localhost:8501/v1/models/my_mnist_model:predict'\n", 992 | " \n", 993 | "response = requests.post(SERVER_URL, data=input_data_json)\n", 994 | "response.raise_for_status()\n", 995 | "response = response.json()" 996 | ] 997 | }, 998 | { 999 | "cell_type": "code", 1000 | "execution_count": 35, 1001 | "metadata": {}, 1002 | "outputs": [ 1003 | { 1004 | "data": { 1005 | "text/plain": [ 1006 | "dict_keys(['predictions'])" 1007 | ] 1008 | }, 1009 | "execution_count": 35, 1010 | "metadata": {}, 1011 | "output_type": "execute_result" 1012 | } 1013 | ], 1014 | "source": [ 1015 | "response.keys()" 1016 | ] 1017 | }, 1018 | { 1019 | "cell_type": "code", 1020 | "execution_count": 36, 1021 | "metadata": {}, 1022 | "outputs": [ 1023 | { 1024 | "data": { 1025 | "text/plain": [ 1026 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 1027 | " [0. , 0. , 0.99, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 1028 | " [0. , 0.96, 0.01, 0. , 0. , 0. , 0. , 0.01, 0.01, 0. ]])" 1029 | ] 1030 | }, 1031 | "execution_count": 36, 1032 | "metadata": {}, 1033 | "output_type": "execute_result" 1034 | } 1035 | ], 1036 | "source": [ 1037 | "y_proba = np.array(response[\"predictions\"])\n", 1038 | "y_proba.round(2)" 1039 | ] 1040 | }, 1041 | { 1042 | "cell_type": "markdown", 1043 | "metadata": {}, 1044 | "source": [ 1045 | "# Deploy the model to Google Cloud AI Platform" 1046 | ] 1047 | }, 1048 | { 1049 | "cell_type": "markdown", 1050 | "metadata": {}, 1051 | "source": [ 1052 | "Follow the instructions in the book to deploy the model to Google Cloud AI Platform, download the service account's private key and save it to the `my_service_account_private_key.json` in the project directory. Also, update the `project_id`:" 1053 | ] 1054 | }, 1055 | { 1056 | "cell_type": "code", 1057 | "execution_count": 37, 1058 | "metadata": {}, 1059 | "outputs": [], 1060 | "source": [ 1061 | "project_id = \"onyx-smoke-242003\"" 1062 | ] 1063 | }, 1064 | { 1065 | "cell_type": "code", 1066 | "execution_count": 38, 1067 | "metadata": {}, 1068 | "outputs": [], 1069 | "source": [ 1070 | "import googleapiclient.discovery\n", 1071 | "\n", 1072 | "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"my_service_account_private_key.json\"\n", 1073 | "model_id = \"my_mnist_model\"\n", 1074 | "model_path = \"projects/{}/models/{}\".format(project_id, model_id)\n", 1075 | "model_path += \"/versions/v0001/\" # if you want to run a specific version\n", 1076 | "ml_resource = googleapiclient.discovery.build(\"ml\", \"v1\").projects()" 1077 | ] 1078 | }, 1079 | { 1080 | "cell_type": "code", 1081 | "execution_count": 39, 1082 | "metadata": {}, 1083 | "outputs": [], 1084 | "source": [ 1085 | "def predict(X):\n", 1086 | " input_data_json = {\"signature_name\": \"serving_default\",\n", 1087 | " \"instances\": X.tolist()}\n", 1088 | " request = ml_resource.predict(name=model_path, body=input_data_json)\n", 1089 | " response = request.execute()\n", 1090 | " if \"error\" in response:\n", 1091 | " raise RuntimeError(response[\"error\"])\n", 1092 | " return np.array([pred[output_name] for pred in response[\"predictions\"]])" 1093 | ] 1094 | }, 1095 | { 1096 | "cell_type": "code", 1097 | "execution_count": 40, 1098 | "metadata": {}, 1099 | "outputs": [ 1100 | { 1101 | "data": { 1102 | "text/plain": [ 1103 | "array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. , 0. ],\n", 1104 | " [0. , 0. , 0.99, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n", 1105 | " [0. , 0.96, 0.01, 0. , 0. , 0. , 0. , 0.01, 0.01, 0. ]])" 1106 | ] 1107 | }, 1108 | "execution_count": 40, 1109 | "metadata": {}, 1110 | "output_type": "execute_result" 1111 | } 1112 | ], 1113 | "source": [ 1114 | "Y_probas = predict(X_new)\n", 1115 | "np.round(Y_probas, 2)" 1116 | ] 1117 | }, 1118 | { 1119 | "cell_type": "markdown", 1120 | "metadata": {}, 1121 | "source": [ 1122 | "# Using GPUs" 1123 | ] 1124 | }, 1125 | { 1126 | "cell_type": "code", 1127 | "execution_count": 41, 1128 | "metadata": {}, 1129 | "outputs": [ 1130 | { 1131 | "data": { 1132 | "text/plain": [ 1133 | "False" 1134 | ] 1135 | }, 1136 | "execution_count": 41, 1137 | "metadata": {}, 1138 | "output_type": "execute_result" 1139 | } 1140 | ], 1141 | "source": [ 1142 | "tf.test.is_gpu_available()" 1143 | ] 1144 | }, 1145 | { 1146 | "cell_type": "code", 1147 | "execution_count": 42, 1148 | "metadata": {}, 1149 | "outputs": [ 1150 | { 1151 | "data": { 1152 | "text/plain": [ 1153 | "''" 1154 | ] 1155 | }, 1156 | "execution_count": 42, 1157 | "metadata": {}, 1158 | "output_type": "execute_result" 1159 | } 1160 | ], 1161 | "source": [ 1162 | "tf.test.gpu_device_name()" 1163 | ] 1164 | }, 1165 | { 1166 | "cell_type": "code", 1167 | "execution_count": 43, 1168 | "metadata": {}, 1169 | "outputs": [ 1170 | { 1171 | "data": { 1172 | "text/plain": [ 1173 | "False" 1174 | ] 1175 | }, 1176 | "execution_count": 43, 1177 | "metadata": {}, 1178 | "output_type": "execute_result" 1179 | } 1180 | ], 1181 | "source": [ 1182 | "tf.test.is_built_with_cuda()" 1183 | ] 1184 | }, 1185 | { 1186 | "cell_type": "code", 1187 | "execution_count": 44, 1188 | "metadata": {}, 1189 | "outputs": [ 1190 | { 1191 | "data": { 1192 | "text/plain": [ 1193 | "[name: \"/device:CPU:0\"\n", 1194 | " device_type: \"CPU\"\n", 1195 | " memory_limit: 268435456\n", 1196 | " locality {\n", 1197 | " }\n", 1198 | " incarnation: 11178133101787456811]" 1199 | ] 1200 | }, 1201 | "execution_count": 44, 1202 | "metadata": {}, 1203 | "output_type": "execute_result" 1204 | } 1205 | ], 1206 | "source": [ 1207 | "from tensorflow.python.client.device_lib import list_local_devices\n", 1208 | "\n", 1209 | "devices = list_local_devices()\n", 1210 | "devices" 1211 | ] 1212 | }, 1213 | { 1214 | "cell_type": "markdown", 1215 | "metadata": {}, 1216 | "source": [ 1217 | "# Distributed Training" 1218 | ] 1219 | }, 1220 | { 1221 | "cell_type": "code", 1222 | "execution_count": 45, 1223 | "metadata": {}, 1224 | "outputs": [], 1225 | "source": [ 1226 | "keras.backend.clear_session()\n", 1227 | "tf.random.set_seed(42)\n", 1228 | "np.random.seed(42)" 1229 | ] 1230 | }, 1231 | { 1232 | "cell_type": "code", 1233 | "execution_count": 46, 1234 | "metadata": {}, 1235 | "outputs": [], 1236 | "source": [ 1237 | "def create_model():\n", 1238 | " return keras.models.Sequential([\n", 1239 | " keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n", 1240 | " padding=\"same\", input_shape=[28, 28, 1]),\n", 1241 | " keras.layers.MaxPooling2D(pool_size=2),\n", 1242 | " keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", 1243 | " padding=\"same\"), \n", 1244 | " keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", 1245 | " padding=\"same\"),\n", 1246 | " keras.layers.MaxPooling2D(pool_size=2),\n", 1247 | " keras.layers.Flatten(),\n", 1248 | " keras.layers.Dense(units=64, activation='relu'),\n", 1249 | " keras.layers.Dropout(0.5),\n", 1250 | " keras.layers.Dense(units=10, activation='softmax'),\n", 1251 | " ])" 1252 | ] 1253 | }, 1254 | { 1255 | "cell_type": "code", 1256 | "execution_count": 47, 1257 | "metadata": {}, 1258 | "outputs": [], 1259 | "source": [ 1260 | "batch_size = 100\n", 1261 | "model = create_model()\n", 1262 | "model.compile(loss=\"sparse_categorical_crossentropy\",\n", 1263 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 1264 | " metrics=[\"accuracy\"])\n", 1265 | "model.fit(X_train, y_train, epochs=10,\n", 1266 | " validation_data=(X_valid, y_valid), batch_size=batch_size)" 1267 | ] 1268 | }, 1269 | { 1270 | "cell_type": "code", 1271 | "execution_count": 48, 1272 | "metadata": {}, 1273 | "outputs": [ 1274 | { 1275 | "name": "stderr", 1276 | "output_type": "stream", 1277 | "text": [ 1278 | "WARNING: Logging before flag parsing goes to stderr.\n", 1279 | "W0603 15:31:26.178871 140735810999168 cross_device_ops.py:1178] There is non-GPU devices in `tf.distribute.Strategy`, not using nccl allreduce.\n" 1280 | ] 1281 | } 1282 | ], 1283 | "source": [ 1284 | "keras.backend.clear_session()\n", 1285 | "tf.random.set_seed(42)\n", 1286 | "np.random.seed(42)\n", 1287 | "\n", 1288 | "distribution = tf.distribute.MirroredStrategy()\n", 1289 | "\n", 1290 | "# Change the default all-reduce algorithm:\n", 1291 | "#distribution = tf.distribute.MirroredStrategy(\n", 1292 | "# cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())\n", 1293 | "\n", 1294 | "# Specify the list of GPUs to use:\n", 1295 | "#distribution = tf.distribute.MirroredStrategy(devices=[\"/gpu:0\", \"/gpu:1\"])\n", 1296 | "\n", 1297 | "# Use the central storage strategy instead:\n", 1298 | "#distribution = tf.distribute.experimental.CentralStorageStrategy()\n", 1299 | "\n", 1300 | "#resolver = tf.distribute.cluster_resolver.TPUClusterResolver()\n", 1301 | "#tf.tpu.experimental.initialize_tpu_system(resolver)\n", 1302 | "#distribution = tf.distribute.experimental.TPUStrategy(resolver)\n", 1303 | "\n", 1304 | "with distribution.scope():\n", 1305 | " model = create_model()\n", 1306 | " model.compile(loss=\"sparse_categorical_crossentropy\",\n", 1307 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 1308 | " metrics=[\"accuracy\"])" 1309 | ] 1310 | }, 1311 | { 1312 | "cell_type": "code", 1313 | "execution_count": 49, 1314 | "metadata": {}, 1315 | "outputs": [], 1316 | "source": [ 1317 | "batch_size = 100 # must be divisible by the number of workers\n", 1318 | "model.fit(X_train, y_train, epochs=10,\n", 1319 | " validation_data=(X_valid, y_valid), batch_size=batch_size)" 1320 | ] 1321 | }, 1322 | { 1323 | "cell_type": "code", 1324 | "execution_count": 50, 1325 | "metadata": {}, 1326 | "outputs": [ 1327 | { 1328 | "data": { 1329 | "text/plain": [ 1330 | "array([[0.09101252, 0.07083996, 0.06410537, 0.11957529, 0.06693752,\n", 1331 | " 0.05124901, 0.04676544, 0.23180223, 0.13522181, 0.12249089],\n", 1332 | " [0.08099081, 0.12387844, 0.14915964, 0.13171668, 0.05875394,\n", 1333 | " 0.08834281, 0.16267018, 0.06899565, 0.07834874, 0.05714307],\n", 1334 | " [0.04303756, 0.2682051 , 0.0909673 , 0.11496522, 0.06084979,\n", 1335 | " 0.07125981, 0.08520001, 0.08517107, 0.09236596, 0.0879782 ]],\n", 1336 | " dtype=float32)" 1337 | ] 1338 | }, 1339 | "execution_count": 50, 1340 | "metadata": {}, 1341 | "output_type": "execute_result" 1342 | } 1343 | ], 1344 | "source": [ 1345 | "model.predict(X_new)" 1346 | ] 1347 | }, 1348 | { 1349 | "cell_type": "markdown", 1350 | "metadata": {}, 1351 | "source": [ 1352 | "Custom training loop:" 1353 | ] 1354 | }, 1355 | { 1356 | "cell_type": "code", 1357 | "execution_count": 51, 1358 | "metadata": {}, 1359 | "outputs": [], 1360 | "source": [ 1361 | "keras.backend.clear_session()\n", 1362 | "tf.random.set_seed(42)\n", 1363 | "np.random.seed(42)\n", 1364 | "\n", 1365 | "K = keras.backend\n", 1366 | "\n", 1367 | "distribution = tf.distribute.MirroredStrategy()\n", 1368 | "\n", 1369 | "with distribution.scope():\n", 1370 | " model = create_model()\n", 1371 | " optimizer = keras.optimizers.SGD()\n", 1372 | "\n", 1373 | "with distribution.scope():\n", 1374 | " dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)).repeat().batch(batch_size)\n", 1375 | " input_iterator = distribution.make_dataset_iterator(dataset)\n", 1376 | " \n", 1377 | "@tf.function\n", 1378 | "def train_step():\n", 1379 | " def step_fn(inputs):\n", 1380 | " X, y = inputs\n", 1381 | " with tf.GradientTape() as tape:\n", 1382 | " Y_proba = model(X)\n", 1383 | " loss = K.sum(keras.losses.sparse_categorical_crossentropy(y, Y_proba)) / batch_size\n", 1384 | "\n", 1385 | " grads = tape.gradient(loss, model.trainable_variables)\n", 1386 | " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", 1387 | " return loss\n", 1388 | "\n", 1389 | " per_replica_losses = distribution.experimental_run(step_fn, input_iterator)\n", 1390 | " mean_loss = distribution.reduce(tf.distribute.ReduceOp.SUM,\n", 1391 | " per_replica_losses, axis=None)\n", 1392 | " return mean_loss\n", 1393 | "\n", 1394 | "n_epochs = 10\n", 1395 | "with distribution.scope():\n", 1396 | " input_iterator.initialize()\n", 1397 | " for epoch in range(n_epochs):\n", 1398 | " print(\"Epoch {}/{}\".format(epoch + 1, n_epochs))\n", 1399 | " for iteration in range(len(X_train) // batch_size):\n", 1400 | " print(\"\\rLoss: {:.3f}\".format(train_step().numpy()), end=\"\")\n", 1401 | " print()" 1402 | ] 1403 | }, 1404 | { 1405 | "cell_type": "code", 1406 | "execution_count": 52, 1407 | "metadata": {}, 1408 | "outputs": [], 1409 | "source": [ 1410 | "batch_size = 100 # must be divisible by the number of workers\n", 1411 | "model.fit(X_train, y_train, epochs=10,\n", 1412 | " validation_data=(X_valid, y_valid), batch_size=batch_size)" 1413 | ] 1414 | }, 1415 | { 1416 | "cell_type": "markdown", 1417 | "metadata": {}, 1418 | "source": [ 1419 | "## Training across multiple servers" 1420 | ] 1421 | }, 1422 | { 1423 | "cell_type": "markdown", 1424 | "metadata": {}, 1425 | "source": [ 1426 | "A TensorFlow cluster is a group of TensorFlow processes running in parallel, usually on different machines, and talking to each other to complete some work, for example training or executing a neural network. Each TF process in the cluster is called a \"task\" (or a \"TF server\"). It has an IP address, a port, and a type (also called its role or its job). The type can be `\"worker\"`, `\"chief\"`, `\"ps\"` (parameter server) or `\"evaluator\"`:\n", 1427 | "* Each **worker** performs computations, usually on a machine with one or more GPUs.\n", 1428 | "* The **chief** performs computations as well, but it also handles extra work such as writing TensorBoard logs or saving checkpoints. There is a single chief in a cluster. If no chief is specified, then the first worker is the chief.\n", 1429 | "* A **parameter server** (ps) only keeps track of variable values, it is usually on a CPU-only machine.\n", 1430 | "* The **evaluator** obviously takes care of evaluation. There is usually a single evaluator in a cluster.\n", 1431 | "\n", 1432 | "The set of tasks that share the same type is often called a \"job\". For example, the \"worker\" job is the set of all workers.\n", 1433 | "\n", 1434 | "To start a TensorFlow cluster, you must first specify it. This means defining all the tasks (IP address, TCP port, and type). For example, the following cluster specification defines a cluster with 3 tasks (2 workers and 1 parameter server). It's a dictionary with one key per job, and the values are lists of task addresses:\n", 1435 | "\n", 1436 | "```\n", 1437 | "{\n", 1438 | " \"worker\": [\"my-worker0.example.com:9876\", \"my-worker1.example.com:9876\"],\n", 1439 | " \"ps\": [\"my-ps0.example.com:9876\"]\n", 1440 | "}\n", 1441 | "```\n", 1442 | "\n", 1443 | "Every task in the cluster may communicate with every other task in the server, so make sure to configure your firewall to authorize all communications between these machines on these ports (it's usually simpler if you use the same port on every machine).\n", 1444 | "\n", 1445 | "When a task is started, it needs to be told which one it is: its type and index (the task index is also called the task id). A common way to specify everything at once (both the cluster spec and the current task's type and id) is to set the `TF_CONFIG` environment variable before starting the program. It must be a JSON-encoded dictionary containing a cluster specification (under the `\"cluster\"` key), and the type and index of the task to start (under the `\"task\"` key). For example, the following `TF_CONFIG` environment variable defines a simple cluster with 2 workers and 1 parameter server, and specifies that the task to start is the first worker:" 1446 | ] 1447 | }, 1448 | { 1449 | "cell_type": "code", 1450 | "execution_count": 53, 1451 | "metadata": {}, 1452 | "outputs": [ 1453 | { 1454 | "name": "stdout", 1455 | "output_type": "stream", 1456 | "text": [ 1457 | "TF_CONFIG='{\"cluster\": {\"worker\": [\"my-work0.example.com:9876\", \"my-work1.example.com:9876\"], \"ps\": [\"my-ps0.example.com:9876\"]}, \"task\": {\"type\": \"worker\", \"index\": 0}}'\n" 1458 | ] 1459 | } 1460 | ], 1461 | "source": [ 1462 | "import os\n", 1463 | "import json\n", 1464 | "\n", 1465 | "os.environ[\"TF_CONFIG\"] = json.dumps({\n", 1466 | " \"cluster\": {\n", 1467 | " \"worker\": [\"my-work0.example.com:9876\", \"my-work1.example.com:9876\"],\n", 1468 | " \"ps\": [\"my-ps0.example.com:9876\"]\n", 1469 | " },\n", 1470 | " \"task\": {\"type\": \"worker\", \"index\": 0}\n", 1471 | "})\n", 1472 | "print(\"TF_CONFIG='{}'\".format(os.environ[\"TF_CONFIG\"]))" 1473 | ] 1474 | }, 1475 | { 1476 | "cell_type": "markdown", 1477 | "metadata": {}, 1478 | "source": [ 1479 | "Some platforms (e.g., Google Cloud ML Engine) automatically set this environment variable for you." 1480 | ] 1481 | }, 1482 | { 1483 | "cell_type": "markdown", 1484 | "metadata": {}, 1485 | "source": [ 1486 | "Then you would write a short Python script to start a task. The same script can be used on every machine, since it will load the `TF_CONFIG` variable, which will tell it which task to start:" 1487 | ] 1488 | }, 1489 | { 1490 | "cell_type": "code", 1491 | "execution_count": 54, 1492 | "metadata": {}, 1493 | "outputs": [], 1494 | "source": [ 1495 | "import tensorflow as tf\n", 1496 | "\n", 1497 | "resolver = tf.distribute.cluster_resolver.TFConfigClusterResolver()\n", 1498 | "worker0 = tf.distribute.Server(resolver.cluster_spec(),\n", 1499 | " job_name=resolver.task_type,\n", 1500 | " task_index=resolver.task_id)" 1501 | ] 1502 | }, 1503 | { 1504 | "cell_type": "markdown", 1505 | "metadata": {}, 1506 | "source": [ 1507 | "Another way to specify the cluster specification is directly in Python, rather than through an environment variable:" 1508 | ] 1509 | }, 1510 | { 1511 | "cell_type": "code", 1512 | "execution_count": 55, 1513 | "metadata": {}, 1514 | "outputs": [], 1515 | "source": [ 1516 | "cluster_spec = tf.train.ClusterSpec({\n", 1517 | " \"worker\": [\"127.0.0.1:9901\", \"127.0.0.1:9902\"],\n", 1518 | " \"ps\": [\"127.0.0.1:9903\"]\n", 1519 | "})" 1520 | ] 1521 | }, 1522 | { 1523 | "cell_type": "markdown", 1524 | "metadata": {}, 1525 | "source": [ 1526 | "You can then start a server simply by passing it the cluster spec and indicating its type and index. Let's start the two remaining tasks (remember that in general you would only start a single task per machine; we are starting 3 tasks on the localhost just for the purpose of this code example):" 1527 | ] 1528 | }, 1529 | { 1530 | "cell_type": "code", 1531 | "execution_count": 56, 1532 | "metadata": {}, 1533 | "outputs": [], 1534 | "source": [ 1535 | "#worker1 = tf.distribute.Server(cluster_spec, job_name=\"worker\", task_index=1)\n", 1536 | "ps0 = tf.distribute.Server(cluster_spec, job_name=\"ps\", task_index=0)" 1537 | ] 1538 | }, 1539 | { 1540 | "cell_type": "code", 1541 | "execution_count": 57, 1542 | "metadata": {}, 1543 | "outputs": [ 1544 | { 1545 | "name": "stdout", 1546 | "output_type": "stream", 1547 | "text": [ 1548 | "'{\"cluster\": {\"worker\": [\"127.0.0.1:9901\", \"127.0.0.1:9902\"], \"ps\": [\"127.0.0.1:9903\"]}, \"task\": {\"type\": \"worker\", \"index\": 1}}'\n" 1549 | ] 1550 | } 1551 | ], 1552 | "source": [ 1553 | "os.environ[\"TF_CONFIG\"] = json.dumps({\n", 1554 | " \"cluster\": {\n", 1555 | " \"worker\": [\"127.0.0.1:9901\", \"127.0.0.1:9902\"],\n", 1556 | " \"ps\": [\"127.0.0.1:9903\"]\n", 1557 | " },\n", 1558 | " \"task\": {\"type\": \"worker\", \"index\": 1}\n", 1559 | "})\n", 1560 | "print(repr(os.environ[\"TF_CONFIG\"]))" 1561 | ] 1562 | }, 1563 | { 1564 | "cell_type": "code", 1565 | "execution_count": 58, 1566 | "metadata": {}, 1567 | "outputs": [], 1568 | "source": [ 1569 | "distribution = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n", 1570 | "\n", 1571 | "keras.backend.clear_session()\n", 1572 | "tf.random.set_seed(42)\n", 1573 | "np.random.seed(42)\n", 1574 | "\n", 1575 | "os.environ[\"TF_CONFIG\"] = json.dumps({\n", 1576 | " \"cluster\": {\n", 1577 | " \"worker\": [\"127.0.0.1:9901\", \"127.0.0.1:9902\"],\n", 1578 | " \"ps\": [\"127.0.0.1:9903\"]\n", 1579 | " },\n", 1580 | " \"task\": {\"type\": \"worker\", \"index\": 1}\n", 1581 | "})\n", 1582 | "#CUDA_VISIBLE_DEVICES=0 \n", 1583 | "\n", 1584 | "with distribution.scope():\n", 1585 | " model = create_model()\n", 1586 | " model.compile(loss=\"sparse_categorical_crossentropy\",\n", 1587 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 1588 | " metrics=[\"accuracy\"])" 1589 | ] 1590 | }, 1591 | { 1592 | "cell_type": "code", 1593 | "execution_count": 59, 1594 | "metadata": {}, 1595 | "outputs": [], 1596 | "source": [ 1597 | "import tensorflow as tf\n", 1598 | "from tensorflow import keras\n", 1599 | "import numpy as np\n", 1600 | "\n", 1601 | "# At the beginning of the program (restart the kernel before running this cell)\n", 1602 | "distribution = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n", 1603 | "\n", 1604 | "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.mnist.load_data()\n", 1605 | "X_train_full = X_train_full[..., np.newaxis] / 255.\n", 1606 | "X_test = X_test[..., np.newaxis] / 255.\n", 1607 | "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n", 1608 | "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n", 1609 | "X_new = X_test[:3]\n", 1610 | "\n", 1611 | "n_workers = 2\n", 1612 | "batch_size = 32 * n_workers\n", 1613 | "dataset = tf.data.Dataset.from_tensor_slices((X_train[..., np.newaxis], y_train)).repeat().batch(batch_size)\n", 1614 | " \n", 1615 | "def create_model():\n", 1616 | " return keras.models.Sequential([\n", 1617 | " keras.layers.Conv2D(filters=64, kernel_size=7, activation=\"relu\",\n", 1618 | " padding=\"same\", input_shape=[28, 28, 1]),\n", 1619 | " keras.layers.MaxPooling2D(pool_size=2),\n", 1620 | " keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", 1621 | " padding=\"same\"), \n", 1622 | " keras.layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\",\n", 1623 | " padding=\"same\"),\n", 1624 | " keras.layers.MaxPooling2D(pool_size=2),\n", 1625 | " keras.layers.Flatten(),\n", 1626 | " keras.layers.Dense(units=64, activation='relu'),\n", 1627 | " keras.layers.Dropout(0.5),\n", 1628 | " keras.layers.Dense(units=10, activation='softmax'),\n", 1629 | " ])\n", 1630 | "\n", 1631 | "with distribution.scope():\n", 1632 | " model = create_model()\n", 1633 | " model.compile(loss=\"sparse_categorical_crossentropy\",\n", 1634 | " optimizer=keras.optimizers.SGD(lr=1e-2),\n", 1635 | " metrics=[\"accuracy\"])\n", 1636 | "\n", 1637 | "model.fit(dataset, steps_per_epoch=len(X_train)//batch_size, epochs=10)" 1638 | ] 1639 | }, 1640 | { 1641 | "cell_type": "code", 1642 | "execution_count": 60, 1643 | "metadata": {}, 1644 | "outputs": [], 1645 | "source": [ 1646 | "# Hyperparameter tuning\n", 1647 | "\n", 1648 | "# Only talk to ps server\n", 1649 | "config_proto = tf.ConfigProto(device_filters=['/job:ps', '/job:worker/task:%d' % tf_config['task']['index']])\n", 1650 | "config = tf.estimator.RunConfig(session_config=config_proto)\n", 1651 | "# default since 1.10" 1652 | ] 1653 | }, 1654 | { 1655 | "cell_type": "code", 1656 | "execution_count": 61, 1657 | "metadata": {}, 1658 | "outputs": [], 1659 | "source": [ 1660 | "strategy.num_replicas_in_sync" 1661 | ] 1662 | } 1663 | ], 1664 | "metadata": { 1665 | "kernelspec": { 1666 | "display_name": "Python 3", 1667 | "language": "python", 1668 | "name": "python3" 1669 | }, 1670 | "language_info": { 1671 | "codemirror_mode": { 1672 | "name": "ipython", 1673 | "version": 3 1674 | }, 1675 | "file_extension": ".py", 1676 | "mimetype": "text/x-python", 1677 | "name": "python", 1678 | "nbconvert_exporter": "python", 1679 | "pygments_lexer": "ipython3", 1680 | "version": "3.7.6" 1681 | } 1682 | }, 1683 | "nbformat": 4, 1684 | "nbformat_minor": 4 1685 | } 1686 | --------------------------------------------------------------------------------