├── .gitmodules
├── Data-Handling
    └── README.md
├── Deep-Learning-TensorFlow.md
├── Graph-Neural-Networks
    ├── Install-Graph-Nets.md
    └── README.md
├── Images
    ├── BH.jpeg
    ├── DDLP.png
    ├── DL.jpg
    ├── Deep-Learning-PyTorch.jpg
    ├── Dive-Into-DL.png
    ├── GDL.jpg
    ├── HH.jpg
    ├── Hands-On-ML.jpg
    ├── K.jpg
    ├── Mafakheri.jpg
    ├── NNLM.jpg
    ├── PPDL.jpg
    ├── ProDeep.jpg
    ├── R.jpg
    ├── README.md
    ├── Y.jpg
    └── data-fallacies-to-avoid.jpg
├── NoteBooks
    ├── Generator.ipynb
    ├── Lec1.ipynb
    ├── Lec2.ipynb
    ├── Lec3.ipynb
    ├── README.MD
    └── The-Confusing-Parts-2.ipynb
├── Optimization
    └── Readme.md
├── Projects
    └── README.md
├── PyTorch
    └── README.MD
├── README.md
├── Recitation-Assignments
    ├── Assignment_Set_1_Sample.ipynb
    └── README.md
├── TensorFlow-1
    └── README.md
├── _config.yml
└── _layouts
    └── default.html


/.gitmodules:
--------------------------------------------------------------------------------
 1 | [submodule "TFLearn"]
 2 | 	path = TFLearn
 3 | 	url = https://github.com/tflearn/tflearn
 4 | [submodule "TensorFlow-Examples"]
 5 | 	path = TensorFlow-Examples
 6 | 	url = https://github.com/aymericdamien/TensorFlow-Examples
 7 | [submodule "Books-Papers-Roadmap"]
 8 | 	path = Books-Papers-Roadmap
 9 | 	url = https://github.com/floodsung/Deep-Learning-Papers-Reading-Roadmap
10 | [submodule "Most-Cited-Deep-Learning-Papers"]
11 | 	path = Most-Cited-Deep-Learning-Papers
12 | 	url = https://github.com/terryum/awesome-deep-learning-papers
13 | [submodule "CheatSheets"]
14 | 	path = CheatSheets
15 | 	url = https://github.com/kailashahirwar/cheatsheets-ai
16 | [submodule "PyTorch"]
17 | 	path = PyTorch
18 | 	url = https://github.com/pytorch/pytorch
19 | 


--------------------------------------------------------------------------------
/Data-Handling/README.md:
--------------------------------------------------------------------------------
  1 | # Data Handling
  2 | Here we introduce several blogs related to data and data handling and also some resources of datasets.   
  3 | * [Improve Your Data Literacy Skills and Make the Most of Data](https://www.geckoboard.com/learn/data-literacy/) by Geckoboard Company <br>
  4 |   - [Tips for Effective Data Visualization](https://www.geckoboard.com/learn/data-literacy/data-visualization-tips/) <br>
  5 |   - [Common Data Mistakes to Avoid](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/) <br>
  6 |     
  7 |   ![Poster](../Images/data-fallacies-to-avoid.jpg) <br>
  8 | 
  9 |   
 10 |   Common | Data |  Mistakes | to | Avoid
 11 |     -------- | ------- | --------| -----------| ----------
 12 |   [Cherry Picking](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/cherry-picking/) |  [Data Dredging](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/data-dredging/) | [Survivorship Bias](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/survivorship-bias/) | [Cobra Effect](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/cobra-effect/) | [False Causality](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/false-causality/)
 13 |   [Gerrymandering](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/gerrymandering/) | [Sampling Bias](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/sampling-bias/) | [Gambler's Fallacy](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/gamblers-fallacy/) | [Hawthorne Effect](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/hawthorne-effect/) | [Regression Toward the Mean](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/regression-toward-the-mean/)  
 14 |   [Simpson's Paradox](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/simpsons-paradox/) | [McNamara Fallacy](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/mcnamara-fallacy/) | [Overfitting](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/overfitting/) | [Publication Bias](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/publication-bias/) | [Danger of Summary Metrics](https://www.geckoboard.com/learn/data-literacy/statistical-fallacies/danger-of-summary-metrics/)
 15 |   
 16 | ## Dealing with Data 
 17 | * Slide: [Data Preparation](https://web.fe.up.pt/~ec/files_1112/week_03_Data_Preparation.pdf) by João Mendes Moreira and José Luís Borges    
 18 | * Slide: [Data Preprocessing](http://www.csun.edu/~twang/595DM/Slides/Week2.pdf) by Taehyung Wang   
 19 | * Slide: [Learning with Missing Labels](https://svivek.com/teaching/machine-learning/fall2018/slides/em/missing-labels.pdf) by Vivek Srikumar  
 20 | * Slide: [Data Cleaning and Data Preprocessing](https://www.mimuw.edu.pl/~son/datamining/DM/4-preprocess.pdf) by Nguyen Hung Son    
 21 | * Blog: [Applying Wrapper Methods in Python for Feature Selection](https://stackabuse.com/applying-wrapper-methods-in-python-for-feature-selection/) 
 22 | by Usman Malik  
 23 | * Blog: [Basics of Feature Selection with Python](https://www.kaggle.com/ar2017/basics-of-feature-selection-with-python) by Andika Rachman  
 24 | * Blog: [Exhaustive Feature Selector](http://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/) by Sebastian Raschka   
 25 | * Blog: [Need for Feature Engineering in Machine Learning](https://towardsdatascience.com/need-for-feature-engineering-in-machine-learning-897df2ed00e6) by Ashish Bansal    
 26 | * Blog: [Data Preprocessing](http://www.cs.ccsu.edu/~markov/ccsu_courses/DataMining-3.html) by Zdravko Markov   
 27 | * Blog: [How to Handle Correlated Features?](https://www.kaggle.com/reisel/how-to-handle-correlated-features) by Reinhard Sellmair  
 28 | * Blog: [5 Ways To Handle Missing Values In Machine Learning Datasets](https://www.analyticsindiamag.com/5-ways-handle-missing-values-machine-learning-datasets/)
 29 | * Blog: [Handling Missing Data](http://www.emgo.nl/kc/handling-missing-data/)
 30 | * Blog: [How to Handle Missing Data](https://towardsdatascience.com/how-to-handle-missing-data-8646b18db0d4)
 31 | * Blog: [7 Techniques to Handle Imbalanced Data](https://www.kdnuggets.com/2017/06/7-techniques-handle-imbalanced-data.html)
 32 | * Blog: [Application of Synthetic Minority Over-sampling Technique (SMOTe) for Imbalanced Datasets](https://medium.com/towards-artificial-intelligence/application-of-synthetic-minority-over-sampling-technique-smote-for-imbalanced-data-sets-509ab55cfdaf) by Navoneel Chakrabarty  
 33 | * Paper: [SMOTE: Synthetic Minority Over-sampling Technique](https://arxiv.org/pdf/1106.1813.pdf) by Nitesh V. Chawla, Kevin W. Bowyer, Lawrence O. Hall, and W. Philip Kegelmeyer   
 34 | * Blog: [How to Handle Imbalanced Data: An Overview](https://www.datascience.com/blog/imbalanced-data)
 35 | * Blog: [Visualize Missing Data with VIM Package](https://www.datacamp.com/community/tutorials/visualize-data-vim-package)
 36 | * [Ultimate Guide to Handle Big Datasets for Machine Learning Using Dask (in Python)](https://www.analyticsvidhya.com/blog/2018/08/dask-big-datasets-machine_learning-python/)
 37 | 
 38 | ## Datasets  
 39 | The following resources may be helpful for those still undecided about their course projects.
 40 | * [VisualData:](https://www.visualdata.io) Discover computer vision datasets <br>
 41 | * [OpenML:](https://www.openml.org) An open science platform for machine learning <br>
 42 | * [Open Datasets:](https://skymind.ai/wiki/open-datasets) A list of links to publicly available datasets for a variety of domains.
 43 | * [DataHub](https://datahub.io/) has a lot of structured data in formats such as RDF and CSV. 
 44 | * [Datasets for Machine Learning](https://www.datasetlist.com) 
 45 | * [UC Irvine Machine Learning Repository](http://archive.ics.uci.edu/ml/index.php)
 46 | * [Kaggle Datasets](https://www.kaggle.com/datasets)
 47 | * [Awesome Public Datasets](https://github.com/awesomedata/awesome-public-datasets)
 48 | * [CrowdFlower Data for Everyone library](http://www.crowdflower.com/data-for-everyone)
 49 | * [Stanford Large Network Dataset Collection](https://snap.stanford.edu/data/index.html)
 50 | * [Data Science Weekly](https://www.datascienceweekly.org/data-science-resources/data-science-datasets)
 51 | * [Awesome Data Science](https://github.com/bulutyazilim/awesome-datascience#data-sets)
 52 | * [Get Financial Data Directly Into R](https://www.quandl.com/tools/r)
 53 | * [Listen Data from the Green Bank Telescope](http://seti.berkeley.edu/frb-machine/)
 54 | * [Cafebazaar](https://research.cafebazaar.ir/visage/datasets/) 
 55 | * [25 Open Datasets for Deep Learning Every Data Scientist Must Work With](https://www.analyticsvidhya.com/blog/2018/03/comprehensive-collection-deep-learning-datasets/) by Pranav Dar  
 56 | * [DigiKala](https://www.dataacademy.ir/داده-های-باز-در-دیجی-کالا) (Persian)  
 57 | * [Statistical Center of Iran](https://www.amar.org.ir/english/) 
 58 |     - [Dataset (Persian)](https://www.amar.org.ir/دادهها-و-اطلاعات-آماری/هزینه-و-درامد-خانوار/هزینه-و-درامد-کل-کشور#103181018---)   
 59 | 
 60 | To know more datasets, refer to the following webpage of [KDnuggets](https://www.kdnuggets.com/index.html):
 61 | * [Datasets for Data Mining and Data Science](https://www.kdnuggets.com/datasets/index.html)  
 62 | 
 63 | ## Datasets of Molecules and Their Properties  
 64 | 
 65 | - Blog: [MoleculeNet](http://moleculenet.ai/) is a benchmark specially designed for testing machine learning methods of molecular properties. As we aim to facilitate the development of molecular machine learning method, this work curates a number of dataset collections, creates a suite of software that implements many known featurizations and previously proposed algorithms. All methods and datasets are integrated as parts of the open source **DeepChem** package(MIT license).  
 66 | - Blog: [ChEMBL](https://www.ebi.ac.uk/chembl/) is a manually curated database of bioactive molecules with drug-like properties. It brings together chemical, bioactivity and genomic data to aid the translation of genomic information into effective new drugs.
 67 | - Blog: [Tox21](https://tripod.nih.gov/tox21/challenge/): The 2014 Tox21 data challenge was designed to help scientists understand the potential of the chemicals and compounds being tested through the Toxicology in the 21st Century initiative to disrupt biological pathways in ways that may result in toxic effects. The Tox21 Program (Toxicology in the 21st Century) is an ongoing collaboration among federal agencies to characterize the potential toxicity of chemicals using cells and isolated molecular targets instead of laboratory animals.  
 68 | 
 69 | ## Datasets of Graphs
 70 | 
 71 | - Blog: [Network Repository. An Interactive Scientific Network Data Repository:](http://networkrepository.com) The first interactive data and network data repository with real-time visual analytics. Network repository is not only the first interactive repository, but also the largest network repository with thousands of donations in 30+ domains (from biological to social network data). This repository was made by Ryan A. Rossi and Nesreen K. Ahmed.  
 72 | - Blog: [Graph Classification:](https://paperswithcode.com/task/graph-classification/latest) The mission of Papers With Code is to create a free and open resource with Machine Learning papers, code and evaluation tables.  
 73 | - Blog: [Graph Challenge Data Sets:](https://graphchallenge.mit.edu/data-sets) Amazon is making the Graph Challenge data sets available to the community free of charge as part of the AWS Public Data Sets program. The data is being presented in several file formats, and there are a variety of ways to access it.    
 74 | - Blog: [The House of Graphs:](https://hog.grinvin.org) a database of interesting graphs by G. Brinkmann, K. Coolsaet, J. Goedgebeur, and H. Mélot (also see Discrete Applied Mathematics, 161(1-2): 311-314, 2013 ([DOI](http://dx.doi.org/10.1016/j.dam.2012.07.018))).   
 75 |     * [Search for Graphs](https://hog.grinvin.org/StartSearch.action)  
 76 | - Blog: [A Repository of Benchmark Graph Datasets for Graph Classification](https://github.com/shiruipan/graph_datasets) by 
 77 | Shiruipan    
 78 | - Blog: [Collection and Streaming of Graph Datasets](https://www.eecs.wsu.edu/~yyao/StreamingGraphs.html) by Yibo Yao   
 79 | - Blog: [Big Graph Data Sets](https://lgylym.github.io/big-graph/dataset.html) by Yongming Luo   
 80 | - Blog: [MIVIA LDGraphs Dataset:](https://mivia.unisa.it/datasets/graph-database/mivia2-graph-database/) The MIVIA LDGraphs (MIVIA Large Dense Graphs) dataset is a new dataset for benchmarking exact graph matching algorithms. It aims to extend the MIVIA graphs dataset, widely used in the last ten years, with bigger and more dense graphs, so as to face with the problems nowadays encountered in real applications devoted for instance to bioinformatics and social network analysis. 
 81 | - Blog: [Datasets](https://sites.wustl.edu/neumann/research/datasets/) by Marion Neumann  
 82 | - Blog: [Graph Dataset](https://sites.google.com/site/xiaomengsite/research/resources/graph-dataset) by Xiao Meng   
 83 | - Blog: [Constructors and Databases of Graphs in Sage](http://doc.sagemath.org/html/en/reference/graphs/index.html)
 84 | - Datasets in GitHub:   
 85 |   - [Benchmark Dataset for Graph Classification:](https://github.com/FilippoMB/Benchmark_dataset_for_graph_classification) This repository contains datasets to quickly test graph classification algorithms, such as Graph Kernels and Graph Neural Networks by Filippo Bianchi.     
 86 |   - [GAM:](https://github.com/benedekrozemberczki/GAM) A PyTorch implementation of "Graph Classification Using Structural Attention" (KDD 2018) by Benedek Rozemberczki.
 87 |   - [CapsGNN:](https://github.com/benedekrozemberczki/CapsGNN) A PyTorch implementation of "Capsule Graph Neural Network" (ICLR 2019) by Benedek Rozemberczki.   
 88 |   
 89 |  ### Tools for Creating Graphs
 90 | 
 91 | - Package: [Networkx:](https://networkx.github.io) a Python package for the creation, manipulation, and study of the structure, dynamics, and functions of complex networks.  
 92 |   - [Graph Generators](https://networkx.github.io/documentation/stable/reference/generators.html)  
 93 |   - [Converting to and from Other Data Formats To NetworkX Graph](https://networkx.github.io/documentation/stable/reference/convert.html)  
 94 |   - [Reading and Writing Graphs](https://networkx.github.io/documentation/stable/reference/readwrite/index.html)    
 95 |   
 96 | - Package: [Sage:](https://www.sagemath.org) a viable free open source alternative to Magma, Maple, Mathematica and Matlab.
 97 |   - [CoCalc:](https://www.sagemath.org/notebook-vs-cloud.html) an [online service](https://cocalc.com/) for running SageMath computations online to avoid your own installation of Sage. CoCalc will allow you to work with multiple persistent worksheets in Sage, IPython, LaTeX, and much, much more!
 98 |   - [Graph Theory in Sage](http://doc.sagemath.org/html/en/reference/graphs/index.html) 
 99 | 
100 | 
101 | ## Data Science Competition Platforms  
102 | * [Kaggle](https://www.kaggle.com/datasets)
103 |   * [Kaggle Competition Past Solutions](http://www.chioka.in/kaggle-competition-solutions/) 
104 |   * [Kaggle Past Solutions](https://ndres.me/kaggle-past-solutions/) by Eliot Andres 
105 |   * [The Tips and Tricks I Used to Succeed on Kaggle](https://www.dataquest.io/blog/kaggle-tips-tricks/) by Vik Paruchuri 
106 | * [DrivenData](http://www.drivendata.org)
107 | * [TunedIT](http://www.tunedit.org)
108 | * [InnoCentive](https://www.innocentive.com)
109 | * [CrowdAnalytix](http://www.crowdanalytix.com)
110 | 


--------------------------------------------------------------------------------
/Deep-Learning-TensorFlow.md:
--------------------------------------------------------------------------------
  1 | # Deep Learning Using TensorFlow
  2 | 
  3 | Lecturer: [Hossein Hajiabolhassan](http://facultymembers.sbu.ac.ir/hhaji/)     
  4 | [Data Science Center](http://ds.sbu.ac.ir), [Shahid Beheshti University](http://www.sbu.ac.ir/)    
  5 | 
  6 | ---
  7 | 
  8 | ### **Index:**
  9 | - [Course Overview](#Course-Overview)
 10 | - [Main TextBooks](#Main-TextBooks)
 11 | - [Slides and Papers](#Slides-and-Papers)
 12 |   1. Lecture 1: [Introduction](#Introduction) 
 13 |   2. Lecture 2: [Toolkit Lab 1: Google Colab and Anaconda](#Part-1) 
 14 |   3. Lecture 3: [Toolkit Lab 2: Image Preprocessing by Keras](#Part-2) 
 15 |   4. Lecture 4: [Deep Feedforward Networks](#DFN) 
 16 |   5. Lecture 5: [Toolkit Lab 3: Introduction to Artificial Neural Networks with Keras](#Part-3) 
 17 |   6. Lecture 6: [Regularization for Deep Learning](#RFDL) 
 18 |   7. Lecture 7: [Optimization for Training Deep Models](#OFTDM) 
 19 |   8. Lecture 8: [Toolkit Lab 4: Training Deep Neural Networks](#Part-4) 
 20 |   9. Lecture 9: [Toolkit Lab 5: Custom Models and Training with TensorFlow 2.0](#Part-5) 
 21 |   10. Lecture 10: [Convolutional Networks](#CNN) 
 22 |   11. Lecture 11: [Toolkit Lab 6: TensorBoard](#Part-6) 
 23 |   12. Lecture 12: [Sequence Modeling: Recurrent and Recursive Networks](#SMRARN) 
 24 |   13. Lecture 13: [Practical Methodology](#Practical-Methodology)  
 25 |   14. Lecture 14: [Applications](#Applications) 
 26 |   15. Lecture 15: [Autoencoders](#Autoencoders)
 27 |   16. Lecture 16: [Generative Adversarial Networks](#GAN)  
 28 |   17. Lecture 17: [Graph Neural Networks](#GNN)
 29 | - [Additional Resources](#ANAS)
 30 | - [Class Time and Location](#Class-Time-and-Location)
 31 | - [Projects](#Projects)
 32 |   - [Google Colab](#Google-Colab)
 33 |   - [Fascinating Guides For Machine Learning](#Fascinating-Guides-For-Machine-Learning)
 34 |   - [Latex](#Latex)
 35 | - [Grading](#Grading)
 36 | - [Prerequisites](#Prerequisites)
 37 |   - [Linear Algebra](#Linear-Algebra)
 38 |   - [Probability and Statistics](#Probability-and-Statistics)
 39 | - [Topics](#Topics)
 40 | - [Account](#Account)
 41 | - [Academic Honor Code](#Academic-Honor-Code)
 42 | - [Questions](#Questions)
 43 | - Miscellaneous: 
 44 |      * [Data Handling](https://github.com/hhaji/Deep-Learning/tree/master/Data-Handling)  
 45 | 
 46 | ---
 47 | 
 48 | ## <a name="Course-Overview"></a>Course Overview:
 49 | ```javascript
 50 | In this course, you will learn the foundations of Deep Learning, understand how to build 
 51 | neural networks, and learn how to lead successful machine learning projects. You will learn 
 52 | about Convolutional networks, RNNs, LSTM, Adam, Dropout, BatchNorm, and more.
 53 | ```
 54 | 
 55 | ## <a name="Main-TextBooks"></a>Main TextBooks:
 56 | ![Book 1](/Images/DL.jpg)  ![Book 2](/Images/Hands-On-ML.jpg) ![Book 3](/Images/Dive-Into-DL.png) ![Book 4](/Images/NNLM.jpg) ![Book 5](/Images/DDLP.png)
 57 | 
 58 | ```
 59 | Main TextBooks:
 60 | ```
 61 | 
 62 | * [Deep Learning](http://www.deeplearningbook.org) (available in online) by Bengio, Yoshua, Ian J. Goodfellow, and Aaron Courville <br>
 63 | * [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron <br>
 64 | 
 65 | ```
 66 | Additional TextBooks:
 67 | ```
 68 | 
 69 | * [Dive into Deep Learning](https://d2l.ai) by  Mag Gardner, Max Drummy, Joanne Quinn, Joanne McEachen, and Michael Fullan   
 70 |     - GitHub: [Codes](https://github.com/dsgiitr/d2l-pytorch)   
 71 | * [Neural Networks and Learning Machines (3rd Edition)](https://www.amazon.com/Neural-Networks-Learning-Machines-Comprehensive-ebook/dp/B008VIX57I) by Simon Haykin  
 72 | * [Deep Learning with Python](https://machinelearningmastery.com/deep-learning-with-python/) by J. Brownlee   
 73 | 
 74 | 
 75 | ## <a name="Slides-and-Papers"></a>Slides and Papers:  
 76 |   Recommended Slides & Papers:
 77 |   
 78 | 1. ### <a name="Introduction"></a>Introduction  
 79 | 
 80 | ```
 81 | Required Reading:
 82 | ```
 83 | 
 84 |   * [Chapter 1](http://www.deeplearningbook.org/contents/intro.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br>
 85 |   * Slide: [Introduction](https://www.deeplearningbook.org/slides/01_intro.pdf)  by Ian Goodfellow
 86 |  
 87 | ```
 88 | Suggested Reading:
 89 | ```
 90 |  
 91 |  * Demo: [3D Fully-Connected Network Visualization](http://scs.ryerson.ca/~aharley/vis/fc/) by Adam W. Harley  
 92 | 
 93 | ```
 94 | Additional Resources:
 95 | ```
 96 | 
 97 |   * [Video](https://www.youtube.com/embed//vi7lACKOUao) of lecture by Ian Goodfellow and discussion of Chapter 1 at a reading group in San Francisco organized by Alena Kruchkova <br>
 98 |   * Paper: [On the Origin of Deep Learning](https://arxiv.org/pdf/1702.07800.pdf) by Haohan Wang and Bhiksha Raj <br>
 99 | 
100 | ```
101 | Applied Mathematics and Machine Learning Basics:
102 | ```
103 | 
104 |  * Slide: [Mathematics for Machine Learning](http://www.deeplearningindaba.com/uploads/1/0/2/6/102657286/2018_maths4ml_vfinal.pdf) by Avishkar Bhoopchand, Cynthia Mulenga, Daniela Massiceti, Kathleen Siminyu, and Kendi Muchungi 
105 | * Blog: [A Gentle Introduction to Maximum Likelihood Estimation and Maximum A Posteriori Estimation (Getting Intuition of MLE and MAP with a Football Example)](https://towardsdatascience.com/a-gentle-introduction-to-maximum-likelihood-estimation-and-maximum-a-posteriori-estimation-d7c318f9d22d) by Shota Horii  
106 |     
107 | 2. ### <a name="Part-1"></a>Toolkit Lab 1: Google Colab and Anaconda  
108 | 
109 | ```
110 | Required Reading:
111 | ```
112 | 
113 |   * Blog: [Google Colab Free GPU Tutorial](https://medium.com/deep-learning-turkey/google-colab-free-gpu-tutorial-e113627b9f5d) by Fuat <br>
114 |   * Blog: [Managing Environments](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#managing-environments) <br>
115 |   * Blog: [Kernels for Different Environments](https://ipython.readthedocs.io/en/stable/install/kernel_install.html#kernels-for-different-environments) <br>
116 |   * Install: [TensorFlow 2.0 RC is Available](https://www.tensorflow.org/install) <br>
117 |  
118 | ```
119 | Suggested Reading:
120 | ```
121 |  
122 |  * Blog: [Stop Installing Tensorflow Using pip for Performance Sake!](https://towardsdatascience.com/stop-installing-tensorflow-using-pip-for-performance-sake-5854f9d9eb0c) by Michael Nguyen <br> 
123 |  * Blog: [Using Pip in a Conda Environment](https://www.anaconda.com/using-pip-in-a-conda-environment/) by Jonathan Helmus <br> 
124 |  * Blog: [How to Import Dataset to Google Colab Notebook?](https://mc.ai/how-to-import-dataset-to-google-colab-notebook/) 
125 |  * Blog: [How to Upload Large Files to Google Colab and Remote Jupyter Notebooks ](https://www.freecodecamp.org/news/how-to-transfer-large-files-to-google-colab-and-remote-jupyter-notebooks-26ca252892fa/)(For Linux Operating System) by Bharath Raj  <br>
126 | 
127 | ```
128 | Additional Resources:
129 | ```
130 |   * PDF: [Conda Cheat Sheet](https://docs.conda.io/projects/conda/en/latest/_downloads/1f5ecf5a87b1c1a8aaf5a7ab8a7a0ff7/conda-cheatsheet.pdf) 
131 |   * Blog: [Conda Commands (Create Virtual Environments for Python with Conda)](http://deeplearning.lipingyang.org/2018/12/25/conda-commands-create-virtual-environments-for-python-with-conda/) by LipingY <br>  
132 |   * Blog: [Colab Tricks](https://rohitmidha23.github.io/Colab-Tricks/) by  Rohit Midha <br>
133 |   
134 | 3. ### <a name="Part-2"></a>Toolkit Lab 2: Image Preprocessing by Keras 
135 | 
136 | ```
137 | Required Reading:
138 | ```
139 |     
140 |   * Blog: [How to Load, Convert, and Save Images With the Keras API](https://machinelearningmastery.com/how-to-load-convert-and-save-images-with-the-keras-api/) by Jason Brownlee 
141 |   * Blog: [Classify Butterfly Images with Deep Learning in Keras](https://towardsdatascience.com/classify-butterfly-images-with-deep-learning-in-keras-b3101fe0f98) by Bert Carremans  
142 |     Read the part of Data augmentation of images  
143 |   * Blog: [Keras ImageDataGenerator Methods: An Easy Guide](https://medium.com/datadriveninvestor/keras-imagedatagenerator-methods-an-easy-guide-550ecd3c0a92) by Ashish Verma  
144 |   
145 | ```
146 | Suggested Reading:
147 | ```
148 |    
149 |   * Blog: [Keras ImageDataGenerator and Data Augmentation](https://www.pyimagesearch.com/2019/07/08/keras-imagedatagenerator-and-data-augmentation/) by  Adrian Rosebrock  
150 |   * Blog: [How to Configure Image Data Augmentation in Keras](https://machinelearningmastery.com/how-to-configure-image-data-augmentation-when-training-deep-learning-neural-networks/) by Jason Brownlee 
151 |   * Blog: [A Quick Guide To Python Generators and Yield Statements](https://medium.com/@jasonrigden/a-quick-guide-to-python-generators-and-yield-statements-89a4162c0ef8) by Jason Rigden 
152 |   * NoteBook: [Iterable, Generator, and Iterator](https://github.com/hhaji/Deep-Learning/blob/master/NoteBooks/Generator.ipynb)  
153 |   * Blog: [Vectorization in Python](https://www.geeksforgeeks.org/vectorization-in-python/) 
154 |   * Blog: [numpy.vectorize](https://docs.scipy.org/doc/numpy-1.15.1/reference/generated/numpy.vectorize.html)
155 |  
156 | ```
157 | Additional Resources:
158 | ```
159 | 
160 |  * Blog: [Learn about ImageDataGenerator](https://fairyonice.github.io/Learn-about-ImageDataGenerator.html) by Yumi 
161 |  * Blog: [Images Augmentation for Deep Learning with Keras](https://rock-it.pl/images-augmentation-for-deep-learning-with-keras/) by Jakub Skałecki  
162 |  * Blog: [A Detailed Example of How to Use Data Generators with Keras](https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly#disqus-thread) by Afshine Amidi and Shervine Amidi  
163 |   * Blog: [Iterables vs. Iterators vs. Generators](https://nvie.com/posts/iterators-vs-generators/) by Vincent Driessen   
164 |   
165 | 4. ### <a name="DFN"></a>Deep Feedforward Networks  
166 | 
167 | ```
168 | Required Reading:
169 | ```
170 | 
171 |   * [Chapter 6](https://www.deeplearningbook.org/contents/mlp.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br> 
172 |   * Slide: [Feedforward Neural Networks (Lecture 2)](http://wavelab.uwaterloo.ca/wp-content/uploads/2017/04/Lecture_2.pdf) by Ali Harakeh  
173 |   * Slides: Deep Feedforward Networks [1](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L8-deep_feedforward_networks.pdf)  and [2](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L9-deep_feedforward_networks-2.pdf) by U Kang   
174 |   * Chapter 20 of [Understanding Machine Learning: From Theory to Algorithms](http://www.cs.huji.ac.il/~shais/UnderstandingMachineLearning) <br>
175 |   * Slide: [Neural Networks](https://www.cs.huji.ac.il/~shais/Lectures2014/lecture10.pdf) by Shai Shalev-Shwartz <br>
176 |   * Slide: [Backpropagation and Neural Networks](http://cs231n.stanford.edu/slides/2017/cs231n_2017_lecture4.pdf) by Fei-Fei Li, Justin Johnson, and  Serena Yeung  
177 |   * Blog: [7 Types of Neural Network Activation Functions: How to Choose?](https://missinglink.ai/guides/neural-network-concepts/7-types-neural-network-activation-functions-right/) <br>
178 |   * Blog: [Back-Propagation, an Introduction](https://www.offconvex.org/2016/12/20/backprop/) by Sanjeev Arora and Tengyu Ma <br>
179 | 
180 | ```
181 | Interesting Questions:
182 | ```
183 | 
184 | * [Why are non Zero-Centered Activation Functions a Problem in Backpropagation?](https://stats.stackexchange.com/questions/237169/why-are-non-zero-centered-activation-functions-a-problem-in-backpropagation)   
185 |   
186 | ```
187 | Suggested Reading:
188 | ```
189 | 
190 |   * Blog: [The Gradient](https://www.khanacademy.org/math/multivariable-calculus/multivariable-derivatives/partial-derivative-and-gradient-articles/a/the-gradient) by Khanacademy <br>
191 |   * Blog: [Calculus on Computational Graphs: Backpropagation](http://colah.github.io/posts/2015-08-Backprop/) by Christopher Olah 
192 |   
193 | ```
194 | Additional Resources:
195 | ```
196 | 
197 |   * Blog: [Activation Functions](https://sefiks.com/tag/activation-function/) by Sefik Ilkin Serengil   
198 |   * Paper: [Mish: A Self Regularized Non-Monotonic Neural Activation Function](https://arxiv.org/abs/1908.08681v2) by Diganta Misra  
199 |   * Blog: [Activation Functions](https://ml-cheatsheet.readthedocs.io/en/latest/activation_functions.html#id5)  
200 |   * Blog: [Analytical vs Numerical Solutions in Machine Learning](https://machinelearningmastery.com/analytical-vs-numerical-solutions-in-machine-learning/) by Jason Brownlee  
201 |   * Blog: [Validating Analytic Gradient for a Neural Network](https://medium.com/@shivajbd/how-to-validate-your-gradient-expression-for-a-neural-network-8284ede6272) by  Shiva Verma  
202 |   * Blog: [Stochastic vs Batch Gradient Descent](https://medium.com/@divakar_239/stochastic-vs-batch-gradient-descent-8820568eada1) by Divakar Kapil  
203 |   * [Video](https://drive.google.com/file/d/0B64011x02sIkRExCY0FDVXFCOHM/view?usp=sharing): (.flv) of a presentation by Ian  Goodfellow and a group discussion at a reading group at Google organized by Chintan Kaur. <br>
204 |   * **Extra Slide:**
205 |     - Slide: [Deep Feedforward Networks](https://www.deeplearningbook.org/slides/06_mlp.pdf)  by Ian Goodfellow  
206 |    
207 | 5. ### <a name="Part-3"></a>Toolkit Lab 3: Introduction to Artificial Neural Networks with Keras    
208 | ```
209 | Required Reading:
210 | ```
211 |     
212 |   * NoteBook: [Chapter 10 – Introduction to Artificial Neural Networks with Keras](https://github.com/ageron/handson-ml2/blob/master/10_neural_nets_with_keras.ipynb) from [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron      
213 | 
214 | ```
215 | Suggested Reading:
216 | ```
217 | 
218 |   * Blog: [Epoch vs Batch Size vs Iterations](https://towardsdatascience.com/epoch-vs-iterations-vs-batch-size-4dfb9c7ce9c9) by Sagar Sharma  
219 |   * Blog: [How to Load Large Datasets From Directories for Deep Learning in Keras](https://machinelearningmastery.com/how-to-load-large-datasets-from-directories-for-deep-learning-with-keras/) by Jason Brownlee  
220 |   * Blog: [A Thing You Should Know About Keras if You Plan to Train a Deep Learning Model on a Large Dataset](https://medium.com/difference-engine-ai/keras-a-thing-you-should-know-about-keras-if-you-plan-to-train-a-deep-learning-model-on-a-large-fdd63ce66bd2) by Soumendra P   
221 |     * Question: [Keras2 ImageDataGenerator or TensorFlow tf.data?](https://stackoverflow.com/questions/55627995/keras2-imagedatagenerator-or-tensorflow-tf-data)
222 |     * Blog: [Better Performance with tf.data](https://www.tensorflow.org/guide/data_performance) by the [TensorFlow Team](https://www.tensorflow.org)    
223 |   * Blog: [Standardizing on Keras: Guidance on High-level APIs in TensorFlow 2.0](https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a) by the [TensorFlow Team](https://www.tensorflow.org)  
224 |   * Blog & NoteBook: [How to Grid Search Hyperparameters for Deep Learning Models in Python With Keras](https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/) by Jason Brownlee  
225 |     
226 | ```
227 | Additional Resources:
228 | ``` 
229 |   * PDF: [Keras Cheat Sheet](https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Keras_Cheat_Sheet_Python.pdf)
230 |   * Blog: [Properly Setting the Random Seed in ML Experiments. Not as Simple as You Might Imagine](https://medium.com/@ODSC/properly-setting-the-random-seed-in-ml-experiments-not-as-simple-as-you-might-imagine-219969c84752) by [Open Data Science](https://opendatascience.com)    
231 |   * Blog: [Technical Notes On Using Data Science & Artificial Intelligence: To Fight For Something That Matters](https://chrisalbon.com/) by Chris Albon (read the Keras section)  
232 |   * Blog: [Keras Tutorial: Develop Your First Neural Network in Python Step-By-Step](https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/) by Jason Brownlee  
233 |   * Blog: [How to Use the Keras Functional API for Deep Learning](https://machinelearningmastery.com/keras-functional-api-deep-learning/) by Jason Brownlee  
234 |   * Blog: [Keras Tutorial for Beginners with Python: Deep Learning Example](https://www.guru99.com/keras-tutorial.html)
235 |   * Blog: [Learn Tensorflow 1: The Hello World of Machine Learning](https://codelabs.developers.google.com/codelabs/tensorflow-lab1-helloworld/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)    
236 |   * Blog: [Learn Tensorflow 2: Introduction to Computer Vision (Fashion MNIST)](https://codelabs.developers.google.com/codelabs/tensorflow-lab2-computervision/)  by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)    
237 |   * Blog: [Your first Keras Model, with Transfer Learning](https://codelabs.developers.google.com/codelabs/keras-flowers-transfer-learning/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
238 |   * Blog & NoteBook: [How to Choose Loss Functions When Training Deep Learning Neural Networks](https://machinelearningmastery.com/how-to-choose-loss-functions-when-training-deep-learning-neural-networks/) by Jason Brownlee  
239 |   * Blog: [TensorFlow 2.0 Tutorial 02: Transfer Learning](https://lambdalabs.com/blog/tensorflow-2-0-tutorial-02-transfer-learning/) by Chuan Li   
240 |  
241 |     
242 | ```
243 | Building Dynamic Models Using the Subclassing API:
244 | ``` 
245 |     
246 | * Object-Oriented Programming:
247 |     
248 |    * Blog: [Object-Oriented Programming (OOP) in Python 3](https://realpython.com/python3-object-oriented-programming/) by the Real Python Team   
249 |    * Blog: [How to Explain Object-Oriented Programming Concepts to a 6-Year-Old](https://www.freecodecamp.org/news/object-oriented-programming-concepts-21bb035f7260/)  
250 |    * Blog: [Understanding Object-Oriented Programming Through Machine Learning](https://dziganto.github.io/classes/data%20science/linear%20regression/machine%20learning/object-oriented%20programming/python/Understanding-Object-Oriented-Programming-Through-Machine-Learning/) by David Ziganto  
251 |    * Blog: [Object-Oriented Programming for Data Scientists: Build your ML Estimator](https://towardsdatascience.com/object-oriented-programming-for-data-scientists-build-your-ml-estimator-7da416751f64) by Tirthajyoti Sarkar  
252 |    * Blog: [Python Callable Class Method](https://medium.com/@nunenuh/python-callable-class-1df8e122b30c) by Lalu Erfandi Maula Yusnu  
253 |   
254 | * The Model Subclassing API:
255 |    * Blog: [How Objects are Called in Keras](https://adaickalavan.github.io/tensorflow/how-objects-are-called-in-keras/) by Adaickalavan  
256 | 
257 | 6. ### <a name="RFDL"></a>Regularization for Deep Learning  
258 | 
259 | ```
260 | Required Reading:
261 | ```
262 | 
263 |   * [Chapter 7](http://www.deeplearningbook.org/contents/regularization.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br>
264 |   * Slide: [Regularization For Deep Models (Lecture 3)](http://wavelab.uwaterloo.ca/wp-content/uploads/2017/04/Lecture_3.pdf) by Ali Harakeh  
265 |   * Slide: [Bagging and Random Forests](https://davidrosenberg.github.io/mlcourse/Archive/2017/Lectures/9a.bagging-random-forests.pdf) by David Rosenberg <br>
266 |   * Slide: [Deep Learning Tutorial](http://speech.ee.ntu.edu.tw/~tlkagk/slide/Deep%20Learning%20Tutorial%20Complete%20(v3)) (Read the Part of Dropout) by Hung-yi Lee   
267 |  
268 | ```
269 | Suggested Reading:
270 | ```
271 |  * Blog & NoteBook: [How to Build a Neural Network with Keras Using the IMDB Dataset](https://builtin.com/data-science/how-build-neural-network-keras) by Niklas Donges  
272 |  * Blog & NoteBook: [Neural Network Weight Regularization](https://chrisalbon.com/deep_learning/keras/neural_network_weight_regularization/) by Chris Albon   
273 |  * Blog: [Train Neural Networks With Noise to Reduce Overfitting](https://machinelearningmastery.com/train-neural-networks-with-noise-to-reduce-overfitting/) by Jason Brownlee  
274 |  * Blog & NoteBook: [How to Improve Deep Learning Model Robustness by Adding Noise](https://machinelearningmastery.com/how-to-improve-deep-learning-model-robustness-by-adding-noise/) by Jason Brownlee   
275 |  * Paper: [Ensemble Methods in Machine Learnin](http://web.engr.oregonstate.edu/~tgd/publications/mcs-ensembles.pdf) by Thomas G. Dietterich <br>
276 |  * Paper: [Dropout: A Simple Way to Prevent Neural Networks from Overfitting](http://jmlr.org/papers/volume15/srivastava14a.old/srivastava14a.pdf) by Nitish Srivastava, Geoffrey Hinton, Alex Krizhevsky, Ilya Sutskever, Ruslan Salakhutdinov  
277 | 
278 | ```
279 | Additional Reading:
280 | ```
281 |     
282 |   * Blog: [TensorFlow 2.0 Tutorial 04: Early Stopping](https://lambdalabs.com/blog/tensorflow-2-0-tutorial-04-early-stopping/) by Chuan Li   
283 |   * Blog: [Analysis of Dropout](https://pgaleone.eu/deep-learning/regularization/2017/01/10/anaysis-of-dropout/) by Paolo Galeone  
284 |   * **Extra Slides:**
285 |     - Slide: [Regularization for Deep Learning](https://www.deeplearningbook.org/slides/07_regularization.pdf)  by Ian Goodfellow
286 |     - Slides: Regularization for Deep Learning [1](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L13-regularization.pdf)  and [2](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L14-regularization-2.pdf) by U Kang 
287 |     - Slide: [Training Deep Neural Networks](https://web.cs.hacettepe.edu.tr/~aykut/classes/spring2018/cmp784/slides/lec4-training-deep-nets.pdf) by Aykut Erdem   
288 | 
289 | 7. ### <a name="OFTDM"></a>Optimization for Training Deep Models  
290 | 
291 | ```
292 | Required Reading:
293 | ```  
294 | 
295 |    * [Chapter 8](http://www.deeplearningbook.org/contents/optimization.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br> 
296 |    * Slide: [Optimization for Training Deep Models (Lecture 4)](http://wavelab.uwaterloo.ca/wp-content/uploads/2017/04/Lecture-4-1.pdf) by Ali Harakeh  
297 |    * Slide: [Optimization for Training Deep Models - Algorithms (Lecture 4)](http://wavelab.uwaterloo.ca/wp-content/uploads/2017/04/Lecture_4_2-1.pdf) by Ali Harakeh  
298 |    * Blog: [Batch Normalization in Deep Networks](https://www.learnopencv.com/batch-normalization-in-deep-networks/) by Sunita Nayak  
299 |    
300 | 
301 | ```
302 | Suggested Reading:
303 | ```
304 | 
305 |    * Lecture Note: [Matrix Norms and Condition Numbers](http://faculty.nps.edu/rgera/MA3042/2009/ch7.4.pdf) by Ralucca Gera  
306 |    * Blog: [Initializing Neural Networks](https://www.deeplearning.ai/ai-notes/initialization/) by Katanforoosh & Kunin, [deeplearning.ai](https://www.deeplearning.ai), 2019   
307 |    * Blog: [How to Initialize Deep Neural Networks? Xavier and Kaiming Initialization](https://pouannes.github.io/blog/initialization/) by Pierre Ouannes  
308 |    * Blog: [What Is Covariate Shift?](https://medium.com/@izadi/what-is-covariate-shift-d7a7af541e6) by Saeed Izadi 
309 |    * Blog: [Stay Hungry, Stay Foolish:](https://www.adityaagrawal.net/blog/) This interesting blog contains the computation of back propagation of different layers of deep learning prepared by Aditya Agrawal    
310 |    
311 | ```
312 | Additional Reading:
313 | ```
314 |    * Blog: [Why Momentum Really Works](https://distill.pub/2017/momentum/) by Gabriel Goh  
315 |    * Blog: [Understanding the Backward Pass Through Batch Normalization Layer](https://kratzert.github.io/2016/02/12/understanding-the-gradient-flow-through-the-batch-normalization-layer.html) by Frederik Kratzert   
316 |    * [Video](https://www.youtube.com/watch?v=Xogn6veSyxA) of lecture / discussion: This video covers a presentation by Ian Goodfellow and group discussion on the end of Chapter 8 and entirety of Chapter 9 at a reading group in San Francisco organized by Taro-Shigenori Chiba. <br>         
317 |    * Blog: [Preconditioning the Network](https://cnl.salk.edu/~schraudo/teach/NNcourse/precond.html) by Nic Schraudolph and Fred Cummins  
318 |    * Paper: [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/pdf/1502.01852.pdf) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun  
319 |    * Blog: [Neural Network Optimization](https://towardsdatascience.com/neural-network-optimization-7ca72d4db3e0) by Matthew Stewart  
320 |    * Paper: [Understanding the Disharmony between Dropout and Batch Normalization by Variance Shift](https://arxiv.org/pdf/1801.05134.pdf) by Xiang Li, Shuo Chen, Xiaolin Hu, and Jian Yang   
321 |    * **Extra Slides:**  
322 |     - Slide: [Conjugate Gradient Descent](http://www.cs.cmu.edu/~pradeepr/convexopt/Lecture_Slides/conjugate_direction_methods.pdf) by Aarti Singh  
323 |     - Slide: [Training Deep Neural Networks](https://web.cs.hacettepe.edu.tr/~aykut/classes/spring2018/cmp784/slides/lec4-training-deep-nets.pdf) by Aykut Erdem   
324 |     - Slides: Optimization for Training Deep Models [1](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L15-opt.pdf)  and [2](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L16-opt-2.pdf) by U Kang  
325 |     
326 | 8. ### <a name="Part-4"></a>Toolkit Lab 4: Training Deep Neural Networks  
327 | ```
328 | Required Reading:
329 | ```
330 |     
331 |   * NoteBook: [Chapter 11 – Training Deep Neural Networks](https://github.com/ageron/handson-ml2/blob/master/11_training_deep_neural_networks.ipynb) from [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron      
332 | 
333 | ```
334 | Suggested Reading:
335 | ```
336 | 
337 |   * Blog: [How to Accelerate Learning of Deep Neural Networks With Batch Normalization](https://machinelearningmastery.com/how-to-accelerate-learning-of-deep-neural-networks-with-batch-normalization/) by Jason Brownlee 
338 |   * Blog: [Why is my Validation Loss Lower than my Training Loss?](https://www.pyimagesearch.com/2019/10/14/why-is-my-validation-loss-lower-than-my-training-loss/) by Adrian Rosebrock   
339 |     
340 | ```
341 | Additional Resources:
342 | ``` 
343 |   * PDF: [Self-Normalizing Neural Networks](https://arxiv.org/pdf/1706.02515.pdf) by Günter Klambauer, Thomas Unterthiner, Andreas Mayr, and Sepp Hochreiter  
344 |   
345 |   
346 | 9. ### <a name="Part-5"></a>Toolkit Lab 5: Custom Models and Training with TensorFlow 2.0  
347 | ```
348 | Required Reading:
349 | ```
350 | 
351 |   * NoteBook: [Chapter 12 – Custom Models and Training with TensorFlow](https://github.com/ageron/handson-ml2/blob/master/12_custom_models_and_training_with_tensorflow.ipynb) from [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron    
352 |   * Slide: [Introducing tf.data:](https://docs.google.com/presentation/d/16kHNtQslt-yuJ3w8GIx-eEH6t_AvFeQOchqGRFpAD7U/edit#slide=id.g254d08e080_0_38) The [tf.data](https://www.tensorflow.org/beta/guide/data) module contains a collection of classes that allows you to easily load data, manipulate it, and pipe it into your model. The slides were prepared by Derek Murray, the creator of tf.data explaining the API (don’t forget to read the speaker notes below the slides). 
353 |   * NoteBook: [Chapter 13 – Loading and Preprocessing Data with TensorFlow](https://github.com/ageron/handson-ml2/blob/master/13_loading_and_preprocessing_data.ipynb) from [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron    
354 | 
355 | ```
356 | Suggested Reading:
357 | ```
358 | 
359 |   * Blog: [What’s Coming in TensorFlow 2.0](https://medium.com/tensorflow/whats-coming-in-tensorflow-2-0-d3663832e9b8) by the TensorFlow Team  <br>
360 |   * Blog: [TF.data Reborn from the Ashes](https://medium.com/@prince.canuma/tf-data-reborn-from-the-ashes-5600512c27d6) by Prince Canuma   
361 |   * Blog: [Introducing Ragged Tensors](https://medium.com/tensorflow/introducing-ragged-tensors-ac301c31fd38) by Laurence Moroney <br>
362 |   * Blog & NoteBook: [Load Images with tf.data (A File from a URL, If It is not Already in the Cache)](https://www.tensorflow.org/beta/tutorials/load_data/images)
363 |   * Blog: [How to use Dataset and Iterators in Tensorflow with Code Samples](https://medium.com/ymedialabs-innovation/how-to-use-dataset-and-iterators-in-tensorflow-with-code-samples-3bb98b6b74ab) by Prasad Pai  
364 |   * Blog: Analyzing tf.function to Discover AutoGraph Strengths and Subtleties: [Part 1](https://pgaleone.eu/tensorflow/tf.function/2019/03/21/dissecting-tf-function-part-1/), [Part 2](https://pgaleone.eu/tensorflow/tf.function/2019/04/03/dissecting-tf-function-part-2/), and [Part 3](https://pgaleone.eu/tensorflow/tf.function/2019/05/10/dissecting-tf-function-part-3/)  by Paolo Galeone   
365 |   * Blog: [TPU-Speed Data Pipelines: tf.data.Dataset and TFRecords](https://codelabs.developers.google.com/codelabs/keras-flowers-data/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
366 |   
367 | ```
368 | Additional Resources:
369 | ```
370 |   
371 |   * Blog: [Building a Data Pipeline (Using Tensorflow 1 and tf.data for Text and Images)](http://cs230.stanford.edu/blog/datapipeline/)
372 |   * Blog: [Swift](https://swift.org) was announced in 2014. The Swift programming language has quickly become one of the fastest growing languages in history. Swift makes it easy to write software that is incredibly fast and safe by design. 
373 |   * GitHub: [Swift for TensorFlow](https://github.com/tensorflow/swift)  
374 |   
375 | ```
376 | TensorFlow 1.0:
377 | ```
378 |   * To Learn TensorFlow 1.0, Check the Section of [TensorFlow-1](https://github.com/hhaji/Deep-Learning/blob/master/TensorFlow-1). 
379 | 
380 | 10. ### <a name="CNN"></a>Convolutional Networks  
381 | 
382 | ```
383 | Required Reading:
384 | ```
385 | 
386 |    * [Chapter 9](http://www.deeplearningbook.org/contents/convnets.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br> 
387 |    * Slide: [Convolutional Neural Networks (Lecture 6)](http://wavelab.uwaterloo.ca/wp-content/uploads/2017/04/Lecture_6.pdf) by Ali Harakeh   
388 |    * Slide: [Convolutional Networks](http://www.deeplearningbook.org/slides/09_conv.pdf)  by Ian Goodfellow  <br>
389 |   
390 | ```
391 | Suggested Reading:
392 | ```
393 | 
394 |    * Blog: [Convolutional Neural Networks CheatSheet](https://stanford.edu/~shervine/teaching/cs-230/cheatsheet-convolutional-neural-networks) by Afshine Amidi and Shervine Amidi  
395 |    * NoteBook: [Chapter 14 – Deep Computer Vision Using Convolutional Neural Networks](https://github.com/ageron/handson-ml2/blob/master/14_deep_computer_vision_with_cnns.ipynb) from [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow (2nd Edition)](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/) by Aurelien Geron  
396 |    * Blog: [Image Convolution Examples](http://aishack.in/tutorials/image-convolution-examples/) by Utkarsh Sinha  
397 |    * Blog: [Convolutions and Backpropagations](https://medium.com/@pavisj/convolutions-and-backpropagations-46026a8f5d2c) by Pavithra Solai  
398 |    * Blog: [Understanding Convolutions](http://colah.github.io/posts/2014-07-Understanding-Convolutions/) by Christopher Olah <br> 
399 |    * Blog: [A Comprehensive Guide to Convolutional Neural Networks — the ELI5 Way](https://towardsdatascience.com/a-comprehensive-guide-to-convolutional-neural-networks-the-eli5-way-3bd2b1164a53) by Sumit Saha  
400 |    * Blog: [A Basic Introduction to Separable Convolutions](https://towardsdatascience.com/a-basic-introduction-to-separable-convolutions-b99ec3102728) by Chi-Feng Wang  
401 |    * Blog: [Depth wise Separable Convolutional Neural Networks](https://www.geeksforgeeks.org/depth-wise-separable-convolutional-neural-networks/) by Mayank Chaurasia  
402 |    * Blog: [Type of convolutions: Deformable and Transformable Convolution](https://towardsdatascience.com/type-of-convolutions-deformable-and-transformable-convolution-1f660571eb91) by Ali Raza  
403 |    * Blog: [Review: DilatedNet — Dilated Convolution (Semantic Segmentation)](https://towardsdatascience.com/review-dilated-convolution-semantic-segmentation-9d5a5bd768f5) by Sik-Ho Tsang  
404 |    * Blog: [Region of Interest Pooling Explained](https://deepsense.ai/region-of-interest-pooling-explained/) by Tomasz Grel     
405 | 
406 | ```
407 | Additional Reading:  
408 | ```  
409 |   
410 |    * Blog: [A Convolutional Neural Network Tutorial in Keras and TensorFlow 2](https://www.machineislearning.com/convolutional-neural-network-keras-tensorflow-2/) by Isak Bosman <br>
411 |    * Blog & NoteBook: [Cats and Dogs Image Classification Using Keras](https://pythonistaplanet.com/image-classification-using-deep-learning/) by Ashwin Joy   
412 |    * Blog & NoteBook: [Learn Tensorflow 3: Introduction to Convolutions](https://codelabs.developers.google.com/codelabs/tensorflow-lab3-convolutions/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)
413 |    * Blog & NoteBook: [Learn Tensorflow 4: Convolutional Neural Networks (CNNs)](https://codelabs.developers.google.com/codelabs/tensorflow-lab4-cnns/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)    
414 |    * Blog & NoteBook: [Learn Tensorflow 5: Complex Images](https://codelabs.developers.google.com/codelabs/tensorflow-lab5-compleximages/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
415 |    * Blog & NoteBook: [Learn Tensorflow 6: Use CNNS with Larger Datasets](https://codelabs.developers.google.com/codelabs/tensorflow-lab6-largecnns/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
416 |    * Blog & NoteBook: [Convolutional Neural Networks, with Keras and TPUs](https://codelabs.developers.google.com/codelabs/keras-flowers-convnets/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
417 |    * Blog & NoteBook: [Modern Convnets, Squeezenet, with Keras and TPUs](https://codelabs.developers.google.com/codelabs/keras-flowers-squeezenet/) by [Google Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)  
418 |    * Blog & NoteBook: [TensorFlow 2.0 Tutorial 01: Basic Image Classification](https://lambdalabs.com/blog/tensorflow-2-0-tutorial-01-image-classification-basics/) by Chuan Li  
419 |    
420 | ```
421 | Fourier Transformation:
422 | ```   
423 | 
424 |    * Blog: [Fourier Transformation and Its Mathematics](https://towardsdatascience.com/fourier-transformation-and-its-mathematics-fff54a6f6659) by Akash Dubey    
425 |    * Blog: [Fourier Transformation for a Data Scientist](https://towardsdatascience.com/fourier-transformation-for-a-data-scientist-1f3731115097) by Nagesh Singh Chauhan        
426 |    * Blog: [Purrier Series (Meow) and Making Images Speak](http://bilimneguzellan.net/en/purrier-series-meow-and-making-images-speak/) by Bilim Ne Güzel Lan   
427 |    * Blog: [Follow up to Fourier Series](http://bilimneguzellan.net/en/follow-up-to-fourier-series-2/) by Bilim Ne Güzel Lan  
428 | 
429 | 
430 | 11. ### <a name="Part-6"></a>Toolkit Lab 6: TensorBoard    
431 |    
432 | ```
433 | TensorBoard:
434 | ```
435 |     
436 |   * Video: [Inside TensorFlow: Summaries and TensorBoard](https://www.youtube.com/watch?v=OI4cskHUslQ)
437 |   * Blog: [TensorBoard Overview](https://www.tensorflow.org/tensorboard/r1/overview)
438 |   * NoteBook: [Get started with TensorBoard](https://github.com/tensorflow/tensorboard/blob/master/docs/get_started.ipynb)
439 |   * NoteBook: [Examining the TensorFlow Graph](https://github.com/tensorflow/tensorboard/blob/master/docs/graphs.ipynb)
440 |   * NoteBook: [Displaying Image Data in TensorBoard](https://github.com/tensorflow/tensorboard/blob/master/docs/image_summaries.ipynb) 
441 |   * NoteBook: [Using TensorBoard in Notebooks](https://github.com/tensorflow/tensorboard/blob/master/docs/tensorboard_in_notebooks.ipynb)
442 |   
443 | ```
444 | Suggested Reading:
445 | ```
446 |   
447 |   * Blog & NoteBook: [TensorBoard: Graph Visualization](https://www.tensorflow.org/tensorboard/r1/graphs)
448 |   * Blog & NoteBook: [TensorBoard Histogram Dashboard](https://www.tensorflow.org/tensorboard/r1/histograms)
449 |   * Blog & NoteBook: [TensorBoard: Visualizing Learning](https://www.tensorflow.org/tensorboard/r1/summaries)
450 |   
451 | ```
452 | Additional Reading:
453 | ```
454 |   * NoteBook: [TensorBoard Scalars: Logging Training Metrics in Keras](https://github.com/tensorflow/tensorboard/blob/master/docs/scalars_and_keras.ipynb)
455 |   * NoteBook: [Hyperparameter Tuning with the HParams Dashboard](https://github.com/tensorflow/tensorboard/blob/master/docs/hyperparameter_tuning_with_hparams.ipynb)
456 |   * NoteBook: [TensorBoard Profile: Profiling basic training metrics in Keras](https://github.com/tensorflow/tensorboard/blob/master/docs/tensorboard_profiling_keras.ipynb)
457 |   * Blog: [TensorFlow 2.0 Tutorial 03: Saving Checkpoints](https://lambdalabs.com/blog/tensorflow-2-0-tutorial-03-saving-checkpoints/) by Chuan Li  
458 | 
459 | 
460 | 12. ### <a name="SMRARN"></a>Sequence Modeling: Recurrent and Recursive Networks  
461 | 
462 | ```
463 | Required Reading:
464 | ```
465 |   
466 |    * [Chapter 10](http://www.deeplearningbook.org/contents/rnn.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br>
467 |    * Slide: [Sequence Modeling: Recurrent and Recursive Networks](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L12-rnn.pdf) by U Kang <br> 
468 |    * Slide: [Training Recurrent Nets](http://web.eecs.utk.edu/~hqi/deeplearning/lecture14-rnn-training.pdf) by Arvind Ramanathan  
469 |    * Slide: [Long-Short Term Memory and Other Gated RNNs](https://cedar.buffalo.edu/~srihari/CSE676/10.10%20LSTM.pdf) by Sargur Srihari  
470 | 
471 | ```
472 | Suggested Reading:
473 | ```
474 |  
475 |    * Blog: [Understanding LSTM Networks](http://colah.github.io/posts/2015-08-Understanding-LSTMs/) by Christopher Olah  <br>
476 |    * Blog: [Illustrated Guide to LSTM’s and GRU’s: A Step by Step Explanation](https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21) by Michael Nguyen  
477 |   
478 | ```
479 | Additional Reading:
480 | ```
481 |   
482 |    * [Video](https://www.youtube.com/watch?v=ZVN14xYm7JA&feature=youtu.be) of lecture / discussion. This video covers a presentation by Ian Goodfellow and a group discussion of Chapter 10 at a reading group in San Francisco organized by Alena Kruchkova. <br>
483 |    * Blog: [Gentle introduction to Echo State Networks](https://towardsdatascience.com/gentle-introduction-to-echo-state-networks-af99e5373c68) by Madalina Ciortan  <br>
484 |    * Blog: [Understanding GRU Networks](https://towardsdatascience.com/understanding-gru-networks-2ef37df6c9be) by Simeon Kostadinov  <br>
485 |    * Blog: [Animated RNN, LSTM and GRU](https://towardsdatascience.com/animated-rnn-lstm-and-gru-ef124d06cf45) by Raimi Karim <br>
486 |    * Slide: [An Introduction to: Reservoir Computing and Echo State Networks](http://didawiki.di.unipi.it/lib/exe/fetch.php/magistraleinformatica/aa2/rnn4-esn.pdf) by Claudio Gallicchio <br>
487 | 
488 |     
489 | 13. ### <a name="Practical-Methodology"></a>Practical Methodology  
490 | 
491 | ```
492 | Required Reading:
493 | ```
494 | 
495 |    * [Chapter 11](http://www.deeplearningbook.org/contents/guidelines.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook.  
496 |    * Slides: [Practical Methodology](https://cedar.buffalo.edu/~srihari/CSE676/) by Sargur Srihari
497 |       - Part 0: [Practical Design Process](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.0%20PractMethOverview.pdf)
498 |       - Part 1: [Performance Metrics](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.1%20PerformMetrics.pdf)
499 |       - Part 2: [Default Baseline Models](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.2%20BaselineModels.pdf)
500 |       - Part 3: [Whether to Gather More Data](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.3%20MoreData.pdf)
501 |       - Part 4: [Selecting Hyperparameters](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.4%20Hyperparams.pdf)
502 |       - Part 5: [Debugging Strategies](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/11.5%20Debugging.pdf)
503 | 
504 | ```
505 | Suggested Reading:
506 | ```
507 | 
508 |   * Metrics:
509 |        - Blog: [Demystifying KL Divergence](https://medium.com/activating-robotic-minds/demystifying-kl-divergence-7ebe4317ee68) by Naoki Shibuya  
510 |        - Blog: [Demystifying Cross-Entropy](https://medium.com/activating-robotic-minds/demystifying-cross-entropy-e80e3ad54a8) by Naoki Shibuya  
511 |        - Blog: [Deep Quantile Regression](https://towardsdatascience.com/deep-quantile-regression-c85481548b5a) by Sachin Abeywardana  
512 |        - Blog: [An Illustrated Guide to the Poisson Regression Model](https://towardsdatascience.com/an-illustrated-guide-to-the-poisson-regression-model-50cccba15958) by Sachin Date   
513 |        - Blog: [Generalized Linear Models](https://towardsdatascience.com/generalized-linear-models-8738ae0fb97d) by Semih Akbayrak  
514 |        - Blog: [ROC curves and Area Under the Curve Explained (Video)](https://www.dataschool.io/roc-curves-and-auc-explained/) by Data School
515 |        - Blog: [Introduction to the ROC (Receiver Operating Characteristics) Plot](https://classeval.wordpress.com/introduction/introduction-to-the-roc-receiver-operating-characteristics-plot/) 
516 |        - Slide: [ROC Curves](https://web.uvic.ca/~maryam/DMSpring94/Slides/9_roc.pdf) by Maryam Shoaran  
517 |        - Blog: [Precision-Recall Curves](https://www.andybeger.com/2015/03/16/precision-recall-curves/) by Andreas Beger  
518 | 
519 | ```
520 | Additional Reading:
521 | ```
522 |  
523 |    * Slide: [Practical Methodology](http://www.deeplearningbook.org/slides/11_practical.pdf)  by Ian Goodfellow  <br>
524 |    * Slide: [Practical Methodology](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L17-practical-method.pdf) by U Kang <br> 
525 |    * Paper: [The Relationship Between Precision-Recall and ROC Curves](https://www.biostat.wisc.edu/~page/rocpr.pdf) by Jesse Davis and Mark Goadrich     
526 | 
527 |     
528 | 14. ### <a name="Applications"></a>Applications 
529 | 
530 | ```
531 | Required Reading:
532 | ```
533 |    * [Chapter 12](http://www.deeplearningbook.org/contents/applications.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br>
534 |    * Slide: [Applications](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L18-applications.pdf) by U Kang  
535 |    
536 | ```
537 | Suggested Reading:
538 | ```
539 | 
540 |    * Blog: [How Neural Networks Learn Distributed Representations](https://www.oreilly.com/ideas/how-neural-networks-learn-distributed-representations) By Garrett Hoffman <br>
541 | 
542 | ```
543 | Additional Reading:
544 | ```
545 |    * Blog: [30 Amazin Applications of Deep Learning](http://www.yaronhadad.com/deep-learning-most-amazing-applications/) by Yaron Hadad  
546 |    * Slides: [Applications](https://cedar.buffalo.edu/~srihari/CSE676/) by Sargur Srihari  
547 | 
548 |     
549 | 15. ### <a name="Autoencoders"></a>Autoencoders
550 | 
551 | ```
552 | Required Reading:
553 | ```
554 |    * [Chapter 14](http://www.deeplearningbook.org/contents/autoencoders.html) of the [Deep Learning](http://www.deeplearningbook.org) textbook. <br>
555 |    * Slide: [Autoencoders](http://www.cedar.buffalo.edu/%7Esrihari/CSE676/14%20Autoencoders.pdf) by Sargur Srihari  
556 |    * Blog: [Understanding Variational Autoencoders (VAEs)](https://towardsdatascience.com/understanding-variational-autoencoders-vaes-f70510919f73) by Joseph Rocca  
557 |    * Blog: [Tutorial - What is a Variational Autoencoder?](https://jaan.io/what-is-variational-autoencoder-vae-tutorial/) by Jaan Altosaar 
558 |   
559 | ```
560 | Suggested Reading:
561 | ```
562 | 
563 |    * Slide: [Variational Autoencoders](http://slazebni.cs.illinois.edu/spring17/lec12_vae.pdf) by Raymond Yeh, Junting Lou, and Teck-Yian Lim  
564 |    * Blog: [Autoencoders vs PCA: When to Use?](https://towardsdatascience.com/autoencoders-vs-pca-when-to-use-which-73de063f5d7) by Urwa Muaz  
565 |    * Blog: [Intuitively Understanding Variational Autoencoder: 
566 | And Why They’re so Useful in Creating Your Own Generative Text, Art and Even Music](https://towardsdatascience.com/intuitively-understanding-variational-autoencoders-1bfe67eb5daf) by Irhum Shafkat  
567 |    * Blog: [Generative Modeling: What is a Variational Autoencoder (VAE)?](https://www.mlq.ai/what-is-a-variational-autoencoder/) by Peter Foy  
568 |    * Slide: [Generative Models](https://hpi.de/fileadmin/user_upload/fachgebiete/meinel/team/Haojin/competitive_problem_solving_with_deep_learning/Class_Generative_Models.pptx.pdf) by Mina Rezaei  
569 |    * Blog: [A High-Level Guide to Autoencoders](https://towardsdatascience.com/a-high-level-guide-to-autoencoders-b103ccd45924) by Shreya Chaudhary  
570 |    * Blog: [Variational Autoencoder: Intuition and Implementation](https://wiseodd.github.io/techblog/2016/12/10/variational-autoencoder/) by Agustinus Kristiadi     
571 |    * Blog: [Conditional Variational Autoencoder: Intuition and Implementation](https://wiseodd.github.io/techblog/2016/12/17/conditional-vae/) by Agustinus Kristiadi    
572 |     
573 | ```
574 | Additional Reading:
575 | ```
576 |     
577 |    * Blog: [Tutorial - What is a Variational Autoencoder?](https://jaan.io/what-is-variational-autoencoder-vae-tutorial/) by Jaan Altosaar <br>
578 |    * Slide: [Autoencoders](https://datalab.snu.ac.kr/~ukang/courses/17S-DL/L19-autoencoder.pdf) by U Kang <br> 
579 |     
580 | 16. ### <a name="GAN"></a>Generative Adversarial Networks  
581 | 
582 | ```
583 | Required Reading:
584 | ```
585 | 
586 | Slide: [Generative Adversarial Networks (GANs)](http://slazebni.cs.illinois.edu/spring17/lec11_gan.pdf) by Binglin, Shashank, and Bhargav  
587 | Paper: [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/pdf/1701.00160.pdf) by Ian Goodfellow  
588 |   
589 | ```
590 | Suggested Reading:
591 | ```
592 | 
593 | * Blog: [Generative Adversarial Networks (GANs), Some Open Questions](https://www.offconvex.org/2017/03/15/GANs/) by Sanjeev Arora   
594 | * Paper: [Generative Adversarial Networks: An Overview](https://arxiv.org/pdf/1710.07035.pdf) by Antonia Creswell, Tom White, Vincent Dumoulin, Kai Arulkumaran, Biswa Sengupta, and Anil A Bharath  
595 | 
596 | ```
597 | Additional Reading:
598 | ```
599 | 
600 | * Blog: [GANs Comparison Without Cherry-Picking](https://github.com/khanrc/tf.gans-comparison) by Junbum Cha  
601 | * Blog: [New Progress on GAN Theory and Practice](https://casmls.github.io/general/2017/04/13/gan.html) by Liping Liu  
602 | * Blog: [Play with Generative Adversarial Networks (GANs) in your browser!](https://poloclub.github.io/ganlab/)  
603 | * Blog: [The GAN Zoo](https://github.com/hindupuravinash/the-gan-zoo) by Avinash Hindupur  
604 | * [Generative Adversarial Networks (GANs), Some Open Questions](https://www.offconvex.org/2017/03/15/GANs/) by Sanjeev Arora  
605 | 
606 | 17. ### <a name="GNN"></a>Graph Neural Networks:     
607 | 
608 | ```
609 | Required Reading:
610 | ```
611 | 
612 | - Slide: [Graph Neural Networks](http://ir.hit.edu.cn/~xiachongfeng/slides/Graph%20Neural%20Networks.pdf) by Xiachong Feng  
613 | - Paper: [A Comprehensive Survey on Graph Neural Networks](https://arxiv.org/pdf/1901.00596.pdf) by Zonghan Wu, Shirui Pan, Fengwen Chen, Guodong Long, Chengqi Zhang, Philip S. Yu  
614 | 
615 | ```
616 | Suggested Reading:
617 | ```
618 | 
619 | - Book: [Graph Representation Learning](https://www.cs.mcgill.ca/~wlh/grl_book/) by William L. Hamilton    
620 | - Blog: [Deep Graph Library (DGL)](https://www.dgl.ai): A Python package that interfaces between existing tensor libraries and data being expressed as graphs.   
621 | 
622 | ```
623 | Additional Reading:
624 | ```  
625 | 
626 | - GitHub: [Graph Neural Networks](https://github.com/hhaji/Deep-Learning/tree/master/Graph-Neural-Networks)   
627 |     
628 | ### <a name="ANAS"></a>Additional Resources:    
629 | - Papers:  
630 |   * [Papers with Code:](https://paperswithcode.com) The mission of Papers With Code is to create a free and open resource with Machine Learning papers, code and evaluation tables.
631 |   * [Deep Learning Papers Reading Roadmap](https://github.com/floodsung/Deep-Learning-Papers-Reading-Roadmap) by Flood Sung <br>
632 |   * [Awesome - Most Cited Deep Learning Papers](https://github.com/terryum/awesome-deep-learning-papers) by  Terry Taewoong Um <br>
633 |  - Deep Learning Courses:  
634 |     * [Tensorflow for Deep Learning Research](http://web.stanford.edu/class/cs20si/syllabus.html) by Chip Huyen
635 |     * [Deep Learning](https://web.cs.hacettepe.edu.tr/~aykut/classes/spring2018/cmp784/index.html) by Aykut Erdem <br>
636 | - Program:  
637 |   * [Ludwig](https://eng.uber.com/introducing-ludwig/) is a toolbox built on top of TensorFlow that allows to train and test deep learning models without the need to write code.
638 |         - [Installation](https://github.com/uber/ludwig)
639 |   * [TensorFlow Playground:](https://playground.tensorflow.org) an interactive visualization of neural networks, written in typescript using d3.js by Daniel Smilkov and Shan Carter   
640 | - The blog of [Christopher Olah:](http://colah.github.io) Fascinating tutorials about neural networks  
641 | - The blog of [Adit Deshpande:](https://adeshpande3.github.io/adeshpande3.github.io/) The Last 5 Years In Deep Learning  
642 | - [Fascinating Tutorials on Deep Learning](https://r2rt.com/)   
643 | - [Deep Learning (Faster Data Science Education by Kaggle)](https://www.kaggle.com/learn/deep-learning) by Dan Becker   
644 | 
645 | ## <a name="Class-Time-and-Location"></a>Class Time and Location:
646 | Sunday and Tuesday 13:00-14:30 AM (Fall 2019)
647 | 
648 | ## <a name="Projects"></a>Projects:
649 | Projects are programming assignments that cover the topic of this course. Any project is written by **[Jupyter Notebook](http://jupyter.org)**. Projects will require the use of Python 3.7, as well as additional Python libraries. 
650 | 
651 | * [Get Started with TensorFlow](https://www.tensorflow.org/tutorials/)
652 | 
653 | ### <a name="Google-Colab"></a>Google Colab:
654 | [Google Colab](https://colab.research.google.com) is a free cloud service and it supports free GPU! 
655 |   - [How to Use Google Colab](https://www.geeksforgeeks.org/how-to-use-google-colab/) by Souvik Mandal <br> 
656 |   - [Primer for Learning Google Colab](https://medium.com/dair-ai/primer-for-learning-google-colab-bb4cabca5dd6)
657 |   - [Deep Learning Development with Google Colab, TensorFlow, Keras & PyTorch](https://www.kdnuggets.com/2018/02/google-colab-free-gpu-tutorial-tensorflow-keras-pytorch.html)
658 | 
659 | ### <a name="Fascinating-Guides-For-Machine-Learning"></a>Fascinating Guides For Machine Learning:
660 | * [Technical Notes On Using Data Science & Artificial Intelligence: To Fight For Something That Matters](https://chrisalbon.com) by Chris Albon
661 | 
662 | ### <a name="Latex"></a>Latex:
663 | The students can include mathematical notation within markdown cells using LaTeX in their **[Jupyter Notebooks](http://jupyter.org)**.<br>
664 |   - A Brief Introduction to LaTeX [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/latex.pdf)  <br>
665 |   - Math in LaTeX [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/math.pdf) <br>
666 |   - Sample Document [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/sample.pdf) <br>
667 |   - [TikZ:](https://github.com/PetarV-/TikZ) A collection Latex files of PGF/TikZ figures (including various neural networks) by Petar Veličković. 
668 | 
669 | ## <a name="Grading"></a>Grading:
670 | * Projects and Midterm – 50%
671 | * Endterm – 50%
672 | 
673 | ## <a name="Prerequisites"></a>Prerequisites:
674 | General mathematical sophistication; and a solid understanding of Algorithms, Linear Algebra, and 
675 | Probability Theory, at the advanced undergraduate or beginning graduate level, or equivalent.
676 | 
677 | ### <a name="Linear-Algebra"></a>Linear Algebra:
678 | * Video: Professor Gilbert Strang's [Video Lectures](https://ocw.mit.edu/courses/mathematics/18-06-linear-algebra-spring-2010/video-lectures/) on linear algebra.
679 | 
680 | ### <a name="Probability-and-Statistics"></a>Probability and Statistics:
681 | * [Learn Probability and Statistics Through Interactive Visualizations:](https://seeing-theory.brown.edu/index.html#firstPage) Seeing Theory was created by Daniel Kunin while an undergraduate at Brown University. The goal of this website is to make statistics more accessible through interactive visualizations (designed using Mike Bostock’s JavaScript library D3.js).
682 | * [Statistics and Probability:](https://stattrek.com) This website provides training and tools to help you solve statistics problems quickly, easily, and accurately - without having to ask anyone for help.
683 | * Jupyter NoteBooks: [Introduction to Statistics](https://github.com/rouseguy/intro2stats) by Bargava
684 | * Video: Professor John Tsitsiklis's [Video Lectures](https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/) on Applied Probability.
685 | * Video: Professor Krishna Jagannathan's [Video Lectures](https://nptel.ac.in/courses/108106083/) on Probability Theory.
686 | 
687 | ## <a name="Topics"></a>Topics:
688 | Have a look at some reports of [Kaggle](https://www.kaggle.com/) or Stanford students ([CS224N](http://nlp.stanford.edu/courses/cs224n/2015/), [CS224D](http://cs224d.stanford.edu/reports_2016.html)) to get some general inspiration.
689 | 
690 | ## <a name="Account"></a>Account:
691 | It is necessary to have a [GitHub](https://github.com/) account to share your projects. It offers 
692 | plans for both private repositories and free accounts. Github is like the hammer in your toolbox, 
693 | therefore, you need to have it!
694 | 
695 | ## <a name="Academic-Honor-Code"></a>Academic Honor Code:
696 | Honesty and integrity are vital elements of the academic works. All your submitted assignments must be entirely your own (or your own group's).
697 | 
698 | We will follow the standard of Department of Mathematical Sciences approach: 
699 | * You can get help, but you MUST acknowledge the help on the work you hand in
700 | * Failure to acknowledge your sources is a violation of the Honor Code
701 | *  You can talk to others about the algorithm(s) to be used to solve a homework problem; as long as you then mention their name(s) on the work you submit
702 | * You should not use code of others or be looking at code of others when you write your own: You can talk to people but have to write your own solution/code
703 | 
704 | ## <a name="Questions"></a>Questions?
705 | I will be having office hours for this course on Sunday (09:00 AM--10:00 AM). If this is not convenient, email me at hhaji@sbu.ac.ir or talk to me after class.
706 | 
707 | 


--------------------------------------------------------------------------------
/Graph-Neural-Networks/Install-Graph-Nets.md:
--------------------------------------------------------------------------------
 1 | # Install Graph Nets Library
 2 | 
 3 | [Graph Nets](https://github.com/deepmind/graph_nets) is DeepMind's library for building graph networks in Tensorflow and Sonnet. 
 4 | 
 5 | ## Install Graph Nets Library via pip    
 6 | The Graph Nets library can be installed from [pip](https://github.com/deepmind/graph_nets/#Installation).  
 7 | To install the Graph Nets library for CPU, run:  
 8 | 
 9 | $ pip install graph_nets "tensorflow>=1.15,<2" tensorflow_probability  
10 | 
11 | To install the Graph Nets library for GPU, run:
12 | 
13 | $ pip install graph_nets "tensorflow_gpu>=1.15,<2" tensorflow_probability  
14 | 
15 | ## Install Graph Nets Library via Conda  
16 | First, make a conda environment, e.g. GN, as follows:  
17 | 
18 | $ conda create -n GN  
19 | 
20 | Then, activate your environment:  
21 | 
22 | $ conda activate GN  
23 | 
24 | Next, install the requirement packages using conda:  
25 | 
26 | $ conda install python=3.6 tensorflow=1.15 tensorflow-probability=0.8.0 jupyterlab matplotlib  
27 | 
28 | Next, we need to install  dm-sonnet and graph_nets  packages using pip:  
29 | 
30 | $ pip install dm-sonnet graph_nets   
31 | 
32 | Finally, assign a name to your kernel (e.g. GN):  
33 | 
34 | $ python -m ipykernel install --user --name GN --display-name "GN"    
35 | 


--------------------------------------------------------------------------------
/Graph-Neural-Networks/README.md:
--------------------------------------------------------------------------------
  1 | # Graph Neural Networks  
  2 | Graph Neural Networks have received increasing attentions due to their superior performance in many node and graph classification tasks. 
  3 | 
  4 | ### **Index:**
  5 | - [Graph Neural Networks](#Blogs)
  6 |   - [Applications and Limitations of Graph Neural Networks](#ALGNN)
  7 |   - [Video](#Video)
  8 |   - [Survey](#Survey)
  9 |   - [Graph Convolutionl Networks](#GCN)
 10 |   - [Graph Auto-Encoders](#GAE)
 11 |   - [Deep Belief Nets](#DBN)
 12 | - [Graph Represetation Learning](#GRL)
 13 | - [Courses](#Courses) 
 14 | - [Books](#Books)
 15 | - [Graph Neural Networks Libraries](#GNNL) 
 16 |   - [Deep Graph Library (DGL)](#DGL) 
 17 |   - [Node Classification](#NC) 
 18 |   - [Graph Classification](#GC) 
 19 |   - [Graph Nets Library](#GNL) 
 20 |   - [More Libraries](#ML) 
 21 |   - [Save & Load Graphs](#SLG) 
 22 | - [Datasets of Graphs](#DG) 
 23 | - [Tools for Creating Graphs](#TCG) 
 24 | - [Molecular Structure Analysis](#MSA) 
 25 | - [Graph Machine Learning and its Application on Molecular Science](#ML-MS)
 26 | - [Chemical Notations](#Notations) 
 27 | - [An Introduction to Basic Chemistry and Drugs](#Chemistry) 
 28 |    - Chemistry
 29 |    - Drug
 30 | - [Datasets of Molecules and Their Properties](#DMTP)  
 31 |    - Chemical Datasets
 32 |    - Biological Datasets 
 33 | - [Libraries & Packages](#Libraries) 
 34 | - [Online Softwares](#Online-Softwares) 
 35 | - [Softwares (Draw a Molecule)](#Draw-Molecule) 
 36 | 
 37 | ## <a name="Blogs"></a>Graph Neural Networks    
 38 | - Blog: [Awesome Resources on Graph Neural Networks](https://github.com/nnzhan/Awesome-Graph-Neural-Networks) by Zonghan Wu. This is a collection of resources related with graph neural networks.  
 39 | - Blog: [Deep Learning on Graphs: Successes, Challenges, and Next Steps](https://towardsdatascience.com/deep-learning-on-graphs-successes-challenges-and-next-steps-7d9ec220ba8) by Michael Bronstein   
 40 | - Blog: [A Gentle Introduction to Graph Neural Networks](https://distill.pub/2021/gnn-intro/)    
 41 | - Blog: [Graph Convolutionl Networks](http://tkipf.github.io/graph-convolutional-networks/) by Thomas Kipf  
 42 | - Blog: [Graph Convolutional Networks I](https://atcold.github.io/pytorch-Deep-Learning/en/week13/13-1/) by  Xavier Bresson  
 43 | - Blog: [Graph Convolutional Networks II](https://atcold.github.io/pytorch-Deep-Learning/en/week13/13-2/) by Xavier Bresson   
 44 | - Blog: [Graph Convolutional Networks III](https://atcold.github.io/pytorch-Deep-Learning/en/week13/13-3/) by  Alfredo Canziani  
 45 | - Blog: [Emotion Recognition Using Graph Convolutional Networks](https://towardsdatascience.com/emotion-recognition-using-graph-convolutional-networks-9f22f04b244e) by Kevin Shen    
 46 | - Blog & NoteBook: [Graph Convolutional Network](https://docs.dgl.ai/en/0.4.x/tutorials/models/1_gnn/1_gcn.html) by Qi Huang, Minjie Wang, Yu Gai, Quan Gan, and Zheng Zhang  
 47 | - Blog: [An Attempt at Demystifying Graph Deep Learning](https://ericmjl.github.io/essays-on-data-science/machine-learning/graph-nets/) by Eric Ma   
 48 | - Blog: [Deep Learning on Graphs (a Tutorial)](https://cloud4scieng.org/2020/08/28/deep-learning-on-graphs-a-tutorial/) by Gannon  
 49 | - Blog: [Graph Neural Networks and its Variants](https://docs.dgl.ai/en/0.4.x/tutorials/models/)   
 50 | - Blog: [Graph Neural Networks and Recommendations](https://github.com/yazdotai/graph-networks) by Yazdotai
 51 | - Blog: [Must-Read Papers on Graph Neural Networks (GNN)](https://github.com/thunlp/GNNPapers) contributed by Jie Zhou, Ganqu Cui, Zhengyan Zhang and Yushi Bai. 
 52 | - Blog: [A Gentle Introduction to Graph Neural Networks (Basics, DeepWalk, and GraphSage)](https://towardsdatascience.com/a-gentle-introduction-to-graph-neural-network-basics-deepwalk-and-graphsage-db5d540d50b3) by Steeve Huang  
 53 | - Blog: [Deep Learning with Knowledge Graphs](https://medium.com/octavian-ai/deep-learning-with-knowledge-graphs-3df0b469a61a) 
 54 | - Slide: [Graph Neural Networks: Models and Applications](http://cse.msu.edu/~mayao4/tutorials/aaai2020/) by Yao Ma, Wei Jin, Jiliang Tang, Lingfei Wu, and Tengfei Ma   
 55 | 
 56 | ### <a name="ALGNN"></a>Applications and Limitations of Graph Neural Networks
 57 | - Blog: [Applications of Graph Neural Networks](https://towardsdatascience.com/https-medium-com-aishwaryajadhav-applications-of-graph-neural-networks-1420576be574) by Aishwarya Jadhav  
 58 | - Blog: [Exciting Applications of Graph Neural Networks](https://blog.fastforwardlabs.com/2019/10/30/exciting-applications-of-graph-neural-networks.html) by Keita  
 59 | - Blog: [Can Graph Neural Networks Solve Real-World Problems?](https://hackernoon.com/can-graph-neural-networks-solve-real-world-problems-7hd636dn) by Prince Canuma  
 60 | - Blog: [Limitations of Graph Neural Networks](https://towardsdatascience.com/limitations-of-graph-neural-networks-2412fffe677) by Sergei Ivanov   
 61 | 
 62 | ### <a name="Video"></a>Video  
 63 | - Video: [Graph Neural Networks: Variations and Applications](https://www.youtube.com/watch?v=cWIeTMklzNg) 
 64 | 
 65 | ### <a name="Survey"></a>Survey   
 66 | - Paper: [A Comprehensive Survey on Graph Neural Networks](https://arxiv.org/pdf/1901.00596.pdf) by Zonghan Wu, Shirui Pan, Fengwen Chen, Guodong Long, Chengqi Zhang, Philip S. Yu  
 67 | - Paper: [Relational inductive biases, deep learning, and graph networks](https://arxiv.org/pdf/1806.01261.pdf)
 68 | - Paper: [Graph Neural Networks: A Review of Methods and Applications](https://arxiv.org/pdf/1812.08434.pdf) by 
 69 | Jie Zhou, Ganqu Cui, Zhengyan Zhang, Cheng Yang, Zhiyuan Liu, Lifeng Wang, Changcheng Li, Maosong Sun  
 70 | - Paper: [Attention Models in Graphs: A Survey](https://dl.acm.org/doi/10.1145/3363574) by John Boaz Lee, Ryan A Rossi, Sungchul Kim, Nesreen K Ahmed, and Eunyee Koh   
 71 | 
 72 | ### <a name="GAE"></a>Graph Auto-Encoders        
 73 | - Blog: [Tutorial on Variational Graph Auto-Encoders](https://towardsdatascience.com/tutorial-on-variational-graph-auto-encoders-da9333281129) by Fanghao Han  
 74 | 
 75 | ### <a name="DBN"></a>Deep Belief Nets       
 76 | - Blog: [Deep Learning meets Physics: Restricted Boltzmann Machines Part I](https://towardsdatascience.com/deep-learning-meets-physics-restricted-boltzmann-machines-part-i-6df5c4918c15) by Artem Oppermann    
 77 | - Slide: [Deep Belief Nets](https://www.cs.toronto.edu/~hinton/nipstutorial/nipstut3.pdf) by Geoffrey Hinton 
 78 | 
 79 | ## <a name="GRL"></a>Graph Represetation Learning         
 80 | - Tutorial: [Representation Learning on Networks](http://snap.stanford.edu/proj/embeddings-www/index.html) by Jure Leskovec      
 81 | - Papers: [Papers with Code](https://paperswithcode.com/task/graph-representation-learning)
 82 | - Blog: [Graph Represetation Learning](https://towardsdatascience.com/graph-representation-learning-dd64106c9763) by Marco Brambilla   
 83 | - Slide: [Graph Represetation Learning](https://jian-tang.com/files/AAAI19/aaai-grltutorial-part0-intro.pdf) by 
 84 | William L. Hamilton and Jian Tang   
 85 | - Survey: [Representation Learning on Graphs: Methods and Applications](https://www-cs.stanford.edu/people/jure/pubs/graphrepresentation-ieee17.pdf) by William L. Hamilton, Rex Ying, and Jure Leskovec   
 86 | - Video: [Graph Representation Learning (Stanford university)](https://www.youtube.com/watch?v=YrhBZUtgG4E) by Jure Leskovec  
 87 | - Thesis: [Graph Representation Learning and Graph Classification](https://www.cs.uoregon.edu/Reports/AREA-201706-Riazi.pdf) by Sara Riazi   
 88 | - NeurIPS 2019 Workshop ([Graph Represetation Learning](https://grlearning.github.io)): [Open Problems and Challenges](https://grlearning.github.io/papers/)   
 89 | 
 90 | ## <a name="Courses"></a>Courses       
 91 | - Blog: [Machine Learning with Graphs](http://web.stanford.edu/class/cs224w/index.html#schedule) by Jure Leskovec  
 92 | - Blog: [Graph Represetation Learning](https://cs.mcgill.ca/~wlh/comp766/index.html) by William L. Hamilton   
 93 | - Blog: [Graph Neural Networks](https://gnn.seas.upenn.edu) by Alejandro Ribeiro   
 94 | - Blog: [Basics of Graph Neural Networks](https://www.graphneuralnets.com/p/basics-of-gnns/) by Zak Jost  
 95 | 
 96 | 
 97 | ## <a name="Books"></a>Books       
 98 | - Blog: [Introduction to Graph Neural Networks](https://www.amazon.com/Introduction-Networks-Synthesis-Artificial-Intelligence-ebook/dp/B087LJJNJK) by Zhiyuan Liu and Jie Zhou   
 99 | - Blog: [Graph Representation Learning](https://www.morganclaypoolpublishers.com/catalog_Orig/product_info.php?products_id=1576) by William L. Hamilton   
100 |     - [The Pre-Publication](https://www.cs.mcgill.ca/~wlh/grl_book/files/GRL_Book.pdf)
101 | 
102 | ## <a name="GNNL"></a>Graph Neural Networks Libraries       
103 | ### <a name="DGL"></a>Deep Graph Library (DGL)   
104 | A Python package that interfaces between existing tensor libraries and data being expressed as graphs.  
105 | - Library: [Deep Graph Library (DGL)](https://www.dgl.ai) 
106 |     * Install: [DGL](https://docs.dgl.ai/install/index.html)
107 | - Paper: [Deep Graph Library: Towards Efficient and Scalable Deep Learning on Graphs](https://rlgm.github.io/papers/49.pdf) by Minjie Wang, Lingfan Yu, Da Zheng, Quan Gan, Yu Gai, Zihao Ye, Mufei Li, Jinjing Zhou, Qi Huang, Chao Ma, Ziyue Huang, Qipeng Guo, Hao Zhang, Haibin Lin, Junbo Zhao, Jinyang Li, Alexander Smola, and Zheng Zhang  
108 | - Blog: [Hands-on Graph Neural Networks with PyTorch & PyTorch Geometric](https://towardsdatascience.com/hands-on-graph-neural-networks-with-pytorch-pytorch-geometric-359487e221a8) by Huang Kung-Hsiang   
109 | - Blog: [DGL Walkthrough 01: Data](https://xinhaoli74.github.io/posts/2019/12/DGL-Basic01-Data/) by Xinhao Li  
110 | - Blog: [When Kernel Fusion Meets Graph Neural Networks](https://www.dgl.ai/blog/2019/05/04/kernel.html) By Minjie Wang, Lingfan Yu, Jake Zhao, Jinyang Li, Zheng Zhang  
111 | - Blog: [Built-in Message Passing Functions](https://docs.dgl.ai/features/builtin.html)  
112 | 
113 | ### <a name="NC"></a>Node Classification     
114 | - Blog: [DGL at a Glance](https://docs.dgl.ai/tutorials/basics/1_first.html) by  Minjie Wang, Quan Gan, Jake Zhao, Zheng Zhang  
115 | 
116 | ### <a name="GC"></a>Graph Classification    
117 | - [Tutorial: Batched Graph Classification with DGL](https://docs.dgl.ai/tutorials/basics/4_batch.html)
118 | 
119 | ### <a name="GNL"></a>Graph Nets Library   
120 | A DeepMind's library for building graph networks in Tensorflow and Sonnet.  
121 | - Blog: [Graph Nets Library](https://github.com/deepmind/graph_nets) 
122 | - Jupyter NoteBook: [Tutorial of the Graph Nets Library](https://colab.research.google.com/github/deepmind/graph_nets/blob/master/graph_nets/demos/graph_nets_basics.ipynb)   
123 | 
124 | ### <a name="ML"></a>More Libraries   
125 | - Blog: [StellarGraph Machine Learning Library:](https://github.com/stellargraph/stellargraph) a Python library for machine learning on graph-structured (or equivalently, network-structured) data. 
126 | - Blog: [PyTorch Geometric (PyG)](https://github.com/rusty1s/pytorch_geometric) by Matthias Fey. PyTorch Geometric is a geometric deep learning extension library for PyTorch. 
127 | 
128 | ### <a name="SLG"></a>Save & Load Graphs       
129 | - Blog: [Reading and Writing Graphs](https://networkx.github.io/documentation/stable/reference/readwrite/index.html)
130 | 
131 | ## <a name="DG"></a>Datasets of Graphs       
132 | 
133 | - Blog: [Open Graph Benchmark](https://ogb.stanford.edu/) is a collection of realistic, large-scale, and diverse benchmark datasets for machine learning on graphs.  
134 | - Blog: [Network Repository. An Interactive Scientific Network Data Repository:](http://networkrepository.com) The first interactive data and network data repository with real-time visual analytics. Network repository is not only the first interactive repository, but also the largest network repository with thousands of donations in 30+ domains (from biological to social network data). This repository was made by Ryan A. Rossi and Nesreen K. Ahmed.  
135 | - Blog: [Graph Classification:](https://paperswithcode.com/task/graph-classification/latest) The mission of Papers With Code is to create a free and open resource with Machine Learning papers, code and evaluation tables.  
136 | - Blog: [Graph Challenge Data Sets:](https://graphchallenge.mit.edu/data-sets) Amazon is making the Graph Challenge data sets available to the community free of charge as part of the AWS Public Data Sets program. The data is being presented in several file formats, and there are a variety of ways to access it.    
137 | - Blog: [The House of Graphs:](https://hog.grinvin.org) a database of interesting graphs by G. Brinkmann, K. Coolsaet, J. Goedgebeur, and H. Mélot (also see Discrete Applied Mathematics, 161(1-2): 311-314, 2013 ([DOI](http://dx.doi.org/10.1016/j.dam.2012.07.018))).   
138 |     * [Search for Graphs](https://hog.grinvin.org/StartSearch.action)  
139 | - Blog: [A Repository of Benchmark Graph Datasets for Graph Classification](https://github.com/shiruipan/graph_datasets) by 
140 | Shiruipan    
141 | - Blog: [Collection and Streaming of Graph Datasets](https://www.eecs.wsu.edu/~yyao/StreamingGraphs.html) by Yibo Yao   
142 | - Blog: [Big Graph Data Sets](https://lgylym.github.io/big-graph/dataset.html) by Yongming Luo   
143 | - Blog: [MIVIA LDGraphs Dataset:](https://mivia.unisa.it/datasets/graph-database/mivia2-graph-database/) The MIVIA LDGraphs (MIVIA Large Dense Graphs) dataset is a new dataset for benchmarking exact graph matching algorithms. It aims to extend the MIVIA graphs dataset, widely used in the last ten years, with bigger and more dense graphs, so as to face with the problems nowadays encountered in real applications devoted for instance to bioinformatics and social network analysis. 
144 | - Blog: [Datasets](https://sites.wustl.edu/neumann/research/datasets/) by Marion Neumann  
145 | - Blog: [Graph Dataset](https://sites.google.com/site/xiaomengsite/research/resources/graph-dataset) by Xiao Meng   
146 | - Blog: [Constructors and Databases of Graphs in Sage](http://doc.sagemath.org/html/en/reference/graphs/index.html)
147 | - Datasets in GitHub:   
148 |   - [Benchmark Dataset for Graph Classification:](https://github.com/FilippoMB/Benchmark_dataset_for_graph_classification) This repository contains datasets to quickly test graph classification algorithms, such as Graph Kernels and Graph Neural Networks by Filippo Bianchi.     
149 |   - [GAM:](https://github.com/benedekrozemberczki/GAM) A PyTorch implementation of "Graph Classification Using Structural Attention" (KDD 2018) by Benedek Rozemberczki.
150 |   - [CapsGNN:](https://github.com/benedekrozemberczki/CapsGNN) A PyTorch implementation of "Capsule Graph Neural Network" (ICLR 2019) by Benedek Rozemberczki.   
151 | 
152 | ### <a name="TCG"></a>Tools for Creating Graphs  
153 | 
154 | - Package: [Networkx:](https://networkx.github.io) a Python package for the creation, manipulation, and study of the structure, dynamics, and functions of complex networks.  
155 |   - [Graph Generators](https://networkx.github.io/documentation/stable/reference/generators.html)  
156 |   - [Converting to and from Other Data Formats To NetworkX Graph](https://networkx.github.io/documentation/stable/reference/convert.html)  
157 |   - [Reading and Writing Graphs](https://networkx.github.io/documentation/stable/reference/readwrite/index.html)    
158 |   
159 | - Package: [Sage:](https://www.sagemath.org) a viable free open source alternative to Magma, Maple, Mathematica and Matlab.
160 |   - [CoCalc:](https://www.sagemath.org/notebook-vs-cloud.html) an [online service](https://cocalc.com/) for running SageMath computations online to avoid your own installation of Sage. CoCalc will allow you to work with multiple persistent worksheets in Sage, IPython, LaTeX, and much, much more!
161 |   - [Graph Theory in Sage](http://doc.sagemath.org/html/en/reference/graphs/index.html)  
162 |   
163 | ## <a name="MSA"></a>Molecular Structure Analysis   
164 | Molecular structures have graph structures. By using machine learning and in particular GNNs, one can predict some physical, chemical, biochemical properties of a molecule by it's chemical formula. Also, it is possible to predict new formula and 3D structure for an unknown yet molecule or substance with certain desired properties. 
165 | 
166 | ### <a name="ML-MS"></a>Graph Machine Learning and its Application on Molecular Science  
167 | 
168 | 
169 | - Book: [Deep Learning for the Life Sciences: Applying Deep Learning to Genomics, Microscopy, Drug Discovery, and More](https://www.amazon.com/Deep-Learning-Life-Sciences-Microscopy/dp/1492039837) by Bharath Ramsundar, Peter Eastman, Patrick Walters, and Vijay Pande  
170 |     - [Codes](https://github.com/deepchem/DeepLearningLifeSciences)  
171 | - Blog: [Machine Learning for Drug Development (Tutorial at the 29th International Joint Conference on Artificial Intelligence (IJCAI))](https://zitniklab.hms.harvard.edu/drugml/) 
172 | - Blog: [Graph Neural Networks for Binding Affinity Prediction](https://levelup.gitconnected.com/graph-neural-networks-for-binding-affinity-prediction-f057c495ad15) by Alex Gurbych   
173 | - Blog: [Generating Molecules with the Help of Recurrent Neural Networks](https://towardsdatascience.com/generating-molecules-with-the-help-of-recurrent-neural-networks-c3fe23bd0de2) by Seyone Chithrananda   
174 | - Blog: [Graph Convolutional Neural Networks to Analyze Complex Carbohydrates (Using PyTorch Geometric to Work With Biological Data)](https://towardsdatascience.com/graph-convolutional-neural-networks-to-analyze-complex-carbohydrates-91f6bae297de) by Daniel Bojar  
175 | - Blog: [How to Use Machine Learning for Drug Discovery](https://towardsdatascience.com/how-to-use-machine-learning-for-drug-discovery-1ccb5fdf81ad) by Chanin Nantasenamat   
176 | - Blog: [Tutorial ML In Chemistry Research. RDkit & mol2vec](https://www.kaggle.com/vladislavkisin/tutorial-ml-in-chemistry-research-rdkit-mol2vec/data) by Vlad Kisin  
177 | - Blog: [Papers in Drug Discovery](https://paperswithcode.com/task/drug-discovery)  
178 | - Blog: [Review: Deep Learning In Drug Discovery](https://towardsdatascience.com/review-deep-learning-in-drug-discovery-f4c89e3321e1) by Hosein Fooladi   
179 | - Blog: [A Practical Introduction to the Use of Molecular Fingerprints in Drug Discovery](https://towardsdatascience.com/a-practical-introduction-to-the-use-of-molecular-fingerprints-in-drug-discovery-7f15021be2b1) by Laksh  
180 | - Blog: [Public Coronavirus Prediction Models](https://www.aicures.mit.edu/post/public-coronavirus-prediction-models)  
181 | - Blog: [DIY Drug Discovery - Using Molecular Fingerprints and Machine Learning for Solubility Prediction](http://www.moreisdifferent.com/2017/9/21/DIY-Drug-Discovery-using-molecular-fingerprints-and-machine-learning-for-solubility-prediction/) by Daniel C. Elton  
182 | - Slide: [Graph Neural Network and its Application on Molecular Science](https://tykimos.github.io/warehouse/2018-6-28-ISS_1st_Deep_Learning_Together_rsy_file.pdf) by Seongok Ryu   
183 | - GitHub:
184 |     - [Graph Neural Network (GNN) for Molecular Property Prediction (SMILES format)](https://github.com/masashitsubaki/molecularGNN_smiles) by Masashi Tsubaki   
185 | - Competition: [Predicting Molecular Properties](https://www.kaggle.com/c/champs-scalar-coupling/overview)    
186 | - Competition: [Fighting Secondary Effects of Covid](https://www.aicures.mit.edu/tasks) COVID-19 presents many health challenges beyond the virus itself. One of them is finding effective antibiotics for secondary infections.   
187 | - Blog: [Machine Learning Formulation](https://www.aicures.mit.edu/post/machine-learning-formulation) 
188 | - Blog: [How to Fight COVID-19 with Machine Learning](https://towardsdatascience.com/fight-covid-19-with-machine-learning-1d1106192d84) by Markus Schmitt  
189 | - Paper: [MoleculeNet: A Benchmark for Molecular Machine Learning](https://arxiv.org/pdf/1703.00564.pdf) by Zhenqin Wu, Bharath Ramsundar, Evan N. Feinberg, Joseph Gomes, Caleb Geniesse, Aneesh S. Pappu, Karl Leswing, Vijay Pande  
190 | 
191 | 
192 | ### <a name="Notations"></a>Chemical Notations     
193 | - Blog: [SMILES Tutorial](https://archive.epa.gov/med/med_archive_03/web/html/smiles.html) Simplified Molecular Input Line Entry System (SMILES) is a chemical notation that allows a user to represent a chemical structure in a way that can be used by the computer. SMILES is an easily learned and flexible notation. The SMILES notation requires that you learn a handful of rules.
194 | - Blog: [SMILES - A Simplified Chemical Language](https://www.daylight.com/dayhtml/doc/theory/theory.smiles.html)  
195 |     - [Try the JSME](https://peter-ertl.com/jsme/JSME_2017-02-26/JSME.html)    
196 | - Blog: [Smiles](https://docs.chemaxon.com/display/docs/smiles.md)
197 | - Blog: [OpenSMILES](http://opensmiles.org/) is a community sponsored open-standards version of the SMILES language for chemistry. OpenSMILES is part of the [Blue Obelisk](https://blueobelisk.github.io/) community. 
198 | - Blog: [SMARTS - A Language for Describing Molecular Patterns](https://www.daylight.com/dayhtml/doc/theory/theory.smarts.html)   
199 | - Blog: [SMIRKS - A Reaction Transform Language](https://www.daylight.com/dayhtml/doc/theory/theory.smirks.html)  
200 | - Slide: [Rolling Smarts: You don’t Always Find What You Want, But if You Try Sometimes, You Find What You Need](https://www.nextmovesoftware.com/talks/Sayle_RollingSMARTS_RDKitUGM_202010.pdf) by Roger Sayle and John Mayfield    
201 | 
202 | ### <a name="Chemistry"></a>An Introduction to Basic Chemistry and Drugs  
203 | 
204 | #### Chemistry
205 | - Blog: [Chemistry Library](https://www.khanacademy.org/science/chemistry)  provides informations about the following subjects: Atoms, compounds, and ions, Molecular composition, Mass spectrometry, Chemical reactions and stoichiometry, Electronic structure of atoms, Periodic table, Chemical bonds, Gases and kinetic molecular theory, States of matter and intermolecular forces, Chemical equilibrium, Acids and bases, Buffers, titrations, and solubility equilibria, Thermodynamics, Redox reactions and electrochemistry, Kinetics, Alkanes, cycloalkanes, and functional groups.  
206 | - Blog: [Functional Group Names, Properties, and Reactions](https://courses.lumenlearning.com/boundless-chemistry/chapter/functional-group-names-properties-and-reactions/)
207 | - Talk: [Encyclopaedia of Functional Groups](https://cisrg.shef.ac.uk/shef2019/talks/Ertl.pdf) by Peter Ertl  
208 | - Simulations  by ChemThink  
209 |     - [The Particulate Nature of Matter](https://simbucket.com/chemthinkserver/chemthink/index.html?pn)
210 |     - [Atomic Structure](https://simbucket.com/chemthinkserver/chemthink/index.html?as)   
211 |     - [Covalent Bonding Tutorial](https://pbslm-contrib.s3.amazonaws.com/WGBH/arct15/SimBucket/Simulations/chemthink-covalentbonding/content/index.html)   
212 |     - [Ions](https://simbucket.com/chemthinkserver/chemthink/index.html?io)  
213 |     - [Ionic Bonding](https://simbucket.com/chemthinkserver/chemthink/index.html?ib)  
214 |     - [Ionic Formulas](https://simbucket.com/chemthinkserver/chemthink/index.html?if)  
215 |     - [Molecular Shapes](https://simbucket.com/chemthinkserver/chemthink/index.html?ms)
216 |     - [Isotopes](https://www.simbucket.com/chemthinkserver/chemthink/index.html?is)
217 | - Blog: [Periodic Table of Elements](https://pubchem.ncbi.nlm.nih.gov/periodic-table/) by [PubChem](https://pubchem.ncbi.nlm.nih.gov/)  
218 | - Blog: [Periodic Table of Elements](https://ptable.com/#Properties) by Michael Dayah   
219 | - Blog: [Ionic Bond Examples](https://examples.yourdictionary.com/ionic-bond-examples.html)   
220 | - Blog: [A Comprehensive Treatment of Aromaticity in the SMILES Language](https://depth-first.com/articles/2020/02/10/a-comprehensive-treatment-of-aromaticity-in-the-smiles-language/)   
221 | 
222 | #### Drug
223 | - Blog: [Chiral Drugs](https://www.khanacademy.org/test-prep/mcat/chemical-processes/stereochemistry/a/chiral-drugs)   
224 | - Blog: [Drug Lipophilicity and Absorption: The Continuous Challenge in Drug Discovery](https://emerypharma.com/blog/drug-lipophilicity-and-absorption-a-continuous-challenge-toward-the-goal-of-drug-discovery/) by Emery Pharma     
225 |     
226 | ## <a name="DMTP"></a>Datasets of Molecules and Their Properties  
227 | 
228 | ### Chemical Datasets
229 | - Blog: [PubChem Dataset](https://pubchemdocs.ncbi.nlm.nih.gov/downloads) is an open chemistry database at the National Institutes of Health (NIH). “Open” means that you can put your scientific data in PubChem and that others may use it.  
230 |     - Blog: [Browse PubChem Data](https://pubchem.ncbi.nlm.nih.gov/classification/#hid=1) using a classification of interest, or search for PubChem records annotated with the desired classification/term  
231 | - Blog: [GDB Databases](https://gdb.unibe.ch/downloads/) GDB-11 enumerates small organic molecules up to 11 atoms of C, N, O and F following simple chemical stability and synthetic feasibility rules. GDB-13 enumerates small organic molecules up to 13 atoms of C, N, O, S and Cl following simple chemical stability and synthetic feasibility rules. With 977 468 314 structures, GDB-13 is the largest publicly available small organic molecule database to date.   
232 | - Blog: [MoleculeNet](http://moleculenet.ai/) is a benchmark specially designed for testing machine learning methods of molecular properties. As we aim to facilitate the development of molecular machine learning method, this work curates a number of dataset collections, creates a suite of software that implements many known featurizations and previously proposed algorithms. All methods and datasets are integrated as parts of the open source **DeepChem** package(MIT license).  
233 | - Blog: [ChEMBL](https://www.ebi.ac.uk/chembl/) is a manually curated database of bioactive molecules with drug-like properties. It brings together chemical, bioactivity and genomic data to aid the translation of genomic information into effective new drugs.
234 | - Blog: [Open Data Repositories by the Blue Obelisk](https://blueobelisk.github.io/opendata.html) Open Data allows access to large data bases. A good example in chemoinformatics is the NMRShiftDB: the spectra and structures in this database can be downloaded for free, and you have the right to redistribute them.  
235 | - Blog: [Tox21](http://bioinf.jku.at/research/DeepTox/tox21.html): The 2014 Tox21 data challenge was designed to help scientists understand the potential of the chemicals and compounds being tested through the Toxicology in the 21st Century initiative to disrupt biological pathways in ways that may result in toxic effects. The [Tox21 Program (Toxicology in the 21st Century)](https://tripod.nih.gov/tox21/challenge/data.jsp) is an ongoing collaboration among federal agencies to characterize the potential toxicity of chemicals using cells and isolated molecular targets instead of laboratory animals.  
236 | - Blog: [ZINC15:](http://zinc15.docking.org/) a free database of commercially-available compounds for virtual screening. ZINC contains over 230 million purchasable compounds in ready-to-dock, 3D formats. ZINC also contains over 750 million purchasable compounds you can search for analogs in under a minute.  
237 | - Blog: [Open Catalyst Project](https://opencatalystproject.org/index.html)  is a collaborative research effort between Facebook AI Research (FAIR) and Carnegie Mellon University’s (CMU) Department of Chemical Engineering. The aim is to use AI to model and discover new catalysts for use in renewable energy storage to help in addressing climate change.  
238 | 
239 | ### Biological Datasets  
240 | - Blog: [PhID](http://phid.ditad.org/)  contains six different types of entities: drugs(chemicals), target(proteins), diseases, genes, side-effects, and pathways, which integreted from 14 public resources.  
241 | - [DrugCentral](https://drugcentral.org/) provides information on active ingredients chemical entities, pharmaceutical products, drug mode of action, indications, pharmacologic action.  
242 | - Blog: [NCATS Inxight (Drugs)](https://drugs.ncats.io/) The National Center for Advancing Translational Sciences (NCATS) has developed Inxight: Drugs as a comprehensive portal for drug development information. NCATS Inxight: Drugs contains information on ingredients in medicinal products.    
243 | - Blog: [Therapeutic Target Database (TTD)](http://db.idrblab.net/ttd/) A database to provide information about the known and explpred therapeutic protein and nucleic acid targets,the targeted disease, pathway information.  
244 | - Blog: [Drug Target Commons (DTC)](https://drugtargetcommons.fimm.fi/) is a crowd-sourcing platform to improve the consensus and use of drug-target interactions. The end users can search, view and download bioactivity data using various compound, target and publications identifiers. Expert users may also submit suggestions to edit and upload new bioactivity data, as well as participate in the assay annotation and data curation processes.   
245 | - Blog: [Pharos](https://pharos.nih.gov/) is the user interface to the Knowledge Management Center (KMC) for the Illuminating the Druggable Genome (IDG) program funded by the National Institutes of Health (NIH) Common Fund.  
246 | - Blog: [PROMISCUOUS2](https://bioinformatics.charite.de/promiscuous2/index.php)  is an one stop resource for drug-repositioning. The database can be used from experts by using the drug or target search as well as non experts, by browsing through the data via the Browse tab.   
247 | - Blog: [Therapeutics Data Commons (Machine Learning Datasets and Tasks for Therapeutics)](https://tdcommons.ai/)     
248 | - Blog: [STITCH](http://stitch.embl.de/) is a database of known and predicted interactions between chemicals and proteins. The interactions include direct (physical) and indirect (functional) associations; they stem from computational prediction, from knowledge transfer between organisms, and from interactions aggregated from other (primary) databases.  
249 | - Blog: [KEGG: Kyoto Encyclopedia of Genes and Genomes](https://www.genome.jp/kegg/)
250 | - Blog: [Drug Repositioning Database (repoDB)](http://apps.chiragjpgroup.org/repoDB/) contains a standard set of drug repositioning successes and failures that can be used to fairly and reproducibly benchmark computational repositioning methods. repoDB data was extracted from DrugCentral and ClinicalTrials.gov.
251 | - Blog: [STRING (Protein-Protein Interaction Networks)](https://string-db.org/) In molecular biology, STRING (Search Tool for the Retrieval of Interacting Genes/Proteins) is a biological database and web resource of known and predicted protein–protein interactions.
252 | 
253 | ### <a name="Libraries"></a>Libraries & Packages  
254 | - Package: RDKit is an open source toolkit for cheminformatics    
255 |     - PDF: [RDKit Documentation](https://buildmedia.readthedocs.org/media/pdf/rdkit/latest/rdkit.pdf)   
256 |     - Blog: [RDKit Cookbook](https://www.rdkit.org/docs/Cookbook.html) by Greg Landrum and Vincent Scalfani   
257 |     - Blog: [RDKit: Simple File Input and Output](https://medium.com/@camkirk/rdkit-simple-file-input-and-output-e6764fc1e35c) by Cam Kirk 
258 |     - Blog: [Getting Started with the RDKit in Python](https://www.rdkit.org/docs/GettingStartedInPython.html) by Greg Landrum   
259 | - Package: [DeepChem](https://deepchem.io/) project aims to create high quality, open source tools for drug discovery, materials science, quantum chemistry, and biology.
260 |     - [Tutorials and Codes:](https://deepchem.io/docs/notebooks/index.html) These tutorials show off various aspects or capabilities of DeepChem. They can be run interactively in Jupyter (IPython) notebook.  
261 | - [OCHEM:](https://ochem.eu/home/show.do) The Online Chemical Modeling Environment is a web-based platform that aims to automate and simplify the typical steps required for QSAR modeling. The platform consists of two major subsystems: the database of experimental measurements and the modeling framework. A user-contributed database contains a set of tools for easy input, search and modification of thousands of records. OCHEM was created  by Iurii Sushko.   
262 | - Blog: [Mol2vec:](https://github.com/samoturk/mol2vec) An unsupervised machine learning approach to learn vector   representations of molecular substructures  
263 | - Package: [Chemprop](https://github.com/chemprop/chemprop)
264 | - Package: [DGL-LifeSci](https://lifesci.dgl.ai/index.html) is a python package for applying graph neural networks to various tasks in chemistry and biology, on top of PyTorch and DGL.  
265 |     - Code: [Property Prediction](https://github.com/awslabs/dgl-lifesci/tree/master/examples/property_prediction/moleculenet)  
266 | - Package: [Mordred Descriptor:](https://github.com/mordred-descriptor/mordred) a molecular descriptor calculator  
267 | - Knowledge Graph: [Drug Repurposing Knowledge Graph (DRKG)](https://github.com/gnn4dr/DRKG/) 
268 | - Platform: [Cytoscape](https://cytoscape.org/)  is an open source bioinformatics software platform for visualizing molecular interaction networks and integrating with gene expression profiles and other state data.    
269 | 
270 | ### <a name="Online-Softwares"></a>Online Softwares   
271 | - Blog: [Chemprop — Machine Learning for Molecular Property Prediction](http://chemprop.csail.mit.edu) This website can be used to predict molecular properties using a Message Passing Neural Network (MPNN).
272 | - Blog: [JSME Molecule Editor:](https://peter-ertl.com/jsme/) JSME is a free molecule editor written in JavaScript. JSME is a direct successor of the JME Molecule Editor applet. JSME supports drawing and editing of molecules. JSME was created  by Peter Ertl and Bruno Bienfait.  
273 | Here are some softwares to calculate molecular descriptors and fingerprints online:   
274 | - Blog: [PaDEL-Descriptor](http://www.yapcwsoft.com/dd/padeldescriptor/) 
275 | - Blog: [RDkit Descriptors Generator](http://www.pirika.com/Program/2019/RDKit18.html) by Hiroshi Yamamoto   
276 | 
277 | ### <a name="Draw-Molecule"></a>Softwares (Draw a Molecule)   
278 |  - Blog: [Marvin JS](https://marvinjs-demo.chemaxon.com/latest/demo.html) provides quick and convenient ways to draw and modify standard and advanced chemical structures. It's seamlessly integrated into third-party web-based applications, and runs smoothly on all major browsers. 
279 |  - Blog: [MolView](https://molview.org/)  is an intuitive, Open-Source web-application to make science and education more awesome! MolView is mainly intended as web-based data visualization platform.
280 |  - Blog: [Draw Structue](https://pubchem.ncbi.nlm.nih.gov/#draw=true) by [PubChem](https://pubchem.ncbi.nlm.nih.gov/)
281 |  - [Simulations:](https://phet.colorado.edu/en/simulations/filter?subjects=chemistry&sort=alpha&view=grid) [Build an Atom](https://phet.colorado.edu/sims/html/build-an-atom/latest/build-an-atom_en.html), [Build a Molecule](https://phet.colorado.edu/sims/html/build-a-molecule/latest/build-a-molecule_en.html), [Molecule Shapes: Basics](https://phet.colorado.edu/en/simulation/molecule-shapes-basics), [Molecule Shapes](https://phet.colorado.edu/en/simulation/molecule-shapes), [Isotopes and Atomic Mass](https://phet.colorado.edu/en/simulation/isotopes-and-atomic-mass), [Molecule Polarity](https://phet.colorado.edu/en/simulation/molecule-polarity)    
282 | 


--------------------------------------------------------------------------------
/Images/BH.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/BH.jpeg


--------------------------------------------------------------------------------
/Images/DDLP.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/DDLP.png


--------------------------------------------------------------------------------
/Images/DL.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/DL.jpg


--------------------------------------------------------------------------------
/Images/Deep-Learning-PyTorch.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/Deep-Learning-PyTorch.jpg


--------------------------------------------------------------------------------
/Images/Dive-Into-DL.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/Dive-Into-DL.png


--------------------------------------------------------------------------------
/Images/GDL.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/GDL.jpg


--------------------------------------------------------------------------------
/Images/HH.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/HH.jpg


--------------------------------------------------------------------------------
/Images/Hands-On-ML.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/Hands-On-ML.jpg


--------------------------------------------------------------------------------
/Images/K.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/K.jpg


--------------------------------------------------------------------------------
/Images/Mafakheri.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/Mafakheri.jpg


--------------------------------------------------------------------------------
/Images/NNLM.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/NNLM.jpg


--------------------------------------------------------------------------------
/Images/PPDL.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/PPDL.jpg


--------------------------------------------------------------------------------
/Images/ProDeep.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/ProDeep.jpg


--------------------------------------------------------------------------------
/Images/R.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/R.jpg


--------------------------------------------------------------------------------
/Images/README.md:
--------------------------------------------------------------------------------
1 | # Images of Main TextBooks
2 | 


--------------------------------------------------------------------------------
/Images/Y.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/Y.jpg


--------------------------------------------------------------------------------
/Images/data-fallacies-to-avoid.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hhaji/Deep-Learning/ca9f5f69aec3075d2965d614d25da06b6743e430/Images/data-fallacies-to-avoid.jpg


--------------------------------------------------------------------------------
/NoteBooks/Generator.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Iterable, Generator, and Iterator  \n",
  8 |     "This notebook was adopted from [A Quick Guide To Python Generators and Yield Statements](https://medium.com/@jasonrigden/a-quick-guide-to-python-generators-and-yield-statements-89a4162c0ef8) by Jason Rigden and [Iterables vs. Iterators vs. Generators](https://nvie.com/posts/iterators-vs-generators/) by Vincent Driessen"
  9 |    ]
 10 |   },
 11 |   {
 12 |    "cell_type": "markdown",
 13 |    "metadata": {},
 14 |    "source": [
 15 |     "### In this notebook we dicuss the differences between the following related concepts in Python:\n",
 16 |     "\n",
 17 |     "* a container  \n",
 18 |     "* an iterable  \n",
 19 |     "* an iterator  \n",
 20 |     "* a generator  \n",
 21 |     "* a generator expression  \n",
 22 |     "* a {list, set, dict} comprehension  "
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "markdown",
 27 |    "metadata": {},
 28 |    "source": [
 29 |     "## Containers \n",
 30 |     "Containers are data structures holding elements, and that support membership tests. They are data structures that live in memory, and typically hold all their values in memory, too. In Python, some well known examples are:\n",
 31 |     "\n",
 32 |     "* list, deque, …  \n",
 33 |     "* set, frozensets, …  \n",
 34 |     "* dict, defaultdict, OrderedDict, Counter, …  \n",
 35 |     "* tuple, namedtuple, …  \n",
 36 |     "* str  "
 37 |    ]
 38 |   },
 39 |   {
 40 |    "cell_type": "markdown",
 41 |    "metadata": {},
 42 |    "source": [
 43 |     "Containers are easy to grasp, because you can think of them as real life containers: a box, a cubboard, a house, a ship, etc.  \n",
 44 |     "\n",
 45 |     "Technically, an object is a container when it can be asked whether it contains a certain element. You can perform such membership tests on lists, sets, or tuples alike:"
 46 |    ]
 47 |   },
 48 |   {
 49 |    "cell_type": "code",
 50 |    "execution_count": 1,
 51 |    "metadata": {},
 52 |    "outputs": [],
 53 |    "source": [
 54 |     "assert 1 in [1, 2, 3]      # lists\n",
 55 |     "assert 4 not in [1, 2, 3]\n",
 56 |     "assert 1 in {1, 2, 3}      # sets\n",
 57 |     "assert 4 not in {1, 2, 3}\n",
 58 |     "assert 1 in (1, 2, 3)      # tuples\n",
 59 |     "assert 4 not in (1, 2, 3)"
 60 |    ]
 61 |   },
 62 |   {
 63 |    "cell_type": "markdown",
 64 |    "metadata": {},
 65 |    "source": [
 66 |     "The following [figure](https://nvie.com/posts/iterators-vs-generators/) was depicted by Vincent Driessen"
 67 |    ]
 68 |   },
 69 |   {
 70 |    "cell_type": "markdown",
 71 |    "metadata": {},
 72 |    "source": [
 73 |     "![Iterable, Generator, Iterator](./Iterator/Iterator.pdf)"
 74 |    ]
 75 |   },
 76 |   {
 77 |    "cell_type": "markdown",
 78 |    "metadata": {},
 79 |    "source": [
 80 |     "## Iterables: \n",
 81 |     " \n",
 82 |     "As said, most containers are also iterable. But many more things are iterable as well. Examples are open files, open sockets, etc. Where containers are typically finite, an iterable may just as well represent an infinite source of data.\n",
 83 |     "\n",
 84 |     "An iterable is any object, not necessarily a data structure, that can return an iterator (with the purpose of returning all of its elements). That sounds a bit awkward, but there is an important difference between an iterable and an iterator. "
 85 |    ]
 86 |   },
 87 |   {
 88 |    "cell_type": "code",
 89 |    "execution_count": 2,
 90 |    "metadata": {},
 91 |    "outputs": [
 92 |     {
 93 |      "name": "stdout",
 94 |      "output_type": "stream",
 95 |      "text": [
 96 |       "0\n",
 97 |       "1\n",
 98 |       "2\n",
 99 |       "3\n",
100 |       "4\n"
101 |      ]
102 |     }
103 |    ],
104 |    "source": [
105 |     "data = [0, 1, 2, 3, 4]\n",
106 |     "for each in data:\n",
107 |     "    print(each)"
108 |    ]
109 |   },
110 |   {
111 |    "cell_type": "markdown",
112 |    "metadata": {},
113 |    "source": [
114 |     "## List Comprehensions"
115 |    ]
116 |   },
117 |   {
118 |    "cell_type": "code",
119 |    "execution_count": 3,
120 |    "metadata": {},
121 |    "outputs": [
122 |     {
123 |      "name": "stdout",
124 |      "output_type": "stream",
125 |      "text": [
126 |       "0\n",
127 |       "1\n",
128 |       "4\n",
129 |       "9\n",
130 |       "16\n"
131 |      ]
132 |     }
133 |    ],
134 |    "source": [
135 |     "data = [0, 1, 2, 3, 4]\n",
136 |     "result = [x*x for x in data]\n",
137 |     "for each in result:\n",
138 |     "    print(each)"
139 |    ]
140 |   },
141 |   {
142 |    "cell_type": "markdown",
143 |    "metadata": {},
144 |    "source": [
145 |     "## Set Comprehensions"
146 |    ]
147 |   },
148 |   {
149 |    "cell_type": "code",
150 |    "execution_count": 4,
151 |    "metadata": {},
152 |    "outputs": [
153 |     {
154 |      "data": {
155 |       "text/plain": [
156 |        "{0, 1, 4, 9, 16}"
157 |       ]
158 |      },
159 |      "execution_count": 4,
160 |      "metadata": {},
161 |      "output_type": "execute_result"
162 |     }
163 |    ],
164 |    "source": [
165 |     "numbers = [0, 1, 2, 3, 4]\n",
166 |     "{x * x for x in numbers}"
167 |    ]
168 |   },
169 |   {
170 |    "cell_type": "markdown",
171 |    "metadata": {},
172 |    "source": [
173 |     "## Dictionary Comprehensions"
174 |    ]
175 |   },
176 |   {
177 |    "cell_type": "code",
178 |    "execution_count": 5,
179 |    "metadata": {},
180 |    "outputs": [
181 |     {
182 |      "data": {
183 |       "text/plain": [
184 |        "{0: 0, 1: 1, 2: 4, 3: 9, 4: 16}"
185 |       ]
186 |      },
187 |      "execution_count": 5,
188 |      "metadata": {},
189 |      "output_type": "execute_result"
190 |     }
191 |    ],
192 |    "source": [
193 |     "numbers = [0, 1, 2, 3, 4]\n",
194 |     "{x: x*x for x in numbers}"
195 |    ]
196 |   },
197 |   {
198 |    "cell_type": "markdown",
199 |    "metadata": {},
200 |    "source": [
201 |     "## Generators"
202 |    ]
203 |   },
204 |   {
205 |    "cell_type": "code",
206 |    "execution_count": 6,
207 |    "metadata": {},
208 |    "outputs": [
209 |     {
210 |      "name": "stdout",
211 |      "output_type": "stream",
212 |      "text": [
213 |       "0\n",
214 |       "1\n",
215 |       "4\n",
216 |       "9\n",
217 |       "16\n"
218 |      ]
219 |     }
220 |    ],
221 |    "source": [
222 |     "data = [0, 1, 2, 3, 4]\n",
223 |     "new_generator = (x*x for x in range(5))\n",
224 |     "for each in new_generator:\n",
225 |     "    print(each)"
226 |    ]
227 |   },
228 |   {
229 |    "cell_type": "markdown",
230 |    "metadata": {},
231 |    "source": [
232 |     "Unlike a list, a generator only can be used once. When it is empty. It is empty."
233 |    ]
234 |   },
235 |   {
236 |    "cell_type": "code",
237 |    "execution_count": 7,
238 |    "metadata": {},
239 |    "outputs": [
240 |     {
241 |      "name": "stdout",
242 |      "output_type": "stream",
243 |      "text": [
244 |       "0\n",
245 |       "1\n",
246 |       "4\n",
247 |       "9\n",
248 |       "16\n"
249 |      ]
250 |     }
251 |    ],
252 |    "source": [
253 |     "data = [0, 1, 2, 3, 4]\n",
254 |     "new_generator = (x*x for x in range(5))\n",
255 |     "for each in new_generator:\n",
256 |     "    print(each)"
257 |    ]
258 |   },
259 |   {
260 |    "cell_type": "code",
261 |    "execution_count": 8,
262 |    "metadata": {},
263 |    "outputs": [],
264 |    "source": [
265 |     "for each in new_generator:\n",
266 |     "    print(each)"
267 |    ]
268 |   },
269 |   {
270 |    "cell_type": "markdown",
271 |    "metadata": {},
272 |    "source": [
273 |     "### Yield is a keyword just like return."
274 |    ]
275 |   },
276 |   {
277 |    "cell_type": "code",
278 |    "execution_count": 9,
279 |    "metadata": {},
280 |    "outputs": [
281 |     {
282 |      "name": "stdout",
283 |      "output_type": "stream",
284 |      "text": [
285 |       "0\n",
286 |       "1\n",
287 |       "4\n",
288 |       "9\n",
289 |       "16\n"
290 |      ]
291 |     }
292 |    ],
293 |    "source": [
294 |     "def generatorGenerator():\n",
295 |     "    for x in range(5):\n",
296 |     "        yield x*x\n",
297 |     "        \n",
298 |     "new_generator = generatorGenerator()\n",
299 |     "\n",
300 |     "for each in new_generator:\n",
301 |     "    print(each)"
302 |    ]
303 |   },
304 |   {
305 |    "cell_type": "code",
306 |    "execution_count": 10,
307 |    "metadata": {},
308 |    "outputs": [],
309 |    "source": [
310 |     "for each in new_generator:\n",
311 |     "    print(each)"
312 |    ]
313 |   },
314 |   {
315 |    "cell_type": "code",
316 |    "execution_count": 11,
317 |    "metadata": {},
318 |    "outputs": [
319 |     {
320 |      "name": "stdout",
321 |      "output_type": "stream",
322 |      "text": [
323 |       "Hello\n",
324 |       "World\n",
325 |       "!\n"
326 |      ]
327 |     }
328 |    ],
329 |    "source": [
330 |     "def multiYield():\n",
331 |     "    yield \"Hello\"\n",
332 |     "    yield \"World\"\n",
333 |     "    yield \"!\"\n",
334 |     "    \n",
335 |     "f=multiYield()\n",
336 |     "    \n",
337 |     "for each in f:\n",
338 |     "    print(each)"
339 |    ]
340 |   },
341 |   {
342 |    "cell_type": "code",
343 |    "execution_count": 12,
344 |    "metadata": {},
345 |    "outputs": [],
346 |    "source": [
347 |     "for each in f:\n",
348 |     "    print(each)"
349 |    ]
350 |   },
351 |   {
352 |    "cell_type": "code",
353 |    "execution_count": 13,
354 |    "metadata": {},
355 |    "outputs": [
356 |     {
357 |      "name": "stdout",
358 |      "output_type": "stream",
359 |      "text": [
360 |       "Hello\n",
361 |       "World\n",
362 |       "!\n"
363 |      ]
364 |     }
365 |    ],
366 |    "source": [
367 |     "for each in multiYield():\n",
368 |     "    print(each)"
369 |    ]
370 |   },
371 |   {
372 |    "cell_type": "code",
373 |    "execution_count": 14,
374 |    "metadata": {},
375 |    "outputs": [
376 |     {
377 |      "name": "stdout",
378 |      "output_type": "stream",
379 |      "text": [
380 |       "Hello\n",
381 |       "World\n",
382 |       "!\n"
383 |      ]
384 |     }
385 |    ],
386 |    "source": [
387 |     "for each in multiYield():\n",
388 |     "    print(each)"
389 |    ]
390 |   },
391 |   {
392 |    "cell_type": "code",
393 |    "execution_count": 15,
394 |    "metadata": {},
395 |    "outputs": [
396 |     {
397 |      "name": "stdout",
398 |      "output_type": "stream",
399 |      "text": [
400 |       "5\n",
401 |       "10\n",
402 |       "100\n"
403 |      ]
404 |     }
405 |    ],
406 |    "source": [
407 |     "def multiYield():\n",
408 |     "    x = 5\n",
409 |     "    yield x\n",
410 |     "    x = x + 5\n",
411 |     "    yield x\n",
412 |     "    x = x * x\n",
413 |     "    yield x\n",
414 |     "for each in multiYield():\n",
415 |     "    print(each)"
416 |    ]
417 |   },
418 |   {
419 |    "cell_type": "code",
420 |    "execution_count": 16,
421 |    "metadata": {},
422 |    "outputs": [
423 |     {
424 |      "name": "stdout",
425 |      "output_type": "stream",
426 |      "text": [
427 |       "5\n",
428 |       "10\n",
429 |       "100\n"
430 |      ]
431 |     }
432 |    ],
433 |    "source": [
434 |     "def multiYield():\n",
435 |     "    x = 5\n",
436 |     "    yield x\n",
437 |     "    x = x + 5\n",
438 |     "    yield x\n",
439 |     "    x = x * x\n",
440 |     "    yield x\n",
441 |     "for each in multiYield():\n",
442 |     "    print(each)"
443 |    ]
444 |   },
445 |   {
446 |    "cell_type": "markdown",
447 |    "metadata": {},
448 |    "source": [
449 |     "## What are they good for? \n",
450 |     "Generators are lazy. They only work on demand. That mean they can save cpu, memory, and other resources. Take a look at the following:"
451 |    ]
452 |   },
453 |   {
454 |    "cell_type": "code",
455 |    "execution_count": 17,
456 |    "metadata": {},
457 |    "outputs": [
458 |     {
459 |      "data": {
460 |       "text/plain": [
461 |        "1000000"
462 |       ]
463 |      },
464 |      "execution_count": 17,
465 |      "metadata": {},
466 |      "output_type": "execute_result"
467 |     }
468 |    ],
469 |    "source": [
470 |     "1_000_000"
471 |    ]
472 |   },
473 |   {
474 |    "cell_type": "code",
475 |    "execution_count": 18,
476 |    "metadata": {},
477 |    "outputs": [
478 |     {
479 |      "name": "stdout",
480 |      "output_type": "stream",
481 |      "text": [
482 |       "8697472\n"
483 |      ]
484 |     }
485 |    ],
486 |    "source": [
487 |     "import sys\n",
488 |     "result = [x*x for x in range(1_000_000)]\n",
489 |     "memory_size = sys.getsizeof(result)\n",
490 |     "print(memory_size)"
491 |    ]
492 |   },
493 |   {
494 |    "cell_type": "markdown",
495 |    "metadata": {},
496 |    "source": [
497 |     "## Fibonacci"
498 |    ]
499 |   },
500 |   {
501 |    "cell_type": "code",
502 |    "execution_count": 19,
503 |    "metadata": {},
504 |    "outputs": [],
505 |    "source": [
506 |     "def fibonacci():\n",
507 |     "    a = 0\n",
508 |     "    b = 1\n",
509 |     "    while True:\n",
510 |     "        yield a\n",
511 |     "        old_a = a\n",
512 |     "        a = b\n",
513 |     "        b = old_a + b"
514 |    ]
515 |   },
516 |   {
517 |    "cell_type": "code",
518 |    "execution_count": 20,
519 |    "metadata": {},
520 |    "outputs": [
521 |     {
522 |      "name": "stdout",
523 |      "output_type": "stream",
524 |      "text": [
525 |       "0\n",
526 |       "1\n",
527 |       "1\n",
528 |       "2\n",
529 |       "3\n",
530 |       "5\n",
531 |       "8\n",
532 |       "13\n",
533 |       "21\n"
534 |      ]
535 |     }
536 |    ],
537 |    "source": [
538 |     "for each in fibonacci():\n",
539 |     "    print(each)\n",
540 |     "    if each > 20:\n",
541 |     "        break"
542 |    ]
543 |   }
544 |  ],
545 |  "metadata": {
546 |   "kernelspec": {
547 |    "display_name": "A2",
548 |    "language": "python",
549 |    "name": "a2"
550 |   },
551 |   "language_info": {
552 |    "codemirror_mode": {
553 |     "name": "ipython",
554 |     "version": 3
555 |    },
556 |    "file_extension": ".py",
557 |    "mimetype": "text/x-python",
558 |    "name": "python",
559 |    "nbconvert_exporter": "python",
560 |    "pygments_lexer": "ipython3",
561 |    "version": "3.7.4"
562 |   }
563 |  },
564 |  "nbformat": 4,
565 |  "nbformat_minor": 4
566 | }
567 | 


--------------------------------------------------------------------------------
/NoteBooks/Lec1.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Welcome to TensorFlow!\n",
  8 |     "\n",
  9 |     "Adapted from [CS 20: Tensorflow for Deep Learning Research](http://web.stanford.edu/class/cs20si/syllabus.html)\n",
 10 |     "\n",
 11 |     "CS 20: TensorFlow for Deep Learning Research  \n",
 12 |     "Lecture 1  \n",
 13 |     "1/12/2018"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "markdown",
 18 |    "metadata": {},
 19 |    "source": [
 20 |     "## Course Description\n",
 21 |     "TensorFlow is a powerful open-source software library for machine learning developed by researchers at Google. \n",
 22 |     "It has many pre-built functions to ease the task of building different neural networks. TensorFlow allows distribution \n",
 23 |     "of computation across different computers, as well as multiple CPUs and GPUs within a single machine. TensorFlow provides a Python API, \n",
 24 |     "as well as a less documented C++ API. For this course, we will be using Python. \n",
 25 |     "\n",
 26 |     "This course will cover the fundamentals and contemporary usage of the Tensorflow library for deep learning research. We aim to help students understand the graphical \n",
 27 |     "computational model of TensorFlow, explore the functions it has to offer, and learn how to build and structure models best suited for a deep learning project. \n",
 28 |     "Through the course, students will use TensorFlow to build models of different complexity, from simple linear/logistic regression to convolutional neural network \n",
 29 |     "and recurrent neural networks to solve tasks such as word embedding, translation, optical character recognition, reinforcement learning. Students will also learn best \n",
 30 |     "practices to structure a model and manage research experiments."
 31 |    ]
 32 |   },
 33 |   {
 34 |    "cell_type": "markdown",
 35 |    "metadata": {},
 36 |    "source": [
 37 |     "## What’s TensorFlow?\n",
 38 |     "* Open source software library for numerical computation using data flow graphs\n",
 39 |     "* Originally developed by Google Brain Team to conduct machine learning and deep neural networks research\n",
 40 |     "* General enough to be applicable in a wide variety of other domains as well\n",
 41 |     " \n",
 42 |     " TensorFlow provides an extensive suite of functions and classes that allow users to build various models from scratch."
 43 |    ]
 44 |   },
 45 |   {
 46 |    "cell_type": "markdown",
 47 |    "metadata": {},
 48 |    "source": [
 49 |     " ## Why TensorFlow?\n",
 50 |     "* Python API\n",
 51 |     "* Portability: deploy computation to one or more CPUs or GPUs in a desktop,\n",
 52 |     "server, or mobile device with a single API\n",
 53 |     "* Flexibility: from Raspberry Pi, Android, Windows, iOS, Linux to server farms\n",
 54 |     "* Visualization (TensorBoard is da bomb)\n",
 55 |     "* Checkpoints (for managing experiments)\n",
 56 |     "* Auto-differentiation autodiff (no more taking derivatives by hand. Yay)\n",
 57 |     "* Large community (> 10,000 commits and > 3000 TF-related repos in 1 year)\n",
 58 |     "* Awesome projects already using TensorFlow "
 59 |    ]
 60 |   },
 61 |   {
 62 |    "cell_type": "markdown",
 63 |    "metadata": {},
 64 |    "source": [
 65 |     "##  Getting Started"
 66 |    ]
 67 |   },
 68 |   {
 69 |    "cell_type": "code",
 70 |    "execution_count": 1,
 71 |    "metadata": {},
 72 |    "outputs": [],
 73 |    "source": [
 74 |     "import tensorflow as tf"
 75 |    ]
 76 |   },
 77 |   {
 78 |    "cell_type": "markdown",
 79 |    "metadata": {},
 80 |    "source": [
 81 |     "##  Graphs and Sessions\n",
 82 |     "###  Data Flow Graphs\n",
 83 |     "\n",
 84 |     "**TensorFlow separates definition of computations from their execution**  \n",
 85 |     "\n",
 86 |     "Phase 1: assemble a graph  \n",
 87 |     "Phase 2: use a session to execute operations in the graph.  "
 88 |    ]
 89 |   },
 90 |   {
 91 |    "cell_type": "markdown",
 92 |    "metadata": {},
 93 |    "source": [
 94 |     "### Tensors\n",
 95 |     "\n",
 96 |     "**What's a tensor?**  \n",
 97 |     "An n-dimensional array  \n",
 98 |     "0-d tensor: scalar (number) 1-d tensor: vector  \n",
 99 |     "2-d tensor: matrix  \n",
100 |     "and so on  \n",
101 |     " \n",
102 |     "Tensors are the basic elements of computation in TensorFlow."
103 |    ]
104 |   },
105 |   {
106 |    "cell_type": "markdown",
107 |    "metadata": {},
108 |    "source": [
109 |     "### Constants\n",
110 |     "\n",
111 |     "tf.constant(value, dtype=None, shape=None, name='Const', verify_shape=False)"
112 |    ]
113 |   },
114 |   {
115 |    "cell_type": "code",
116 |    "execution_count": 28,
117 |    "metadata": {},
118 |    "outputs": [
119 |     {
120 |      "name": "stdout",
121 |      "output_type": "stream",
122 |      "text": [
123 |       "Tensor(\"a_2:0\", shape=(), dtype=int32)\n"
124 |      ]
125 |     }
126 |    ],
127 |    "source": [
128 |     "c=tf.constant(25,name='a')\n",
129 |     "print(c)"
130 |    ]
131 |   },
132 |   {
133 |    "cell_type": "code",
134 |    "execution_count": 31,
135 |    "metadata": {},
136 |    "outputs": [
137 |     {
138 |      "name": "stdout",
139 |      "output_type": "stream",
140 |      "text": [
141 |       "Tensor(\"a_4:0\", shape=(), dtype=float32)\n"
142 |      ]
143 |     }
144 |    ],
145 |    "source": [
146 |     "c=tf.constant(25,name='a', dtype=float)\n",
147 |     "print(c)"
148 |    ]
149 |   },
150 |   {
151 |    "cell_type": "code",
152 |    "execution_count": 30,
153 |    "metadata": {},
154 |    "outputs": [
155 |     {
156 |      "name": "stdout",
157 |      "output_type": "stream",
158 |      "text": [
159 |       "Tensor(\"b_1:0\", shape=(), dtype=int32)\n"
160 |      ]
161 |     }
162 |    ],
163 |    "source": [
164 |     "d=tf.constant(10,name='b')\n",
165 |     "print(d)"
166 |    ]
167 |   },
168 |   {
169 |    "cell_type": "markdown",
170 |    "metadata": {},
171 |    "source": [
172 |     "### Operations\n",
173 |     "tf.add, tf.subtract, tf.multiply, tf.scalar_mul, tf.div, tf.divide"
174 |    ]
175 |   },
176 |   {
177 |    "cell_type": "markdown",
178 |    "metadata": {},
179 |    "source": [
180 |     "## Data Flow Graphs"
181 |    ]
182 |   },
183 |   {
184 |    "cell_type": "code",
185 |    "execution_count": 2,
186 |    "metadata": {},
187 |    "outputs": [
188 |     {
189 |      "name": "stdout",
190 |      "output_type": "stream",
191 |      "text": [
192 |       "Tensor(\"Add:0\", shape=(), dtype=int32)\n"
193 |      ]
194 |     }
195 |    ],
196 |    "source": [
197 |     "a = tf.add(3, 5)\n",
198 |     "print(a)"
199 |    ]
200 |   },
201 |   {
202 |    "cell_type": "markdown",
203 |    "metadata": {},
204 |    "source": [
205 |     "**Nodes: operators, variables, and constants Edges: tensors\n",
206 |     "Tensors are data.**  \n",
207 |     "**TensorFlow = tensor + flow = data + flow (I know, mind=blown)**"
208 |    ]
209 |   },
210 |   {
211 |    "cell_type": "markdown",
212 |    "metadata": {},
213 |    "source": [
214 |     "##  How to get the value of a?\n",
215 |     "\n",
216 |     "**Create a session, assign it to variable sess so we can call it later**   \n",
217 |     "**Within the session, evaluate the graph to fetch the value of a**"
218 |    ]
219 |   },
220 |   {
221 |    "cell_type": "markdown",
222 |    "metadata": {},
223 |    "source": [
224 |     "**tf.Session.run(fetches, feed_dict=None, options=None, run_metadata=None)**  \n",
225 |     "**fetches is a list of tensors whose values you want**  "
226 |    ]
227 |   },
228 |   {
229 |    "cell_type": "code",
230 |    "execution_count": 33,
231 |    "metadata": {},
232 |    "outputs": [
233 |     {
234 |      "name": "stdout",
235 |      "output_type": "stream",
236 |      "text": [
237 |       "8\n"
238 |      ]
239 |     }
240 |    ],
241 |    "source": [
242 |     "sess = tf.Session()\n",
243 |     "print(sess.run(a))\n",
244 |     "sess.close()"
245 |    ]
246 |   },
247 |   {
248 |    "cell_type": "code",
249 |    "execution_count": 5,
250 |    "metadata": {},
251 |    "outputs": [
252 |     {
253 |      "name": "stdout",
254 |      "output_type": "stream",
255 |      "text": [
256 |       "8\n"
257 |      ]
258 |     }
259 |    ],
260 |    "source": [
261 |     "with tf.Session() as sess:\n",
262 |     "    print(sess.run(a))"
263 |    ]
264 |   },
265 |   {
266 |    "cell_type": "markdown",
267 |    "metadata": {},
268 |    "source": [
269 |     " **Note: A Session object encapsulates the environment in which Operation objects are executed, and Tensor objects are evaluated.**\n",
270 |     " \n",
271 |     " **Note: Session will also allocate memory to store the current values of variables.**"
272 |    ]
273 |   },
274 |   {
275 |    "cell_type": "markdown",
276 |    "metadata": {},
277 |    "source": [
278 |     "## Basic Math Operations\n",
279 |     "\n",
280 |     "tf.add_n, tf.abs, tf.negative, tf.sign, tf.reciprocal, tf.square, tf.round, tf.sqrt"
281 |    ]
282 |   },
283 |   {
284 |    "cell_type": "code",
285 |    "execution_count": 34,
286 |    "metadata": {},
287 |    "outputs": [
288 |     {
289 |      "name": "stdout",
290 |      "output_type": "stream",
291 |      "text": [
292 |       "Tensor(\"Abs:0\", shape=(), dtype=int32)\n"
293 |      ]
294 |     }
295 |    ],
296 |    "source": [
297 |     "a = tf.abs(-5)\n",
298 |     "print(a)"
299 |    ]
300 |   },
301 |   {
302 |    "cell_type": "code",
303 |    "execution_count": 35,
304 |    "metadata": {},
305 |    "outputs": [
306 |     {
307 |      "name": "stdout",
308 |      "output_type": "stream",
309 |      "text": [
310 |       "5\n"
311 |      ]
312 |     }
313 |    ],
314 |    "source": [
315 |     "with tf.Session() as sess:\n",
316 |     "    print(sess.run(a))"
317 |    ]
318 |   },
319 |   {
320 |    "cell_type": "markdown",
321 |    "metadata": {},
322 |    "source": [
323 |     "## More Graphs"
324 |    ]
325 |   },
326 |   {
327 |    "cell_type": "code",
328 |    "execution_count": 6,
329 |    "metadata": {},
330 |    "outputs": [],
331 |    "source": [
332 |     "x=2\n",
333 |     "y= 3\n",
334 |     "op1 = tf.add(x, y)\n",
335 |     "op2 = tf.multiply(x, y)  # Note that tf.mul() was removed from TensorFlow\n",
336 |     "op3 = tf.pow(op2, op1) \n",
337 |     "with tf.Session() as sess:\n",
338 |     "     op3 = sess.run(op3)"
339 |    ]
340 |   },
341 |   {
342 |    "cell_type": "markdown",
343 |    "metadata": {},
344 |    "source": [
345 |     "**Because we only want the value of pow_op and pow_op doesn’t depend on useless in the following code, session won’t compute value of useless**"
346 |    ]
347 |   },
348 |   {
349 |    "cell_type": "code",
350 |    "execution_count": 7,
351 |    "metadata": {},
352 |    "outputs": [],
353 |    "source": [
354 |     "x= 2\n",
355 |     "y= 3\n",
356 |     "add_op = tf.add(x, y)\n",
357 |     "mul_op = tf.multiply(x, y)\n",
358 |     "useless = tf.multiply(x, add_op) \n",
359 |     "pow_op = tf.pow(add_op, mul_op) \n",
360 |     "with tf.Session() as sess:\n",
361 |     "     z = sess.run(pow_op)"
362 |    ]
363 |   },
364 |   {
365 |    "cell_type": "markdown",
366 |    "metadata": {},
367 |    "source": [
368 |     "## Subgraphs \n",
369 |     "Possible to break graphs into several chunks and run them parallelly across multiple CPUs, GPUs, or devices"
370 |    ]
371 |   },
372 |   {
373 |    "cell_type": "code",
374 |    "execution_count": 8,
375 |    "metadata": {},
376 |    "outputs": [],
377 |    "source": [
378 |     "# # To put part of a graph on a specific CPU or GPU:\n",
379 |     "# # Creates a graph.\n",
380 |     "# with tf.device('/gpu:2'):\n",
381 |     "#   a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name='a')\n",
382 |     "#   b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name='b')\n",
383 |     "#   c = tf.matmul(a, b)\n",
384 |     "# # Creates a session with log_device_placement set to True.\n",
385 |     "# sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
386 |     "#  # Runs the op.\n",
387 |     "# print(sess.run(c))"
388 |    ]
389 |   },
390 |   {
391 |    "cell_type": "markdown",
392 |    "metadata": {},
393 |    "source": [
394 |     "## Multipe \n",
395 |     "* Multiple graphs require multiple sessions, each will try to use all available resources by default\n",
396 |     "* Can't pass data between them without passing them through python/numpy, which doesn't work in distributed\n",
397 |     "* It’s better to have disconnected subgraphs within one graph"
398 |    ]
399 |   },
400 |   {
401 |    "cell_type": "code",
402 |    "execution_count": 9,
403 |    "metadata": {},
404 |    "outputs": [],
405 |    "source": [
406 |     "# create a graph:\n",
407 |     "g = tf.Graph()"
408 |    ]
409 |   },
410 |   {
411 |    "cell_type": "code",
412 |    "execution_count": 19,
413 |    "metadata": {},
414 |    "outputs": [
415 |     {
416 |      "name": "stdout",
417 |      "output_type": "stream",
418 |      "text": [
419 |       "8\n"
420 |      ]
421 |     }
422 |    ],
423 |    "source": [
424 |     "# to add operators to a graph, set it as default:\n",
425 |     "g = tf.Graph()\n",
426 |     "with g.as_default():\n",
427 |     "     x = tf.add(3, 5)\n",
428 |     "with tf.Session(graph=g) as sess:\n",
429 |     "     print(sess.run(x))"
430 |    ]
431 |   },
432 |   {
433 |    "cell_type": "code",
434 |    "execution_count": 18,
435 |    "metadata": {},
436 |    "outputs": [
437 |     {
438 |      "name": "stdout",
439 |      "output_type": "stream",
440 |      "text": [
441 |       "8\n"
442 |      ]
443 |     }
444 |    ],
445 |    "source": [
446 |     "# to add operators to a graph, set it as default:\n",
447 |     "g = tf.Graph()\n",
448 |     "with g.as_default():\n",
449 |     "    a=3\n",
450 |     "    b= 5\n",
451 |     "    x = tf.add(a, b)\n",
452 |     "sess = tf.Session(graph=g) # session is run on the graph g\n",
453 |     "print(sess.run(x))\n",
454 |     "sess.close()"
455 |    ]
456 |   },
457 |   {
458 |    "cell_type": "code",
459 |    "execution_count": 21,
460 |    "metadata": {},
461 |    "outputs": [],
462 |    "source": [
463 |     "# To handle the default graph:\n",
464 |     "g = tf.get_default_graph()"
465 |    ]
466 |   },
467 |   {
468 |    "cell_type": "code",
469 |    "execution_count": 22,
470 |    "metadata": {},
471 |    "outputs": [],
472 |    "source": [
473 |     "# Do not mix default graph and user created graphs\n",
474 |     "g = tf.Graph()\n",
475 |     "# add ops to the default graph\n",
476 |     "a = tf.constant(3)\n",
477 |     "# add ops to the user created graph\n",
478 |     "with g.as_default():\n",
479 |     "     b = tf.constant(5)"
480 |    ]
481 |   },
482 |   {
483 |    "cell_type": "code",
484 |    "execution_count": 24,
485 |    "metadata": {},
486 |    "outputs": [],
487 |    "source": [
488 |     "# Do not mix default graph and user created graphs\n",
489 |     "g1 = tf.get_default_graph()\n",
490 |     "g2 = tf.Graph()\n",
491 |     "# add ops to the default graph\n",
492 |     "with g1.as_default():\n",
493 |     "     a = tf.constant(3)\n",
494 |     "# add ops to the user created graph\n",
495 |     "with g2.as_default():\n",
496 |     "     b = tf.constant(5)"
497 |    ]
498 |   },
499 |   {
500 |    "cell_type": "markdown",
501 |    "metadata": {},
502 |    "source": [
503 |     "## Why graphs\n",
504 |     "1. Save computation (only run subgraphs that lead to the values you want to fetch)\n",
505 |     "2. Break computation into small, differential pieces to facilitates auto-differentiation\n",
506 |     "3. Facilitate distributed computation, spread the work across multiple CPUs, GPUs, or devices\n",
507 |     "4. Many common machine learning models are commonly taught and visualized as directed graphs already\n",
508 |     " "
509 |    ]
510 |   },
511 |   {
512 |    "cell_type": "markdown",
513 |    "metadata": {},
514 |    "source": [
515 |     "## Next class\n",
516 |     "* Basic operations\n",
517 |     "* Constants and variables\n",
518 |     "* Feeding inputs\n",
519 |     "* Fun with TensorBoard \n",
520 |     "* Feedback: huyenn@stanford.edu \n",
521 |     "\n",
522 |     "Thanks!\n",
523 |     " "
524 |    ]
525 |   }
526 |  ],
527 |  "metadata": {
528 |   "kernelspec": {
529 |    "display_name": "Python 3",
530 |    "language": "python",
531 |    "name": "python3"
532 |   },
533 |   "language_info": {
534 |    "codemirror_mode": {
535 |     "name": "ipython",
536 |     "version": 3
537 |    },
538 |    "file_extension": ".py",
539 |    "mimetype": "text/x-python",
540 |    "name": "python",
541 |    "nbconvert_exporter": "python",
542 |    "pygments_lexer": "ipython3",
543 |    "version": "3.6.7"
544 |   }
545 |  },
546 |  "nbformat": 4,
547 |  "nbformat_minor": 2
548 | }
549 | 


--------------------------------------------------------------------------------
/NoteBooks/Lec3.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Useful Operations in TensorFlow\n",
  8 |     "\n",
  9 |     "Adapted from [A Sweeping Tour of TensorFlow](http://www.goldsborough.me/tensorflow/ml/ai/python/2017/06/28/20-21-45-a_sweeping_tour_of_tensorflow/)\n",
 10 |     "by Peter Goldsborough"
 11 |    ]
 12 |   },
 13 |   {
 14 |    "cell_type": "code",
 15 |    "execution_count": 21,
 16 |    "metadata": {},
 17 |    "outputs": [],
 18 |    "source": [
 19 |     "import os\n",
 20 |     "os.environ['TF_CPP_MIN_LOG_LEVEL']='2' # No more warning\n",
 21 |     "import tensorflow as tf"
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "markdown",
 26 |    "metadata": {},
 27 |    "source": [
 28 |     "## If-Statements  \n",
 29 |     "\n",
 30 |     "**tf.cond(\n",
 31 |     "    pred,\n",
 32 |     "    true_fn=None,\n",
 33 |     "    false_fn=None,\n",
 34 |     "    strict=False,\n",
 35 |     "    name=None,\n",
 36 |     ")**\n",
 37 |     "\n",
 38 |     "The following document was adapted from  [TF documentation](https://www.tensorflow.org/api_docs/python/tf/cond) of this operator. "
 39 |    ]
 40 |   },
 41 |   {
 42 |    "cell_type": "markdown",
 43 |    "metadata": {},
 44 |    "source": [
 45 |     "This operation returns true_fn() if the predicate pred is true else false_fn(). "
 46 |    ]
 47 |   },
 48 |   {
 49 |    "cell_type": "code",
 50 |    "execution_count": 2,
 51 |    "metadata": {},
 52 |    "outputs": [
 53 |     {
 54 |      "name": "stdout",
 55 |      "output_type": "stream",
 56 |      "text": [
 57 |       "WARNING:tensorflow:From /Users/hhaji/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py:423: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
 58 |       "Instructions for updating:\n",
 59 |       "Colocations handled automatically by placer.\n"
 60 |      ]
 61 |     }
 62 |    ],
 63 |    "source": [
 64 |     "x = tf.constant(3)\n",
 65 |     "y = tf.constant(4)\n",
 66 |     "z = tf.multiply(x, y)\n",
 67 |     "result = tf.cond(x < y, lambda: tf.add(x, z), lambda: tf.square(y))"
 68 |    ]
 69 |   },
 70 |   {
 71 |    "cell_type": "code",
 72 |    "execution_count": 3,
 73 |    "metadata": {},
 74 |    "outputs": [
 75 |     {
 76 |      "name": "stdout",
 77 |      "output_type": "stream",
 78 |      "text": [
 79 |       "49\n",
 80 |       "70\n"
 81 |      ]
 82 |     }
 83 |    ],
 84 |    "source": [
 85 |     "with tf.Session() as session:\n",
 86 |     "    print(session.run(result, feed_dict={x: 9, y:7}))\n",
 87 |     "    print(session.run(result, feed_dict={x: 7, y:9}))"
 88 |    ]
 89 |   },
 90 |   {
 91 |    "cell_type": "code",
 92 |    "execution_count": 4,
 93 |    "metadata": {},
 94 |    "outputs": [],
 95 |    "source": [
 96 |     "def f1(): \n",
 97 |     "    return tf.multiply(x, 17)\n",
 98 |     "def f2(): \n",
 99 |     "    return tf.add(y, 23)\n",
100 |     "result = tf.cond(tf.less(x, y), f1, f2)"
101 |    ]
102 |   },
103 |   {
104 |    "cell_type": "code",
105 |    "execution_count": 5,
106 |    "metadata": {},
107 |    "outputs": [
108 |     {
109 |      "name": "stdout",
110 |      "output_type": "stream",
111 |      "text": [
112 |       "30\n",
113 |       "119\n"
114 |      ]
115 |     }
116 |    ],
117 |    "source": [
118 |     "with tf.Session() as session:\n",
119 |     "    print(session.run(result, feed_dict={x: 9, y:7}))\n",
120 |     "    print(session.run(result, feed_dict={x: 7, y:9}))"
121 |    ]
122 |   },
123 |   {
124 |    "cell_type": "code",
125 |    "execution_count": 6,
126 |    "metadata": {},
127 |    "outputs": [
128 |     {
129 |      "name": "stdout",
130 |      "output_type": "stream",
131 |      "text": [
132 |       "[1 2 3]\n",
133 |       "[4 5 6]\n"
134 |      ]
135 |     }
136 |    ],
137 |    "source": [
138 |     "a = tf.constant([1, 2, 3])\n",
139 |     "b = tf.constant([4, 5, 6])\n",
140 |     "x = tf.placeholder(shape=[], dtype=tf.bool)\n",
141 |     "\n",
142 |     "c = tf.cond(x, lambda: a, lambda: b)\n",
143 |     "\n",
144 |     "with tf.Session() as session:\n",
145 |     "    print(session.run(c, feed_dict={x: True}))\n",
146 |     "    print(session.run(c, feed_dict={x: False}))"
147 |    ]
148 |   },
149 |   {
150 |    "cell_type": "markdown",
151 |    "metadata": {},
152 |    "source": [
153 |     "## While Loops \n",
154 |     "**tf.while_loop(\n",
155 |     "    cond,\n",
156 |     "    body,\n",
157 |     "    loop_vars,\n",
158 |     "    shape_invariants=None,\n",
159 |     "    parallel_iterations=10,\n",
160 |     "    back_prop=True,\n",
161 |     "    swap_memory=False,\n",
162 |     "    name=None,\n",
163 |     "    maximum_iterations=None,\n",
164 |     "    return_same_structure=False\n",
165 |     ")**  "
166 |    ]
167 |   },
168 |   {
169 |    "cell_type": "markdown",
170 |    "metadata": {},
171 |    "source": [
172 |     "The following document was adapted from  [TF documentation](https://www.tensorflow.org/api_docs/python/tf/while_loop) of this operator. \n",
173 |     "\n",
174 |     "**tf.while_loop repeat body while the condition cond is true.** cond is a **callable** returning a boolean scalar tensor. body is a **callable** returning a (possibly nested) tuple, namedtuple or list of tensors of the same arity (length and structure) and types as loop_vars. loop_vars is a (possibly nested) tuple, namedtuple or list of tensors that is passed to both cond and body. cond and body both take as many arguments as there are loop_vars.  \n",
175 |     "\n",
176 |     "**Returns:**  \n",
177 |     "\n",
178 |     "The output tensors for the loop variables after the loop. If return_same_structure is True, the return value has the same structure as loop_vars. If return_same_structure is False, the return value is a Tensor, TensorArray or IndexedSlice if the length of loop_vars is 1, or a list otherwise."
179 |    ]
180 |   },
181 |   {
182 |    "cell_type": "code",
183 |    "execution_count": 7,
184 |    "metadata": {},
185 |    "outputs": [
186 |     {
187 |      "name": "stdout",
188 |      "output_type": "stream",
189 |      "text": [
190 |       "16\n"
191 |      ]
192 |     }
193 |    ],
194 |    "source": [
195 |     "i = tf.constant(1)\n",
196 |     "cond = lambda i: tf.less(i, 10)\n",
197 |     "body = lambda i: tf.multiply(i, 2)\n",
198 |     "r = tf.while_loop(cond, body, [i])\n",
199 |     "with tf.Session():\n",
200 |     "    print(r.eval())"
201 |    ]
202 |   },
203 |   {
204 |    "cell_type": "code",
205 |    "execution_count": 8,
206 |    "metadata": {},
207 |    "outputs": [
208 |     {
209 |      "name": "stdout",
210 |      "output_type": "stream",
211 |      "text": [
212 |       "21\n"
213 |      ]
214 |     }
215 |    ],
216 |    "source": [
217 |     "i = tf.constant(1)\n",
218 |     "var_new = tf.while_loop(lambda i: i < 20, lambda i: i + 2, [i])\n",
219 |     "with tf.Session():\n",
220 |     "    print(var_new.eval())"
221 |    ]
222 |   },
223 |   {
224 |    "cell_type": "code",
225 |    "execution_count": 9,
226 |    "metadata": {},
227 |    "outputs": [
228 |     {
229 |      "name": "stdout",
230 |      "output_type": "stream",
231 |      "text": [
232 |       "[10, array([[1, 2],\n",
233 |       "       [3, 4],\n",
234 |       "       [1, 2],\n",
235 |       "       ...,\n",
236 |       "       [3, 4],\n",
237 |       "       [1, 2],\n",
238 |       "       [3, 4]], dtype=int32)]\n"
239 |      ]
240 |     }
241 |    ],
242 |    "source": [
243 |     "i0 = tf.constant(0)\n",
244 |     "m0 = tf.constant([[1,2], [3,4]])\n",
245 |     "cond = lambda i, m: i < 10\n",
246 |     "body = lambda i, m: [i+1, tf.concat([m, m], axis=0)]\n",
247 |     "matrix =tf.while_loop(cond, body, loop_vars=[i0, m0],\n",
248 |     "    shape_invariants=[i0.get_shape(), tf.TensorShape([None, 2])])\n",
249 |     "with tf.Session() as sess:\n",
250 |     "    print(sess.run(matrix))"
251 |    ]
252 |   },
253 |   {
254 |    "cell_type": "markdown",
255 |    "metadata": {},
256 |    "source": [
257 |     "## Convert a Function of Python to TensorFlow Operation (tf.py_func)\n",
258 |     "\n",
259 |     "**tf.py_func(\n",
260 |     "    func,\n",
261 |     "    inp,\n",
262 |     "    Tout,\n",
263 |     "    stateful=True,\n",
264 |     "    name=None\n",
265 |     ")**"
266 |    ]
267 |   },
268 |   {
269 |    "cell_type": "markdown",
270 |    "metadata": {},
271 |    "source": [
272 |     "tf.py_func wraps a python function and uses it as a TensorFlow op. The following document was adapted from  [TF documentation](https://docs.w3cub.com/tensorflow~python/tf/py_func/) of this operator. **Note that the tf.py_func operation will only run on CPU.** "
273 |    ]
274 |   },
275 |   {
276 |    "cell_type": "markdown",
277 |    "metadata": {},
278 |    "source": [
279 |     "Given a python function func, which takes numpy arrays as its arguments and returns numpy arrays as its outputs, wrap this function as an operation in a TensorFlow graph. The following snippet constructs a simple TensorFlow graph that invokes the np.sinh() NumPy function as a operation in the graph:"
280 |    ]
281 |   },
282 |   {
283 |    "cell_type": "code",
284 |    "execution_count": 10,
285 |    "metadata": {},
286 |    "outputs": [],
287 |    "source": [
288 |     "def my_func1(x):\n",
289 |     "  return 2**x\n",
290 |     "input_1 = tf.placeholder(tf.int64)\n",
291 |     "op_1= tf.py_function(my_func1, [input_1], tf.int64)"
292 |    ]
293 |   },
294 |   {
295 |    "cell_type": "code",
296 |    "execution_count": 11,
297 |    "metadata": {},
298 |    "outputs": [
299 |     {
300 |      "name": "stdout",
301 |      "output_type": "stream",
302 |      "text": [
303 |       "32\n"
304 |      ]
305 |     }
306 |    ],
307 |    "source": [
308 |     "with tf.Session() as sess:\n",
309 |     "    print(sess.run(op_1, feed_dict={input_1:5}))"
310 |    ]
311 |   },
312 |   {
313 |    "cell_type": "code",
314 |    "execution_count": 12,
315 |    "metadata": {},
316 |    "outputs": [
317 |     {
318 |      "name": "stdout",
319 |      "output_type": "stream",
320 |      "text": [
321 |       "WARNING:tensorflow:From <ipython-input-12-0a0bb1353a74>:5: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.\n",
322 |       "Instructions for updating:\n",
323 |       "tf.py_func is deprecated in TF V2. Instead, use\n",
324 |       "    tf.py_function, which takes a python function which manipulates tf eager\n",
325 |       "    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to\n",
326 |       "    an ndarray (just call tensor.numpy()) but having access to eager tensors\n",
327 |       "    means `tf.py_function`s can use accelerators such as GPUs as well as\n",
328 |       "    being differentiable using a gradient tape.\n",
329 |       "    \n"
330 |      ]
331 |     }
332 |    ],
333 |    "source": [
334 |     "def my_func2(x):\n",
335 |     "  # x will be a numpy array with the contents of the placeholder below\n",
336 |     "  return np.sinh(x)\n",
337 |     "input= tf.placeholder(tf.float32)\n",
338 |     "op_2= tf.py_func(my_func2, [input], tf.float32)"
339 |    ]
340 |   },
341 |   {
342 |    "cell_type": "code",
343 |    "execution_count": 13,
344 |    "metadata": {},
345 |    "outputs": [
346 |     {
347 |      "name": "stdout",
348 |      "output_type": "stream",
349 |      "text": [
350 |       "8.2581813e+37\n"
351 |      ]
352 |     }
353 |    ],
354 |    "source": [
355 |     "import numpy as np\n",
356 |     "with tf.Session() as sess:\n",
357 |     "    print(sess.run(op_2, feed_dict={input:88}))"
358 |    ]
359 |   },
360 |   {
361 |    "cell_type": "markdown",
362 |    "metadata": {},
363 |    "source": [
364 |     "## Variable Scopes"
365 |    ]
366 |   },
367 |   {
368 |    "cell_type": "markdown",
369 |    "metadata": {},
370 |    "source": [
371 |     "Variable scopes allow you to control variable reuse when calling functions which implicitly create and use variables. They also allow you to name your variables in a hierarchical and understandable way. The following document was adapted from  [TF documentation](https://www.tensorflow.org/api_docs/python/tf/variable_scope) of this operator.\n",
372 |     "\n",
373 |     "Simple example of how to create a new variable:"
374 |    ]
375 |   },
376 |   {
377 |    "cell_type": "code",
378 |    "execution_count": 14,
379 |    "metadata": {},
380 |    "outputs": [],
381 |    "source": [
382 |     "with tf.variable_scope(\"big\"):\n",
383 |     "    with tf.variable_scope(\"bar\"):\n",
384 |     "        v = tf.get_variable(\"v\", [1])\n",
385 |     "        assert v.name == \"big/bar/v:0\""
386 |    ]
387 |   },
388 |   {
389 |    "cell_type": "code",
390 |    "execution_count": 15,
391 |    "metadata": {},
392 |    "outputs": [
393 |     {
394 |      "data": {
395 |       "text/plain": [
396 |        "<tf.Variable 'big/bar/v:0' shape=(1,) dtype=float32_ref>"
397 |       ]
398 |      },
399 |      "execution_count": 15,
400 |      "metadata": {},
401 |      "output_type": "execute_result"
402 |     }
403 |    ],
404 |    "source": [
405 |     "v"
406 |    ]
407 |   },
408 |   {
409 |    "cell_type": "markdown",
410 |    "metadata": {},
411 |    "source": [
412 |     "**Basic example of sharing a variable AUTO_REUSE:**  "
413 |    ]
414 |   },
415 |   {
416 |    "cell_type": "code",
417 |    "execution_count": 16,
418 |    "metadata": {},
419 |    "outputs": [],
420 |    "source": [
421 |     "# AUTO_REUSE: we create variables if they do not exist, and return them otherwise; if None, we inherit \n",
422 |     "# the parent scope's reuse flag.\n",
423 |     "def make_var():\n",
424 |     "    with tf.variable_scope(\"make_var\", reuse=tf.AUTO_REUSE):\n",
425 |     "        v = tf.get_variable(\"new_v\", dtype=tf.int32, initializer=tf.constant([1,2]))\n",
426 |     "    return v\n",
427 |     "\n",
428 |     "v1=make_var()  # Creates v\n",
429 |     "v2=make_var()  # Gets the same, existing v\n",
430 |     "assert v1 == v2"
431 |    ]
432 |   },
433 |   {
434 |    "cell_type": "code",
435 |    "execution_count": 17,
436 |    "metadata": {},
437 |    "outputs": [
438 |     {
439 |      "name": "stdout",
440 |      "output_type": "stream",
441 |      "text": [
442 |       "[1 2]\n",
443 |       "[1 2]\n"
444 |      ]
445 |     }
446 |    ],
447 |    "source": [
448 |     "with tf.Session() as sess:\n",
449 |     "    sess.run(tf.global_variables_initializer())\n",
450 |     "    print(sess.run(v1))\n",
451 |     "    print(sess.run(v2))"
452 |    ]
453 |   },
454 |   {
455 |    "cell_type": "markdown",
456 |    "metadata": {},
457 |    "source": [
458 |     "Basic example of sharing a variable with reuse=True:"
459 |    ]
460 |   },
461 |   {
462 |    "cell_type": "code",
463 |    "execution_count": 18,
464 |    "metadata": {},
465 |    "outputs": [],
466 |    "source": [
467 |     "with tf.variable_scope(\"foo\"):\n",
468 |     "    v = tf.get_variable(\"my_vec\", [1])\n",
469 |     "with tf.variable_scope(\"foo\", reuse=True):\n",
470 |     "    v1 = tf.get_variable(\"my_vec\", [1])\n",
471 |     "assert v1 == v"
472 |    ]
473 |   },
474 |   {
475 |    "cell_type": "markdown",
476 |    "metadata": {},
477 |    "source": [
478 |     "To prevent accidental sharing of variables, we raise an exception when getting an existing variable in a non-reusing scope."
479 |    ]
480 |   },
481 |   {
482 |    "cell_type": "code",
483 |    "execution_count": 19,
484 |    "metadata": {},
485 |    "outputs": [],
486 |    "source": [
487 |     "# #  Raises ValueError(\"... v already exists ...\").\n",
488 |     "# with tf.variable_scope(\"foo\"):\n",
489 |     "#     v = tf.get_variable(\"v\", [1])\n",
490 |     "#     v1= tf.get_variable(\"v\", [1])"
491 |    ]
492 |   },
493 |   {
494 |    "cell_type": "markdown",
495 |    "metadata": {},
496 |    "source": [
497 |     "Similarly, we raise an exception when trying to get a variable that does not exist in reuse mode."
498 |    ]
499 |   },
500 |   {
501 |    "cell_type": "code",
502 |    "execution_count": 20,
503 |    "metadata": {},
504 |    "outputs": [],
505 |    "source": [
506 |     "# #  Raises ValueError(\"... v does not exists ...\").\n",
507 |     "# with tf.variable_scope(\"foo\", reuse=True):\n",
508 |     "#     v = tf.get_variable(\"new_variable\", [1])"
509 |    ]
510 |   }
511 |  ],
512 |  "metadata": {
513 |   "kernelspec": {
514 |    "display_name": "Python 3",
515 |    "language": "python",
516 |    "name": "python3"
517 |   },
518 |   "language_info": {
519 |    "codemirror_mode": {
520 |     "name": "ipython",
521 |     "version": 3
522 |    },
523 |    "file_extension": ".py",
524 |    "mimetype": "text/x-python",
525 |    "name": "python",
526 |    "nbconvert_exporter": "python",
527 |    "pygments_lexer": "ipython3",
528 |    "version": "3.6.8"
529 |   }
530 |  },
531 |  "nbformat": 4,
532 |  "nbformat_minor": 2
533 | }
534 | 


--------------------------------------------------------------------------------
/NoteBooks/README.MD:
--------------------------------------------------------------------------------
1 | # NoteBooks
2 | 


--------------------------------------------------------------------------------
/NoteBooks/The-Confusing-Parts-2.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# TensorFlow: The Confusing Parts (2)\n",
  8 |     "\n",
  9 |     "This document was adapted from  [TensorFlow: The Confusing Parts (2)](https://jacobbuckman.com/post/tensorflow-the-confusing-parts-2/) by Jacob Buckman."
 10 |    ]
 11 |   },
 12 |   {
 13 |    "cell_type": "markdown",
 14 |    "metadata": {},
 15 |    "source": [
 16 |     "This post is the second of a series; click [here](https://jacobbuckman.com/post/tensorflow-the-confusing-parts-1/) for the previous post, or [here](https://jacobbuckman.com/categories/tftcp/) for a list of all posts in this series."
 17 |    ]
 18 |   },
 19 |   {
 20 |    "cell_type": "markdown",
 21 |    "metadata": {},
 22 |    "source": [
 23 |     "## Naming and Scoping"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "markdown",
 28 |    "metadata": {},
 29 |    "source": [
 30 |     "### Naming Variables and Tensors\n",
 31 |     "As we discussed in Part 1, every time you call tf.get_variable(), you need to assign the variable a new, unique name. Actually, it goes deeper than that: every tensor in the graph gets a unique name too. The name can be accessed explicitly with the .name property of tensors, operations, and variables. For the vast majority of cases, the name will be created automatically for you; for example, a constant node will have the name Const, and as you create more of them, they will become Const_1, Const_2, etc.1 You can also explicitly set the name of a node via the name= property, and the enumerative suffix will still be added automatically:"
 32 |    ]
 33 |   },
 34 |   {
 35 |    "cell_type": "code",
 36 |    "execution_count": 1,
 37 |    "metadata": {},
 38 |    "outputs": [
 39 |     {
 40 |      "name": "stderr",
 41 |      "output_type": "stream",
 42 |      "text": [
 43 |       "/Users/hhaji/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
 44 |       "  from ._conv import register_converters as _register_converters\n"
 45 |      ]
 46 |     },
 47 |     {
 48 |      "name": "stdout",
 49 |      "output_type": "stream",
 50 |      "text": [
 51 |       "Const:0 Const_1:0 cool_const:0 cool_const_1:0\n"
 52 |      ]
 53 |     }
 54 |    ],
 55 |    "source": [
 56 |     "import tensorflow as tf\n",
 57 |     "a = tf.constant(0.)\n",
 58 |     "b = tf.constant(1.)\n",
 59 |     "c = tf.constant(2., name=\"cool_const\")\n",
 60 |     "d = tf.constant(3., name=\"cool_const\")\n",
 61 |     "print(a.name, b.name, c.name, d.name)"
 62 |    ]
 63 |   },
 64 |   {
 65 |    "cell_type": "markdown",
 66 |    "metadata": {},
 67 |    "source": [
 68 |     "Explicitly naming nodes is nonessential, but can be very useful when debugging. Oftentimes, when your Tensorflow code crashes, the error trace will refer to a specific operation. If you have many operations of the same type, it can be tough to figure out which one is problematic. By explicitly naming each of your nodes, you can get much more informative error traces, and identify the issue more quickly."
 69 |    ]
 70 |   },
 71 |   {
 72 |    "cell_type": "markdown",
 73 |    "metadata": {},
 74 |    "source": [
 75 |     "### Using Scopes\n",
 76 |     "As your graph gets more complex, it becomes difficult to name everything by hand. Tensorflow provides the tf.variable_scope object, which makes it easier to organize your graphs by subdividing them into smaller chunks. By simply wrapping a segment of your graph creation code in a with tf.variable_scope(scope_name): statement, all nodes created will have their names automatically prefixed with the scope_name string. Additionally, these scopes stack; creating a scope within another will simply chain the prefixes together, delimited by a forward-slash."
 77 |    ]
 78 |   },
 79 |   {
 80 |    "cell_type": "code",
 81 |    "execution_count": 2,
 82 |    "metadata": {},
 83 |    "outputs": [
 84 |     {
 85 |      "name": "stdout",
 86 |      "output_type": "stream",
 87 |      "text": [
 88 |       "WARNING:tensorflow:From /Users/hhaji/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
 89 |       "Instructions for updating:\n",
 90 |       "Colocations handled automatically by placer.\n",
 91 |       "Const_2:0 Const_3:0\n"
 92 |      ]
 93 |     }
 94 |    ],
 95 |    "source": [
 96 |     "a = tf.constant(0.)\n",
 97 |     "b = tf.constant(1.)\n",
 98 |     "with tf.variable_scope(\"first_scope\"):\n",
 99 |     "  c = a + b\n",
100 |     "  d = tf.constant(2., name=\"cool_const\")\n",
101 |     "  coef1 = tf.get_variable(\"coef\", [], initializer=tf.constant_initializer(2.))\n",
102 |     "  with tf.variable_scope(\"second_scope\"):\n",
103 |     "    e = coef1 * d\n",
104 |     "    coef2 = tf.get_variable(\"coef\", [], initializer=tf.constant_initializer(3.))\n",
105 |     "    f = tf.constant(1.)\n",
106 |     "    g = coef2 * f \n",
107 |     "print(a.name, b.name)"
108 |    ]
109 |   },
110 |   {
111 |    "cell_type": "code",
112 |    "execution_count": 3,
113 |    "metadata": {},
114 |    "outputs": [
115 |     {
116 |      "name": "stdout",
117 |      "output_type": "stream",
118 |      "text": [
119 |       "first_scope/add:0 first_scope/cool_const:0\n"
120 |      ]
121 |     }
122 |    ],
123 |    "source": [
124 |     "print(c.name, d.name)"
125 |    ]
126 |   },
127 |   {
128 |    "cell_type": "code",
129 |    "execution_count": 4,
130 |    "metadata": {},
131 |    "outputs": [
132 |     {
133 |      "name": "stdout",
134 |      "output_type": "stream",
135 |      "text": [
136 |       "first_scope/second_scope/mul:0 first_scope/second_scope/Const:0 first_scope/second_scope/mul_1:0\n"
137 |      ]
138 |     }
139 |    ],
140 |    "source": [
141 |     "print(e.name, f.name, g.name)"
142 |    ]
143 |   },
144 |   {
145 |    "cell_type": "code",
146 |    "execution_count": 5,
147 |    "metadata": {},
148 |    "outputs": [
149 |     {
150 |      "name": "stdout",
151 |      "output_type": "stream",
152 |      "text": [
153 |       "first_scope/coef:0\n",
154 |       "first_scope/second_scope/coef:0\n"
155 |      ]
156 |     }
157 |    ],
158 |    "source": [
159 |     "print(coef1.name)\n",
160 |     "print(coef2.name)"
161 |    ]
162 |   },
163 |   {
164 |    "cell_type": "markdown",
165 |    "metadata": {},
166 |    "source": [
167 |     "Notice that we were able to create two variables with the same name - coef - without any issues! This is because the scoping transformed the names into first_scope/coef:0 and first_scope/second_scope/coef:0, which are distinct."
168 |    ]
169 |   },
170 |   {
171 |    "cell_type": "markdown",
172 |    "metadata": {},
173 |    "source": [
174 |     "## Saving and Loading "
175 |    ]
176 |   },
177 |   {
178 |    "cell_type": "markdown",
179 |    "metadata": {},
180 |    "source": [
181 |     "At its core, a trained neural network consists of two essential components:\n",
182 |     "\n",
183 |     "1. The weights of the network, which have been learned to optimize for some task  \n",
184 |     "2. The network graph, which specifies how to actually use the weights to get results  \n",
185 |     "Tensorflow separates these two components, but it’s clear that they need to be very tightly paired. Weights are useless without a graph structure describing how to use them, and a graph with random weights is no good either. In fact, even something as small as swapping two weight matrices is likely to totally break your model. This often leads to frustration among beginner Tensorflow users; using a pre-trained model as a component of a neural network is a great way to speed up training, but can break things in a myriad of ways."
186 |    ]
187 |   },
188 |   {
189 |    "cell_type": "markdown",
190 |    "metadata": {},
191 |    "source": [
192 |     "### Saving A Model\n",
193 |     "When working with only a single model, Tensorflow’s built-in tools for saving and loading are straightforward to use: simply create a tf.train.Saver(). Similarly to the tf.train.Optimizer family, a tf.train.Saver is not itself a node, but instead a higher-level class that performs useful functions on top of pre-existing graphs. And, as you may have anticipated, the ‘useful function’ of a tf.train.Saver is saving and loading the model. Let’s see it in action!"
194 |    ]
195 |   },
196 |   {
197 |    "cell_type": "code",
198 |    "execution_count": 6,
199 |    "metadata": {},
200 |    "outputs": [],
201 |    "source": [
202 |     "# a = tf.get_variable('a1', [])\n",
203 |     "# b = tf.get_variable('b1', [])\n",
204 |     "# init = tf.global_variables_initializer()\n",
205 |     "\n",
206 |     "# saver = tf.train.Saver()\n",
207 |     "# sess = tf.Session()\n",
208 |     "# sess.run(init)\n",
209 |     "# saver.save(sess, './tftcp.model')\n",
210 |     "# sess.close()"
211 |    ]
212 |   },
213 |   {
214 |    "cell_type": "markdown",
215 |    "metadata": {},
216 |    "source": [
217 |     "### Output\n",
218 |     "\n",
219 |     "Four new files:  \n",
220 |     "\n",
221 |     "checkpoint  \n",
222 |     "tftcp.model.data-00000-of-00001  \n",
223 |     "tftcp.model.index  \n",
224 |     "tftcp.model.meta  "
225 |    ]
226 |   },
227 |   {
228 |    "cell_type": "markdown",
229 |    "metadata": {},
230 |    "source": [
231 |     "There’s a lot of stuff to break down here.\n",
232 |     "\n",
233 |     "First of all: Why does it output four files, when we only saved one model? The information needed to recreate the model is divided among them. If you want to copy or back up a model, make sure you bring all three of the files (the three prefixed by your filename). Here’s a quick description of each:\n",
234 |     "\n",
235 |     "* tftcp.model.data-00000-of-00001 contains the weights of your model (the first bullet point from above). It’s most likely the largest file here.\n",
236 |     "* tftcp.model.meta is the network structure of your model (the second bullet point from above). It contains all the information needed to re-create your graph.\n",
237 |     "* tftcp.model.index is an indexing structure linking the first two things. It says “where in the data file do I find the parameters corresponding to this node?”\n",
238 |     "* checkpoint is not actually needed to reconstruct your model, but if you save multiple versions of your model throughout a training run, it keeps track of everything."
239 |    ]
240 |   },
241 |   {
242 |    "cell_type": "markdown",
243 |    "metadata": {},
244 |    "source": [
245 |     "Well, if we’re going to save a model, we need to have something to save. Recall that computations live in the graph, but values live in the session. The tf.train.Saver can access the structure of the network through a global pointer to the graph. But when we go to save the values of the variables (i.e. the weights of the network), we need to access a tf.Session to see what those values are; that’s why sess is passed in as the first argument of the save function. Additionally, attempting to save uninitialized variables will throw an error, because attempting to access the value of an uninitialized variable always throws an error. So, we needed both a session and an initializer (or equivalent, e.g. tf.assign)."
246 |    ]
247 |   },
248 |   {
249 |    "cell_type": "markdown",
250 |    "metadata": {},
251 |    "source": [
252 |     "### Loading A Model\n",
253 |     "Now that we’ve saved our model, let’s load it back in. The first step is to recreate the variables: we want variables with all the same names, shapes, and dtypes as we had when we saved it. The second step is to create a tf.train.Saver just as before, and call the restore function."
254 |    ]
255 |   },
256 |   {
257 |    "cell_type": "code",
258 |    "execution_count": 7,
259 |    "metadata": {},
260 |    "outputs": [],
261 |    "source": [
262 |     "# a = tf.get_variable('a1', [])\n",
263 |     "# b = tf.get_variable('b1', [])\n",
264 |     "# saver = tf.train.Saver()\n",
265 |     "# sess = tf.Session()\n",
266 |     "# saver.restore(sess, './tftcp.model')\n",
267 |     "# sess.run([a,b])\n",
268 |     "# sess.close()"
269 |    ]
270 |   },
271 |   {
272 |    "cell_type": "markdown",
273 |    "metadata": {},
274 |    "source": [
275 |     "Note that we didn’t need to initialize a or b before running them! This is because the restore operation moves the values from our files into the session’s variables. Since the session no longer contains any null-valued variables, initialization is no longer needed. (This can backfire if we aren’t careful: running an init after a restore will override the loaded values with randomly-initialized ones.)"
276 |    ]
277 |   },
278 |   {
279 |    "cell_type": "markdown",
280 |    "metadata": {},
281 |    "source": [
282 |     "### Choosing Your Variables\n",
283 |     "When a tf.train.Saver is initialized, it looks at the current graph and gets the list of variables; this is permanently stored as the list of variables that that saver “cares about”. We can inspect it with the ._var_list property:"
284 |    ]
285 |   },
286 |   {
287 |    "cell_type": "code",
288 |    "execution_count": 8,
289 |    "metadata": {},
290 |    "outputs": [],
291 |    "source": [
292 |     "# a = tf.get_variable('a2', [])\n",
293 |     "# b = tf.get_variable('b2', [])\n",
294 |     "# saver = tf.train.Saver()\n",
295 |     "# c = tf.get_variable('c2', [])\n",
296 |     "# print(saver._var_list)"
297 |    ]
298 |   },
299 |   {
300 |    "cell_type": "markdown",
301 |    "metadata": {},
302 |    "source": [
303 |     "Since c wasn’t around at the time of our saver’s creation, it does not get to be a part of the fun. So in general, make sure that you already have all your variables created before creating a saver.\n",
304 |     "\n",
305 |     "Of course, there are also some specific circumstances where you may actually want to only save a subset of your variables! tf.train.Saver lets you pass the var_list when you create it to specify which subset of available variables you want it to keep track of."
306 |    ]
307 |   },
308 |   {
309 |    "cell_type": "code",
310 |    "execution_count": 9,
311 |    "metadata": {},
312 |    "outputs": [],
313 |    "source": [
314 |     "# a1 = tf.get_variable('a3', [])\n",
315 |     "# b1 = tf.get_variable('b3', [])\n",
316 |     "# c1 = tf.get_variable('c3', [])\n",
317 |     "# saver = tf.train.Saver(var_list=[a1,b1])\n",
318 |     "# print(saver._var_list)"
319 |    ]
320 |   },
321 |   {
322 |    "cell_type": "markdown",
323 |    "metadata": {},
324 |    "source": [
325 |     "### Loading Modified Models\n",
326 |     "The examples above cover the ‘perfect sphere in frictionless vacuum’ scenario of model-loading. As long as you are saving and loading your own models, using your own code, without changing things in between, saving and loading is a breeze. But in many cases, things are not so clean. And in those cases, we need to get a little fancier.\n",
327 |     "\n",
328 |     "Let’s take a look at a couple of scenarios to illustrate the issues. First, something that works without a problem. What if we want to save a whole model, but we only want to load part of it? (In the following code example, I run the two scripts in order.)"
329 |    ]
330 |   },
331 |   {
332 |    "cell_type": "code",
333 |    "execution_count": 10,
334 |    "metadata": {},
335 |    "outputs": [],
336 |    "source": [
337 |     "# a = tf.get_variable('a4', [])\n",
338 |     "# b = tf.get_variable('b4', [])\n",
339 |     "# init = tf.global_variables_initializer()\n",
340 |     "# saver = tf.train.Saver()\n",
341 |     "# sess = tf.Session()\n",
342 |     "# sess.run(init)\n",
343 |     "# saver.save(sess, './tftcp.model')\n",
344 |     "# sess.close()"
345 |    ]
346 |   },
347 |   {
348 |    "cell_type": "code",
349 |    "execution_count": 11,
350 |    "metadata": {},
351 |    "outputs": [],
352 |    "source": [
353 |     "# a = tf.get_variable('a4', [])\n",
354 |     "# init = tf.global_variables_initializer()\n",
355 |     "# saver = tf.train.Saver()\n",
356 |     "# sess = tf.Session()\n",
357 |     "# sess.run(init)\n",
358 |     "# saver.restore(sess, './tftcp.model')\n",
359 |     "# sess.run(a)\n",
360 |     "# sess.close()"
361 |    ]
362 |   },
363 |   {
364 |    "cell_type": "markdown",
365 |    "metadata": {},
366 |    "source": [
367 |     "Good, easy enough! And yet, a failure case emerges when we have the reverse scenario: we want to load one model as a component of a larger model."
368 |    ]
369 |   },
370 |   {
371 |    "cell_type": "code",
372 |    "execution_count": 12,
373 |    "metadata": {},
374 |    "outputs": [],
375 |    "source": [
376 |     "# a = tf.get_variable('a5', [])\n",
377 |     "# init = tf.global_variables_initializer()\n",
378 |     "# saver = tf.train.Saver()\n",
379 |     "# sess = tf.Session()\n",
380 |     "# sess.run(init)\n",
381 |     "# saver.save(sess, './tftcp.model')\n",
382 |     "# sess.close()"
383 |    ]
384 |   },
385 |   {
386 |    "cell_type": "code",
387 |    "execution_count": 13,
388 |    "metadata": {},
389 |    "outputs": [],
390 |    "source": [
391 |     "## Raise an exception if we run the following code\n",
392 |     "# a = tf.get_variable('a5', [])\n",
393 |     "# d = tf.get_variable('d5', [])\n",
394 |     "# init = tf.global_variables_initializer()\n",
395 |     "# saver = tf.train.Saver()\n",
396 |     "# sess = tf.Session()\n",
397 |     "# sess.run(init)\n",
398 |     "# saver.restore(sess, './tftcp.model')\n",
399 |     "# sess.close()"
400 |    ]
401 |   },
402 |   {
403 |    "cell_type": "markdown",
404 |    "metadata": {},
405 |    "source": [
406 |     "We just wanted to load a, while ignoring the new variable d. And yet, we got an error, complaining that d was not present in the checkpoint!\n",
407 |     "\n",
408 |     "A third scenario is where you want to load one model’s parameters into a different model’s computation graph. This throws an error too, for obvious reasons: Tensorflow cannot possibly know where to put all those parameters you just loaded. Luckily, there’s a way to give it a hint.\n",
409 |     "\n",
410 |     "Remember var_list from one section-header ago? Well, it turns out to be a bit of a misnomer. A better name might be “var_list_or_dictionary_mapping_names_to_vars”, but that’s a mouthful, so I can sort of see why they stuck with the first bit.\n",
411 |     "\n",
412 |     "Saving models is one of the key reasons that Tensorflow mandates globally-unique variable names. In a saved-model-file, each saved variable’s name is associated with its shape and value. Loading it into a new computational graph is as easy as mapping the original-names of the variables you want to load to variables in your current model. Here’s an example:"
413 |    ]
414 |   },
415 |   {
416 |    "cell_type": "code",
417 |    "execution_count": 14,
418 |    "metadata": {},
419 |    "outputs": [],
420 |    "source": [
421 |     "# a = tf.get_variable('a6', [])\n",
422 |     "# init = tf.global_variables_initializer()\n",
423 |     "# saver = tf.train.Saver()\n",
424 |     "# sess = tf.Session()\n",
425 |     "# sess.run(init)\n",
426 |     "# saver.save(sess, './tftcp.model')\n",
427 |     "# sess.close()"
428 |    ]
429 |   },
430 |   {
431 |    "cell_type": "code",
432 |    "execution_count": 15,
433 |    "metadata": {},
434 |    "outputs": [],
435 |    "source": [
436 |     "# d = tf.get_variable('d6', [])\n",
437 |     "# init = tf.global_variables_initializer()\n",
438 |     "# saver = tf.train.Saver(var_list={'a6':d})\n",
439 |     "# sess = tf.Session()\n",
440 |     "# sess.run(init)\n",
441 |     "# saver.restore(sess, './tftcp.model')\n",
442 |     "# sess.run(d)\n",
443 |     "# sess.close()"
444 |    ]
445 |   },
446 |   {
447 |    "cell_type": "markdown",
448 |    "metadata": {},
449 |    "source": [
450 |     "This is the key mechanism by which you can combine models that do not have the exact same computational graph. For example, perhaps you got a pre-trained language model off of the internet, and want to re-use the word embeddings. Or, perhaps you changed the parameterization of your model in between training runs, and you want this new version to pick up where the old one left off; you don’t want to have to re-train the whole thing from scratch. In both of these cases, you would simply need to hand-make a dictionary mapping from the old variable names to the new variables.\n",
451 |     "\n",
452 |     "A word of caution: it’s very important to know exactly how the parameters you are loading are meant to be used. If possible, you should use the exact code the original authors used to build their model, to ensure that that component of your computational graph is identical to how it looked during training. If you need to re-implement, keep in mind that basically any change, no matter how minor, is likely to severely damage the performance of your pre-trained net. Always benchmark your reimplementation against the original!"
453 |    ]
454 |   },
455 |   {
456 |    "cell_type": "markdown",
457 |    "metadata": {},
458 |    "source": [
459 |     "### Inspecting Models\n",
460 |     "\n",
461 |     "If the model you want to load came from the internet - or from yourself, >2 months ago - there’s a good chance you won’t know how the original variables were named. To inspect saved models, use [these tools](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/framework/python/framework/checkpoint_utils.py), which come from the official Tensorflow repository. For example:"
462 |    ]
463 |   },
464 |   {
465 |    "cell_type": "code",
466 |    "execution_count": 16,
467 |    "metadata": {},
468 |    "outputs": [],
469 |    "source": [
470 |     "# a = tf.get_variable('a7', [])\n",
471 |     "# b = tf.get_variable('b7', [10,20])\n",
472 |     "# c = tf.get_variable('c7', [])\n",
473 |     "# init = tf.global_variables_initializer()\n",
474 |     "# saver = tf.train.Saver()\n",
475 |     "# sess = tf.Session()\n",
476 |     "# sess.run(init)\n",
477 |     "# saver.save(sess, './tftcp.model')\n",
478 |     "# print(tf.contrib.framework.list_variables('./tftcp.model'))\n",
479 |     "# sess.close()"
480 |    ]
481 |   },
482 |   {
483 |    "cell_type": "markdown",
484 |    "metadata": {},
485 |    "source": [
486 |     "With a little effort and a lot of head-scratching, it’s usually possible to use these tools (in conjunction with the original codebase) to find the names of the variables you want."
487 |    ]
488 |   }
489 |  ],
490 |  "metadata": {
491 |   "kernelspec": {
492 |    "display_name": "Python 3",
493 |    "language": "python",
494 |    "name": "python3"
495 |   },
496 |   "language_info": {
497 |    "codemirror_mode": {
498 |     "name": "ipython",
499 |     "version": 3
500 |    },
501 |    "file_extension": ".py",
502 |    "mimetype": "text/x-python",
503 |    "name": "python",
504 |    "nbconvert_exporter": "python",
505 |    "pygments_lexer": "ipython3",
506 |    "version": "3.6.8"
507 |   }
508 |  },
509 |  "nbformat": 4,
510 |  "nbformat_minor": 2
511 | }
512 | 


--------------------------------------------------------------------------------
/Optimization/Readme.md:
--------------------------------------------------------------------------------
 1 | # Optimization
 2 | 
 3 | ## [Hyperparameter (Machine Learning)](https://en.wikipedia.org/wiki/Hyperparameter_(machine_learning))   
 4 | 
 5 | A model parameter is a variable whose value is estimated from the dataset. Parameters are the values learned during training from the historical data sets like weights and biases. A hyperparameter is a configuration variable that is external to the model. It is defined manually before the training of the model with the historical dataset. Its value cannot be evaluated from the datasets. For example, the number of hidden nodes and layers,input features, learning rate, activation function etc in neural network.  
 6 | 
 7 | In machine learning, the value of a hyperparameter is used to control the learning process. By contrast, the values of parameters (typically edge weights) 
 8 | are derived via training. 
 9 | 
10 | ## [Optimizing Hyperparameters](https://deepai.org/machine-learning-glossary-and-terms/hyperparameter)  
11 | Hyperparameters can have a direct impact on the training of machine learning algorithms. Thus, in order to achieve maximal performance, 
12 | it is important to understand how to optimize them. Here are some common strategies for optimizing hyperparameters:
13 | 
14 | - Grid Search: Search a set of manually predefined hyperparameters for the best performing hyperparameter. Use that value. (This is the traditional method)   
15 | 
16 | - Random Search: Similar to grid search, but replaces the exhaustive search with random search. This can outperform grid search when only a 
17 | small number of hyperparameters are needed to actually optimize the algorithm.   
18 | 
19 | - Bayesian Optimization: Builds a probabilistic model of the function mapping from hyperparameter values to the target evaluated on a validation set.   
20 | 
21 | - Gradient-Based Optimization: Compute gradient using hyperparameters and then optimize hyperparameters using gradient descent.   
22 | 
23 | - Evolutionary Optimization: Uses evolutionary algorithms (e.g. genetic functions) to search the space of possible hyperparameters.   
24 | 
25 | ## [A Novice's Guide to Hyperparameter Optimization at Scale](https://wood-b.github.io/post/a-novices-guide-to-hyperparameter-optimization-at-scale/)  
26 | by Brandon M. Wood   
27 | 
28 | There are two main types of hyperparameters in machine learning and they dictate what HPO strategies are possible.
29 | 
30 | - Model (Structural) Hyperparameters: Establish model architecture    
31 | **Structural hyperparameters** refer to the model selection task and they cannot be inferred while fitting the machine to the training 
32 | set. For example: Number of convolutional layers, Number of fully connected layers, etc.
33 | 
34 | - Algorithm Hyperparameters: Are involved in the learning process   
35 | **Algorithm hyperparameters**, that in principle have no influence on the performance of the model but affect the speed and quality of the learning process.  Examples of algorithm hyperparameters are learning rate and mini-batch size.  
36 | 
37 | **Not all Hyperparameters Can Be Treated the Same:** 
38 | The important takeaway is that not all HPO strategies can handle both structural and algorithm hyperparameters. Population basd training (PBT) is a good example. PBT was designed to evolve and inherit hyperparameters from other high performing workers in the population; however, if workers have different network architectures it is unclear how exactly that would work. There might be a way to do this with PBT, but it is not standard and does not work out-of-the-box with Ray Tune.   
39 | 


--------------------------------------------------------------------------------
/Projects/README.md:
--------------------------------------------------------------------------------
 1 | # Course Project Resources:
 2 | To bring together and apply the various topics covered in this course, you will work on a machine learning project. The goal of the project is to go through the complete knowledge discovery process to answer one or more questions you have about a topic of your own choosing. You will acquire the data, formulate a question (or questions) of interest, perform the data analysis, and communicate the results. Projects are programming assignments that cover the topic of this course. Any project is written by **[Jupyter Notebook](http://jupyter.org)**. The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more. Furthermore, we can include **mathematical notation** within markdown cells of Jupyter Notebook using **LaTeX**.  
 3 | 
 4 | ## َUploading and Presentation of Project 
 5 | ### Uploading Project in GitHub
 6 | Students individually implement given assignments in recommended language. Each Student should create his/her own GitHub repository for his/her project. The repository should comprise a Readme.md file containing the analysis of the result. It should include a short write-up including the following components:
 7 | 
 8 | - Problem statement and hypothesis
 9 | - Description of your data set and how it was obtained
10 | - Description of any pre-processing steps you took
11 | - What you learned from exploring the data, including visualizations
12 | - How you chose which features to use in your analysis
13 | - Details of your modeling process, including how you selected your models and validated them
14 | - Your challenges and successes
15 | - Possible extensions or business applications of your project
16 | - Conclusions and key learnings
17 | 
18 | ### Final Project  
19 | Your project repository on GitHub should contain the following:
20 | 
21 | - Project paper: Any format (PDF, Markdown, etc.)
22 | - Presentation slides: Any format (PDF, PowerPoint, Google Slides, IPython Notebook, etc.)
23 | - Code: Commented Jupyter Notebooks, and any other code you used in the project
24 | - Visualizations: Integrated into your paper and/or slides
25 | - Data: Data files in "raw" or "processed" format
26 | - Data dictionary (aka "code book"): Description of each variable, including units
27 | 
28 | ### Project Presentation in the Class
29 | Each student will explain his/her project in a 10–15 minute presentation to the class. Presentations should clearly convey the project ideas, methods, and results, including the question(s) being addressed, the motivation of the analyses being employed, and relevant evaluations, contributions, and discussion questions.
30 | 
31 | ## Coding:
32 | Programming assignments will require the use of Python 3.7 and Tensorflow, as well as additional Python packages as follows. <br> 
33 | * [Python 3.7:](https://www.python.org/downloads/) An interactive, object-oriented, extensible programming language. 
34 | * [TensorFlow:](https://www.tensorflow.org/beta/guide/effective_tf2) TensorFlow is an end-to-end open source platform for machine learning. 
35 | * [NumPy:](http://www.numpy.org) A Python package for scientific computing.
36 | * [Pandas:](https://pandas.pydata.org) A Python package for high-performance, easy-to-use data structures and data analysis tools.
37 | * [Scikit-Learn:](https://scikit-learn.org/stable/) A Python package for machine learning.
38 | * [Matplotlib:](https://matplotlib.org) A Python package for 2D plotting.
39 | * [SciPy:](https://www.scipy.org) A Python package for mathematics, science, and engineering.
40 | * [IPython:](https://ipython.org) An architecture for interactive computing with Python.
41 | 
42 | Most of the relevant software is a part of the [SciPy stack](https://www.scipy.org), a collection of Python-based open source software for mathematics, science, and engineering (which includes Python, NumPy, the SciPy library, Matplotlib, pandas, IPython, and scikit-learn). The [Anaconda](https://www.anaconda.com) Python Distribution is a free distribution for the SciPy stack that supports Linux, Mac, and Windows. With over 6 million users, the open source Anaconda Distribution is the fastest and easiest way to do Python and R data science and machine learning on Linux, Windows, and Mac OS X. It's the industry standard for developing, testing, and training on a single machine.<br>
43 | 
44 | * [Getting started with conda](https://conda.io/docs/user-guide/getting-started.html)
45 | * [Instalation](https://docs.anaconda.com/anaconda/install/)
46 | 
47 | Tutorial: <br>
48 | * [NumPy Tutorial](http://scipy.github.io/old-wiki/pages/Tentative_NumPy_Tutorial) <br>
49 | You can learn Python via the following websites: <br>
50 | * [SoloLearn](http://www.sololearn.com/) (A great website for getting started with coding. It offers easy to follow lessons, interspersed with quizzes to help you retain what you are learning).
51 | * [Google Developer Python Tutorial](https://developers.google.com/edu/python/)  (highly recommended as a way to master python in just a few hours!) 
52 | 
53 | ### Latex
54 | The students can include mathematical notation within markdown cells using LaTeX in their **[Jupyter Notebooks](http://jupyter.org)**.<br>
55 | A Brief Introduction to LaTeX [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/latex.pdf)  <br>
56 | Math in LaTeX [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/math.pdf) <br>
57 | Sample Document [PDF](https://www.seas.upenn.edu/~cis519/spring2018/assets/resources/latex/sample.pdf) <br>
58 | 
59 | Competitions
60 | 
61 | ## Competitions:
62 | Here are some machine learning and data mining competition platforms:
63 | - [Kaggle](https://www.kaggle.com/)
64 | - [DrivenData](https://www.drivendata.org/)
65 | - [Analytics Vidhya](http://datahack.analyticsvidhya.com/)
66 | - [The Data Science Game](http://www.datasciencegame.com/)
67 | - [InnoCentive](https://www.innocentive.com/)
68 | - [TuneedIT](http://tunedit.org/challenges)
69 | 


--------------------------------------------------------------------------------
/PyTorch/README.MD:
--------------------------------------------------------------------------------
 1 | # PyTorch  
 2 | 
 3 | ## Learn Pytorch Basics
 4 |  
 5 |  - Blog: [Torch Tensors & Types:](https://pytorch.org/docs/stable/tensors.html) A torch.Tensor is a multi-dimensional 
 6 | matrix containing elements of a single data type. Torch defines nine CPU tensor types and nine GPU tensor types. 
 7 | - Blog: [PyTorch - Basic Operations](https://jhui.github.io/2018/02/09/PyTorch-Basic-operations/) by Jonathan Hui   
 8 | - Blog: [Autograd: Automatic Differntiation](https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html) 
 9 | - Blog: [Creating PyTorch Tensors - Best Options](https://deeplizard.com/learn/video/AglLTlms7HU)    
10 | - Blog: [Compare Tensor Copy Operations Such as clone, detach, and copy_](https://www.programmersought.com/article/78494973057/)   
11 | - Blog: [What is PyTorch .detach() method?](https://dev.to/theroyakash/what-is-pytorch-detach-method-15oo) by Theroyakash 
12 | - Blog: [Learning PyTorch with Exampls](https://pytorch.org/tutorials/beginner/pytorch_with_examples.html) by Justin Johnson. 
13 | This tutorial introduces the fundamental concepts of PyTorch through self-contained examples.  
14 | - Blog: [Why PyTorch is the Deep Learning Framework of the Future](https://blog.paperspace.com/why-use-pytorch-deep-learning-framework/) by Dhiraj Kumar  
15 | 
16 | ## Preprocessing Datasets  
17 | 
18 | - Blog: [How to Build a Streaming DataLoader with PyTorch](https://medium.com/speechmatics/how-to-build-a-streaming-dataloader-with-pytorch-a66dd891d9dd) by David MacLeod   
19 | - Blog: [Building Efficient Custom Datasets in PyTorch](https://towardsdatascience.com/building-efficient-custom-datasets-in-pytorch-2563b946fd9f) by Syafiq Kamarul Azman    
20 | 
21 | 
22 | ## Neural Network
23 | 
24 | - Deep Learning via Pytorch by Ayoosh Kathuria  
25 |     - [PyTorch 101, Part 1: Understanding Graphs, Automatic Differentiation and Autograd](https://blog.paperspace.com/pytorch-101-understanding-graphs-and-automatic-differentiation/)  
26 |     - [PyTorch 101, Part 2: Building Your First Neural Network](https://blog.paperspace.com/pytorch-101-building-neural-networks/)  
27 |     - [PyTorch 101, Part 3: Going Deep with PyTorch](https://blog.paperspace.com/pytorch-101-advanced/)  
28 |     - [PyTorch 101, Part 4: Memory Management and Using Multiple GPUs](https://blog.paperspace.com/pytorch-memory-multi-gpu-debugging/)  
29 |     - [PyTorch 101, Part 5: Understanding Hooks](https://blog.paperspace.com/pytorch-hooks-gradient-clipping-debugging/)
30 |     
31 |  - Blog: [Building Efficient Custom Datasets in PyTorch](https://towardsdatascience.com/building-efficient-custom-datasets-in-pytorch-2563b946fd9f) by Syafiq Kamarul Azman   
32 |  - Blog: [Keras vs. PyTorch: Alien vs. Predator Recognition with Transfer Learning](https://deepsense.ai/keras-vs-pytorch-avp-transfer-learning/) by Piotr Migdal, Patryk Miziuła and Rafał Jakubanis  
33 |  - Blog: [Writing Custum Datasets, Dataloaders and Transforms](https://pytorch.org/tutorials/beginner/data_loading_tutorial.html) by Sasank Chilamkurthy    
34 |  - Blog: [PyTorch Tutorial: How to Develop Deep Learning Models with Python](https://machinelearningmastery.com/pytorch-tutorial-develop-deep-learning-models/) by Jason Brownlee    
35 |  - Blog: [Saving/Loading Your Model in PyTorch](https://medium.com/udacity-pytorch-challengers/saving-loading-your-model-in-pytorch-741b80daf3c) by David Ashraf  
36 |  - Blog: [Saving and Loading Your Model to Resume Training in PyTorch](https://medium.com/analytics-vidhya/saving-and-loading-your-model-to-resume-training-in-pytorch-cb687352fa61) by Rachit Jain   
37 |  - Blog: [Frequently Asked Questions (About the Problem of Out of Memory)](https://pytorch.org/docs/stable/notes/faq.html) 
38 |  
39 | ## Pytorch Lightning & PyTorch Ignite  
40 | 
41 | PyTorch-lightning is a recently released library which is a Kera-like ML library for PyTorch. Ignite is a high-level library to help with training and evaluating neural networks in PyTorch flexibly and transparently.  
42 | 
43 | - Blog: [Pytorch Lightning vs PyTorch Ignite vs Fast.ai](https://towardsdatascience.com/pytorch-lightning-vs-pytorch-ignite-vs-fast-ai-61dc7480ad8a) by William Falcon  
44 | - Blog: [Pytorch Lightning Documentation](https://pytorch-lightning.readthedocs.io/en/latest/)   
45 | - Blog: [Lightning Blog](https://www.pytorchlightning.ai/blog)
46 | - Blog: [9 Tips For Training Lightning-Fast Neural Networks In Pytorch](http://taylorliu.com/2019-08-17-pytorch-tips/#pytorch-lightning) by William Falcon   
47 | - Blog: [From PyTorch to PyTorch Lightning — A Gentle Introduction](https://towardsdatascience.com/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565)  by William Falcon   
48 | - Blog: [Getting Started with PyTorch Lightning](https://www.learnopencv.com/getting-started-with-pytorch-lightning/) by Neelabh Madan 
49 | - Colab: [From PyTorch to PyTorch Lightning - An Introduction](https://colab.research.google.com/drive/1Mowb4NzWlRCxzAFjOIJqUmmk_wAT-XP3#scrollTo=x83-rnVKT8Wo) by 
50 | - Blog: [A Guide to Ignite: An Introduction to Ignite Training Loops](https://svenbecker.github.io/programming/deep%20learning/python/ignite/) by Sven Becker  
51 | - Blog: [Debugging](https://pytorch-lightning.readthedocs.io/en/latest/debugging.html) via Pytorch Lightning  
52 | - Blog: [PyTorch Lightning: Metrics](https://medium.com/pytorch/pytorch-lightning-metrics-35cb5ab31857) by Justus Schock  
53 | - Blog: [7 Tips To Maximize PyTorch Performance](https://towardsdatascience.com/7-tips-for-squeezing-maximum-performance-from-pytorch-ca4a40951259) by William Falcon    
54 | 
55 | 
56 | ## Videos
57 | 
58 | - Blog: [Learn with Lightning](https://www.pytorchlightning.ai/tutorials)
59 | 
60 | 
61 | ## Books
62 | 
63 | - Blog: [Dive into Deep Learning](https://d2l.ai) by  Mag Gardner, Max Drummy, Joanne Quinn, Joanne McEachen, Michael Fullan   
64 |     - GitHub: [Codes](https://github.com/dsgiitr/d2l-pytorch)   
65 | - Blog: [Deep Learning with PyTorch](https://pytorch.org/deep-learning-with-pytorch) by Eli Stevens and Luca Antiga  
66 |     - GitHub: [Codes](https://github.com/deep-learning-with-pytorch/dlwpt-code)   
67 | 
68 | 
69 | ## Courses
70 | 
71 | - Blog: [Mini Course in Deep Learning with PyTorch for AIMS](https://github.com/Atcold/pytorch-Deep-Learning-Minicourse) 
72 | by Alfredo Canziani   
73 | - Blog: [Introduction to Pytorch Code Examples](https://cs230.stanford.edu/blog/pytorch/) by Andrew Ng and Kian Katanforoosh     
74 | 
75 | 
76 | 


--------------------------------------------------------------------------------
/Recitation-Assignments/Assignment_Set_1_Sample.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |   "nbformat": 4,
  3 |   "nbformat_minor": 0,
  4 |   "metadata": {
  5 |     "kernelspec": {
  6 |       "display_name": "Python 3",
  7 |       "language": "python",
  8 |       "name": "python3"
  9 |     },
 10 |     "language_info": {
 11 |       "codemirror_mode": {
 12 |         "name": "ipython",
 13 |         "version": 3
 14 |       },
 15 |       "file_extension": ".py",
 16 |       "mimetype": "text/x-python",
 17 |       "name": "python",
 18 |       "nbconvert_exporter": "python",
 19 |       "pygments_lexer": "ipython3",
 20 |       "version": "3.7.6"
 21 |     },
 22 |     "colab": {
 23 |       "name": "Assignment-Set-1_Sample.ipynb",
 24 |       "provenance": []
 25 |     }
 26 |   },
 27 |   "cells": [
 28 |     {
 29 |       "cell_type": "markdown",
 30 |       "metadata": {
 31 |         "id": "FOnVyhXQxVt7",
 32 |         "colab_type": "text"
 33 |       },
 34 |       "source": [
 35 |         "Assignment Set 1 by [Your_Name] for [Deep Learing/Applied Machine Learning] Course at Data Science Center, SBU "
 36 |       ]
 37 |     },
 38 |     {
 39 |       "cell_type": "code",
 40 |       "metadata": {
 41 |         "id": "F9BcydNyxVuA",
 42 |         "colab_type": "code",
 43 |         "colab": {}
 44 |       },
 45 |       "source": [
 46 |         "import time\n",
 47 |         "import numpy as np\n",
 48 |         "import pandas as pd\n",
 49 |         "import matplotlib.pyplot as plt\n",
 50 |         "import random\n",
 51 |         "import sklearn\n",
 52 |         "import tensorflow as tf\n",
 53 |         "import torch "
 54 |       ],
 55 |       "execution_count": 48,
 56 |       "outputs": []
 57 |     },
 58 |     {
 59 |       "cell_type": "code",
 60 |       "metadata": {
 61 |         "id": "KVP_jvJ2xnpr",
 62 |         "colab_type": "code",
 63 |         "colab": {}
 64 |       },
 65 |       "source": [
 66 |         "print(\"Scikit-learn Version: \", sklearn.__version__)\n",
 67 |         "print(\"TensorFlow Version: \", tf.__version__)\n",
 68 |         "print(\"PyTorch Version: \", torch.__version__)"
 69 |       ],
 70 |       "execution_count": null,
 71 |       "outputs": []
 72 |     },
 73 |     {
 74 |       "cell_type": "code",
 75 |       "metadata": {
 76 |         "id": "XAuFwyIuxVuT",
 77 |         "colab_type": "code",
 78 |         "colab": {}
 79 |       },
 80 |       "source": [
 81 |         "start_time = time.time()\n",
 82 |         "data = {'a': np.arange(500),\n",
 83 |         "        'c': np.random.randint(0, 500, 500),\n",
 84 |         "        'd': np.random.randn(500)}\n",
 85 |         "data['b'] = data['a'] + 5 * np.random.randn(500)\n",
 86 |         "data['d'] = np.abs(data['d']) * 100\n",
 87 |         "plt.scatter('a', 'b', c='c', s='d', data=data)\n",
 88 |         "plt.xlabel('A')\n",
 89 |         "plt.ylabel('B')\n",
 90 |         "plt.show()\n",
 91 |         "df = pd.DataFrame(np.random.randn(10, 5)) \n",
 92 |         "print(df)\n",
 93 |         "j=np.pi\n",
 94 |         "for i in range(100000000):\n",
 95 |         "  j+=j    \n",
 96 |         "print(\"\\n-> Execution Time = %s in Seconds <-\" % (time.time() - start_time))\n"
 97 |       ],
 98 |       "execution_count": null,
 99 |       "outputs": []
100 |     }
101 |   ]
102 | }


--------------------------------------------------------------------------------
/Recitation-Assignments/README.md:
--------------------------------------------------------------------------------
  1 | # Teaching Assistants
  2 | 
  3 | 
  4 | * (Head) [Yavar Yeganeh](https://github.com/YavarYeganeh) | [Email](mailto:yavaryeganeh@gmail.com)
  5 |   - Office Hours: Sundays and Wednesdays, 12 am to 1 pm, Professor's Office (TBC)  
  6 | 
  7 | * [Behnaz Hoseyni](https://github.com/behnazhoseyni) | [Email](mailto:hoseyni.sb@gmail.com)
  8 | <!--
  9 | * [Erfaan Rostami](https://github.com/Erfaan-Rostami) | [Email](mailto:r.a.erfan@gmail.com)
 10 | -->
 11 | * [Mostafa Khodayari](https://github.com/MSTF4) | [Email](mailto:mo.khodayari@mail.sbu.ac.ir)
 12 | 
 13 | * [Esmail Mafakheri](https://github.com/E008001) | [Email](mailto:e.mafakheri@mail.sbu.ac.ir)   
 14 | 
 15 | 
 16 | **Please Note**: 
 17 | 
 18 | * A carbon copy (Cc) of your email communications with TAs must be sent to this [email address](mailto:hhhaji@yahoo.com).
 19 | * Response to emails may take a few days. Please be patient!
 20 | * You can use [Piazza](https://piazza.com/sbu.ac.ir/fall2020/dl_dsc_f20) for questions and discussions on the related topics.
 21 | * Recitation classes and seminars will take place in the [Skype Group](https://join.skype.com/kJ6WepEDrsnt).
 22 | * Assignments and the projects should be done utilizing [PyTorch](https://pytorch.org/), except there have been specific instructions for a task.
 23 | 
 24 | # Recitation
 25 | 
 26 | * **Session One** by [Mostafa Khodayari](https://github.com/MSTF4) was on Wednesday, Shahrivar 26 at 16:00
 27 |    - [Toolkit Lab: Getting Started with PyTorch](https://github.com/hhaji/Deep-Learning#Part-2) (Part one) 
 28 |    - Notebook: [Colab](https://colab.research.google.com/drive/1qOAGNcdXuLOuJc9Hq9vQP6Cmlq-PJnEN?usp=sharing)
 29 |    
 30 |  * **Session Two** by [Mostafa Khodayari](https://github.com/MSTF4) was on Tuesday, Mehr 1 at 16:00
 31 |    - [Toolkit Lab: Getting Started with PyTorch](https://github.com/hhaji/Deep-Learning#Part-2) (Part two) 
 32 |    - Notebook: [Colab](https://colab.research.google.com/drive/1d7gKanEvRhb-6RD6jtkhyIMfAySEz1Yt?usp=sharing)
 33 | 
 34 |  * **Session Three** by [Behnaz Hoseyni](https://github.com/behnazhoseyni) was on Wednesday, Mehr 9 at 16:00
 35 |    - [Preprocessing Datasets by PyTorch](https://github.com/hhaji/Deep-Learning#Part-3) 
 36 |    - Notebook: [Colab](https://colab.research.google.com/drive/1WDbyiac97XQvk246OzHE22J8veD0WsUQ?usp=sharing)
 37 |    - Video: Posted in the Skype group (for one month)
 38 |    
 39 |  * **Session Four** by [Behnaz Hoseyni](https://github.com/behnazhoseyni) was on Tuesday, Mehr 15 at 16:00
 40 |    - [Deep Feedforward Networks](https://github.com/hhaji/Deep-Learning#DFN) 
 41 |    - Notebook: [Colab](https://colab.research.google.com/drive/15aMn8jQ2RYoad_GSIFdCBxkW4qsVAC72?usp=sharing)
 42 |    - Video: Posted in the Skype group (for one month)
 43 |    
 44 |  * **Session Five** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was on Wednesday, Mehr 23 at 16:00
 45 |    - [Using a Neural Network to Fit the Data with PyTorch](https://github.com/hhaji/Deep-Learning#Part-4)
 46 |    - Notebook: [Colab](https://colab.research.google.com/drive/1kJsHpvaPYAS522Ay1DAnnejMY-dSNhxZ?usp=sharing)
 47 |    - Video: Posted in the Skype group (for one month)
 48 |  
 49 |  * **Session Six** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was on Tuesday, Mehr 29 at 18:00
 50 |    - [Regularization for Deep Learning](https://github.com/hhaji/Deep-Learning#RFDL) (Part one)
 51 |    - Notebook: [Colab](https://colab.research.google.com/drive/1jzfsfwkAVsnd0-wO_Xtfx28O3oUhWyLk?usp=sharing%22)  
 52 |    - Video: Posted in the Skype group (for one month)
 53 |    
 54 |  * **Video Tutorial One** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was posted on Wednesday, Mehr 30
 55 |    - Introduction to Python Classes and Objects
 56 |    - Building fully connected neural networks without using machine learning libraries in Python
 57 |    - Notebook: [Colab](https://colab.research.google.com/drive/1MswS0QON4atGHxm85oXqCCKR2fypo7Su?usp=sharing) 
 58 |    - Video: Posted in the Skype group (for one month)
 59 |    
 60 |  * **Session Seven** by [Esmail Mafakheri](https://github.com/E008001) was on Wednesday, Aban 7 at 16:00
 61 |    - [Convolutional Networks](https://github.com/hhaji/Deep-Learning#CNN)
 62 |    - Notebook: [Colab](https://colab.research.google.com/drive/1H7L_ni6fupxMpntLToASEi3FWjjWzfA_?usp=sharing)
 63 |    - Video: Posted in the Skype group (for one month)
 64 | 
 65 |  * **Session Eight** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was on Tuesday, Aban 20 at 18:00 
 66 |    - [Regularization for Deep Learning](https://github.com/hhaji/Deep-Learning#RFDL) (Part two)
 67 |    - [Optimization for Training Deep Models](https://github.com/hhaji/Deep-Learning#OFTDM)
 68 |    - Notebook: [Colab](https://colab.research.google.com/drive/1cass1LqzpmTs1dnasSzgHnvuSnHdODZQ?usp=sharing)
 69 |    - Video: Posted in the Skype group (for one month)
 70 |   
 71 |  * **Session Nine** by [Esmail Mafakheri](https://github.com/E008001) was on Wednesday, Aban 28 at 16:00
 72 |    - [Using Convolutions to Generalize](https://github.com/hhaji/Deep-Learning#Part-5)
 73 |    - Notebook: [Colab](https://colab.research.google.com/drive/1I6eNUR6whEct9aaI62zmPeda8A2ttEsb?usp=sharing%22)
 74 |    - Video: Posted in the Skype group (for one month)  
 75 |    
 76 |   * **Session Ten** by [Behnaz Hoseyni](https://github.com/behnazhoseyni) was on Tuesday, Azar 4 at 16:00
 77 |     - [Optuna: Automatic Hyperparameter Optimization Software](https://github.com/hhaji/Deep-Learning#Part-7) 
 78 |     - Notebook: [Colab](https://colab.research.google.com/drive/1Fn3iLMOTMhPz8R3g17wAS-iDVhTk7kGC?usp=sharing)
 79 |     - Video: Posted in the Skype group (for one month) 
 80 |    
 81 |  * **Session Eleven** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was on Tuesday, Azar 18 at 17:00
 82 |    - [Sequence Modeling: Recurrent and Recursive Networks](https://github.com/hhaji/Deep-Learning#SMRARN) (Part one)
 83 |    - Notebook: [Colab](https://colab.research.google.com/drive/10UJlzZ6A0bcziqeVeC6BZKEF6e23W7BI?usp=sharing) 
 84 |    - Video: Posted in the Skype group (for one month)
 85 | 
 86 |  * **Session Twelve** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was on Wednesday, Azar 19 at 17:00
 87 |    - [Toolkit Lab 6: Transfer Learning and Other Tricks](https://github.com/hhaji/Deep-Learning#Part-6)
 88 |    - Notebook: [Colab](https://colab.research.google.com/drive/1xm90WDOSxWzAu5F6m9Cf1IYg-xxI2S54?usp=sharing%22)
 89 |    - Video: Posted in the Skype group (for one month)
 90 |    
 91 |   * **Video Tutorial Two** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was posted on Thursday, Azar 20
 92 |     - [Sequence Modeling: Recurrent and Recursive Networks](https://github.com/hhaji/Deep-Learning#SMRARN) (Part two)
 93 |     - Notebook: [Colab](https://colab.research.google.com/drive/10UJlzZ6A0bcziqeVeC6BZKEF6e23W7BI?usp=sharing) 
 94 |     - Video: Posted in the Skype group (for one month)
 95 |  
 96 |  * **Session Thirteen** by [Mostafa Khodayari](https://github.com/MSTF4) was on Tuesday, Azar 25 at 16:00
 97 |    - [Practical Methodology](https://github.com/hhaji/Deep-Learning#Practical-Methodology)
 98 |    - Video: Posted in the Skype group (for one month)  
 99 | 
100 | * **Session Fourteen** by [Esmail Mafakheri](https://github.com/E008001) was on Tuesday, Dey 9 at 16:00
101 |    - [Autoencoders](https://github.com/hhaji/Deep-Learning#Autoencoders) 
102 |    - Video: Posted in the Skype group (for one month)  
103 |    
104 | * **Video Tutorial Three** by [Behnaz Hoseyni](https://github.com/behnazhoseyni) was posted on Wednesday, Dey 10
105 |    - [Applications](https://github.com/hhaji/Deep-Learning#Applications) 
106 |    - Video: [Link](https://drive.google.com/drive/folders/113GY9ux5wMSVfPHYpos_qP92DOkNePnZ?usp=sharing)
107 |    
108 | * **Session Fifteen** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was planned for Tuesday, Dey 16 at 16:00
109 |    - [Generative Adversarial Networks](https://github.com/hhaji/Deep-Learning#GAN)  
110 |       
111 | * **Session Sixteen** by [Yavar Yeganeh](https://github.com/YavarYeganeh) was planned for Tuesday, Dey 23 at 16:00
112 |    - [Graph Neural Networks](https://github.com/hhaji/Deep-Learning#GNN)  
113 |    
114 | 
115 | # Assignments 
116 | 
117 | ## Assignment Set 1  
118 | 
119 | * Install Anaconda and Create an Environment.    
120 | * Install Python Packages (Pandas, Scikit-Learn, Matplotlib) and Jupyter Lab in your new Environment.   
121 | * Install Machine Learning Libraries (PyTorch and TensorFlow).
122 | * Create a GitHub account.   
123 | * Read the [Submission Instruction](https://github.com/hhaji/Deep-Learning/tree/master/Recitation-Assignments#submission-instruction) and Register.
124 | * Use Markdown to Prepare a Readme file for your Repositories. 
125 | * Modify (at least enter your name) and Run the [Sample Notebook](https://github.com/hhaji/Deep-Learning/blob/master/Recitation-Assignments/Assignment_Set_1_Sample.ipynb) in your Environment, then Upload it in your Assignments' Repository.
126 | 
127 | <-> Deadline: Saturday, Shahrivar 29, 23:59 (Announced at Shahrivar 22)
128 | 
129 | ## Assignment Set 2
130 | 
131 | * Train neural network models (at least two different networks for each dataset, i.e., no. layers, no. neurons, activation, ...) in either Pytorch or Tensorflow to perform classification on the following datasets:
132 |   
133 |    - [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist)   
134 |    - [Cifar-10](https://www.cs.toronto.edu/~kriz/cifar.html) and [Cifar-100](https://www.cs.toronto.edu/~kriz/cifar.html)
135 |    - One different dataset, up to your choice (Optional ~ advantage 10 points: Two different datasets, up to your choice) 
136 |    
137 | * Train neural network models (at least two different networks for each dataset, i.e., no. layers, no. neurons, activations, ...) in either Pytorch or Tensorflow to perform regression on the following datasets:
138 | 
139 |    - [California Housing Values](https://github.com/ageron/handson-ml/tree/master/datasets/housing)
140 |    - One different dataset, up to your choice (Optional ~ advantage 10 points: Two different datasets, up to your choice) 
141 |    
142 | * Build and train a simple neural network in either Python or C++ (i.e., not utilizing machine learning libraries). It should be capable of having several layers and neurons as well as other hyperparameters (e.g., activations, loss function). Object-oriented (class/objects) programming should also be employed. Then compare your models with exactly the same settings at Pytorch and Tensorflow (and Keras) for the following data sets:
143 |    
144 |    - Samples from a high-degree perturbed polynomial 
145 |    - [California Housing Values](https://github.com/ageron/handson-ml/tree/master/datasets/housing)
146 | 
147 | **Please Note**: Datasets must be downloaded and injected manually (i.e., not loading them by libraries). Moreover, you can find many datasets, for instance, on Kaggle. Besides, Try to develop very good classifiers and regressors based on each model. Careless model architectures and hyperparameter selections, which result in poor performance, will not be appreciated and may be scored very low! 
148 | 
149 | <-> Deadline: Tuesday, Aban 6, 23:59 (Announced at Shahrivar 22)
150 | 
151 | ## Assignment Set 3
152 | 
153 | * Create three random tensors, including a 3 * 3 tensor as float, a 4 * 4 tensor as double, and a 5 * 3 * 4 tensor as short type
154 | * Change their elements in the GPU
155 | * Display the order of its elements and its transpose in the memory then use storage offset and contiguous
156 | * Save and load one among the elements of tensors using the torch and the h5py file
157 | * Show the different true and false requires_grad values in Autograd by the backward method
158 | 
159 | <-> Deadline: Thursday, Mehr 10, 23:59 (Announced at Shahrivar 27)
160 | 
161 | ## Assignment Set 4
162 | 
163 | * Download the [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) by Torchvision and split the data into two parts (training and test with a ratio of 70/30)
164 | * Perform tensor conversion, resizing, and normalization of the dataset, then use Dataloader
165 | * Use the two methods, "nn.Sequential" and "nn.Module", to create a 6-layer fully connected model with hidden layers having Sigmoid, Softmax, and Relu activation functions
166 | * Train (in GPU) and evaluate the models
167 | 
168 | <-> Deadline: Saturday, Mehr 19, 23:59 (Announced at Mehr 5)
169 | 
170 | ## Assignment Set 5
171 | 
172 | * Read these [images](https://drive.google.com/drive/folders/1N7vFHFJVmmQadbyPUVW1LsOVgsQUNAdA?usp=sharing) and collate them in one tensor
173 | * Prepare this [data](https://drive.google.com/drive/folders/1N3LD66ysr-msvhoEduVAw5E4emJh20za?usp=sharing) with a dataloader for neural networks
174 | 
175 | <-> Deadline: Tuesday, Mehr 22, 23:59 (Announced at Mehr 8)
176 | 
177 | ## Assignment Set 6
178 | 
179 | * Apply regularization techniques, including *L<sub>2</sub>* & *L<sub>1</sub>* penalties and data augmentation, to the regression and classification models (the first two parts) that you have developed in [Assignment Set 2](https://github.com/hhaji/Deep-Learning/tree/master/Recitation-Assignments#assignment-set-2), then compare them. 
180 | * Write at least a paragraph regarding (the conclusion of) your experiments and regularization effects. 
181 | 
182 | **Please Note**: Try to develop very good classifiers and regressors based on each model. Careless model architectures and hyperparameter selections, which result in poor performance, will not be appreciated and may be scored very low! 
183 | 
184 | <-> Deadline: Wednesday, Aban 21, 23:59 (Announced at Aban 1)
185 | 
186 | ## Assignment Set 7
187 | 
188 | * Set up a simple convolutional neural network model with convolutional, max-pulling, and linear layers (and suitable activation functions) using a different dataset from those that are used in your previous assignments (its size or classes can also be modified).
189 | 
190 | <-> Deadline: Thursday, Aban 22, 23:59 (Announced at Aban 8)
191 | 
192 | ## Assignment Set 8
193 | 
194 | * Apply different optimization algorithms as well as dropout (and/or other regularization methods such as bagging) on the following models:
195 | 
196 |   - The regression and classification models developed in [Assignment Set 6](https://github.com/hhaji/Deep-Learning/blob/master/Recitation-Assignments/README.md#assignment-set-6) or [2](https://github.com/hhaji/Deep-Learning/blob/master/Recitation-Assignments/README.md#assignment-set-2)
197 |   - At least two different models for a classification or regression dataset (up to your choice, but different from previous ones)
198 | 
199 | * Write at least a paragraph regarding (the conclusion of, e.g., comparison of) your experiments and modifications.  
200 | 
201 | **Please Note**: You can find many datasets, for instance, on Kaggle. Moreover, <ins>Try to develop better classifiers and regressors</ins>. Careless model architectures and hyperparameter selections, which result in poor performance, will not be appreciated and may be scored very low! 
202 | 
203 | <-> Deadline: Friday, Azar 7, 23:59 (Announced at Aban 20)
204 | 
205 | ## Assignment Set 9
206 | 
207 | * Develop (at least two) convolutional models for an image dataset (up to your choice, but different from previous ones).
208 | * Exercise one of chapter eight, [Deep Learning with PyTorch](https://www.manning.com/books/deep-learning-with-pytorch)
209 | 
210 | <-> Deadline: Tuesday, Azar 18, 23:59 (Announced at Aban 30)
211 | 
212 | ## Assignment Set 10
213 | 
214 | * Run a hyperparameter optimization study (on a model) with the following settings, then find the best hyperparameters (which lead to the best accuracy).
215 | 
216 |    - 50 Trials | 30, 50, or 70 Epochs | 16, 32, or 64 Batch sizes | At least another hyperparameter, up to your choice
217 |    - Pruning the model if accuracy does not change in 5 epochs
218 | 
219 | **Please Note**: You may find the following blogs helpful: [Pruning — Early Stopping of Poor Trials
220 | ](https://medium.com/pytorch/using-optuna-to-optimize-pytorch-hyperparameters-990607385e36) and
221 | [Pruning Unpromising Trials](https://optuna.readthedocs.io/en/latest/tutorial/007_pruning.html#).
222 | 
223 | <-> Deadline: Friday, Azar 21, 23:59 (Announced at Azar 4)
224 | 
225 | ## Assignment Set 11
226 | 
227 | * Develop at least two recurrent models for (each of) two sequential datasets (up to your choice, but different than previous ones and at least one on the natural language).
228 | * Write at least a paragraph regarding (the conclusion of, e.g., comparison of) your experiments.
229 | 
230 | <-> Deadline: Tuesday, Dey 9, 23:59 (Announced at Azar 20)
231 |  
232 | ## Assignment Set 12
233 | 
234 | * Train at least two classification models for [Linnaeus 5](http://chaladze.com/l5/) dataset utilizing pre-trained [Torchvision.models](https://pytorch.org/docs/stable/torchvision/models.html).
235 | * Develop at least two deep learning models on a task (up to your choice) utilizing transfer learning.
236 | * Write at least a paragraph regarding (the conclusion of, e.g., comparison of) your experiments.
237 | 
238 | <-> Deadline: Thursday, Dey 11, 23:59 (Announced at Azar 20)
239 | 
240 | ## Assignment Set 13
241 | 
242 | * Build and Implement a denoising autoencoder.
243 | * Use autoencoders for feature extraction/dimensionality reduction and then in a classification task (up to your choice, but different from previous ones) in addition to comparison with only-classifier settings.
244 | * Develop a variational autoencoder for generating images (e.g., Fashion-MNIST or CIFAR-10 datasets). 
245 | * Write at least a paragraph regarding (the conclusion of, e.g., comparison of) your experiments. 
246 | 
247 | **Please Note**: For the third part, you may find [these implementations](https://github.com/AntixK/PyTorch-VAE) helpful. Additionally, the tasks must be delivered as your authentic work.
248 | 
249 | <-> Deadline: Friday, Bahman 10, 23:59 (Announced at Dey 23)
250 | 
251 | ## Assignment Set 14 (Optional)
252 | 
253 | * Develop two deep learning tasks utilizing topics discussed in the course or other advanced topics within the field (up to your choice, but different from previous ones).
254 | * Write at least a paragraph for each of the tasks regarding (the conclusion of, e.g., comparison of) your experiments.
255 | 
256 | **Please Note**: It will be rewarded up to 20 percent of the assignments' score. Additionally, the tasks must be delivered as your authentic work.
257 | 
258 | <-> Deadline: Tuesday, Bahman 14, 23:59 (Announced at Dey 23)
259 | 
260 | # Final Project
261 | 
262 | * The final project will be considered as the outcome of the course, which is understanding and effective implementation of deep learning to provide practical solutions to realistic tasks. At least two scenarios for the project can be imagined, applications, and development of deep learning. Developing algorithms and methods is a valuable target, however, it may be challenging. On the other side, applications are recommended for this project. Students should decide on a topic for the project based on their interests as well as discussion with their mentor, who is one of the teaching assistants up to their choice. Mentors could provide advice and assistance during the topic selection and main progress. The following steps and schedule are expected: 
263 | 
264 |     - Contacting a mentor and deciding on a topic - *Deadline: Mehr 30*
265 |     - Submitting a brief proposal of the project - *Deadline: ~~Aban 30~~ Extended to Azar 7*
266 |     - Proposal approval by the mentor - *Please manage it to be done in Aban!*
267 |     - **The main phase of the project**
268 |     - Submitting a poster or a brief report of the progress - *Deadline: ~~Dey 1~~ Extended to Dey 10*
269 |     - Poster/Progress presentation - *~~Dey 3~~ Dey 11 at 16:00*
270 |     - Submitting the final report as well as codes and data - *Deadline: ~~Bahman 10~~ Extended to Bahman 17*
271 |     - Seminar presentation - *~~Bahman 12~~ Bahman 19*
272 |   
273 | **Please Note**: You can find many sample projects as well as datasets on [Kaggle](https://www.kaggle.com/). Moreover, take a look at final projects at Stanford's [CS229](http://cs229.stanford.edu/projects.html). The project will have a notable share in the final score of the course. Creativity, innovations as well as a novelty are (expected and) highly appreciated and will be rewarded accordingly. You should also have an appropriate literature review. Besides, projects will be evaluated by the professor and teaching assistants at the same level. In addition to them, contacts with mentors/assistants are possible through email communications. However, in some cases, skype sessions may be arranged. 
274 | 
275 | **Poster or Progress Report**: We strongly recommend students to prepare a poster (concerning the progress) of their projects. However, a brief report can also be submitted. Furthermore, students should also present them in a meeting on Wednesday, Dey 3 at 16:00 within the [Skype Group](https://join.skype.com/kJ6WepEDrsnt). This meeting is expected to provide an opportunity for students to be familiar with each other's progress on projects as well as sharing ideas. Consequently, students are asked to manage the meeting themselves and the course staff may not attend (However, the meeting will be recorded!). Those who don't prepare a poster should prepare several slides as well!    
276 | 
277 |     Poster/Progress Presentation was held on Thursday, Dey 11 at 16:00! 
278 | 
279 | **The Proposal, Poster/Progress, and Final Report's Format and Submission**: The writing and structure of the three documents are important and will be evaluated. The proposal should be at least three paragraphs; The poster should be in a conventional academic form or  a progress report of at least two pages can be submitted; The final report should also be at least four pages. They should be in the academic/publication format. In addition, documents must be in PDF format, which is written either (preferably) with LaTeX in English. You can use available templates; We recommend you to use [NeurIPS Style](https://nips.cc/Conferences/2020/PaperInformation/StyleFiles). You should upload documents (each document in a folder) as well as codes and data into a repository named exactly "Deep_Learning_F20_Project" (you can rename it once the course ended!). Please enter the title of your project as well as the link to its repository in the following link: [Registration](https://docs.google.com/document/d/1wMK93A3ikwGmQKNKvCK1CXL2c3hgMQ2cr-DzDUU5u14/edit?usp=sharing)  
280 | 
281 | **Seminar Presentations**: Bahman 12 within the [Skype Group](https://join.skype.com/kJ6WepEDrsnt). Please prepare 5 to 10 slides for the presentation. Each student will have 10 minutes to talk and 5 minutes to answer questions. Professor, course staff, and possibly other researchers will attend. Therefore, practice to perfectly present your project. Presentations are also considered in the score of the project. The program of the seminar will be announced prior to the event.
282 |     
283 |     Deep Learning Projects' Seminar was held on Tuesday, Bahman 21 at 15:00!
284 |     
285 | **Please consult with your mentor for more information!**
286 | 
287 | <-> The final project was announced at Shahrivar 26! Please follow the schedule!
288 | 
289 | # Submission Instruction 
290 | 
291 | * Please register through the link: [Registration](https://docs.google.com/document/d/1wMK93A3ikwGmQKNKvCK1CXL2c3hgMQ2cr-DzDUU5u14/edit?usp=sharing)
292 | * Create a repository inside your Github account with the exact name "Deep_Learning_F20_Assignments".
293 | * After completing all tasks of every assignment set, add related Jupyter notebooks (and/or other files) in a folder in the repository, for instance, assignments-1.ipynb inside Assingment_Set_1 folder, for the first set.
294 | * Projects should be located at another repository named "Deep_Learning_F20_Project".
295 | * Please review the [Projects](https://github.com/hhaji/Deep-Learning/tree/master/Projects) for further instructions.
296 | * Solutions to the exercises, as well as mathematical notations, could be written in [LaTeX](https://github.com/hhaji/Deep-Learning#latex) (which is recommended), either in Jupyter notebook or PDF. MS Office or any other software could also be used, otherwise, images of your handwriting should be included, which is not recommended! Please acknowledge which one is used.
297 | 
298 | # Scores
299 | 
300 | * Please follow the [link](https://docs.google.com/spreadsheets/d/1SW1yOld30DYG7Wl6Ex2Oa6cgBpmGU5KhLrQMFaTG5kg/edit#gid=0) for scores (announced at Bahman 22).
301 | * Each assignment set is scored from 100 points as well as extra rewards. Therefore, it can be more than 100.
302 | * **Submission after the deadline will not be accepted!** 
303 | * Failure to comply with the [Academic Honor Code](https://github.com/hhaji/Deep-Learning#academic-honor-code) will result in the ZERO score in each set and may have additional consequences!
304 | * The score of the final project will be calculated from 100 points. It may also be more than 100 due to the added rewards! 
305 | * The final score of assignments will be calculated based on the weighted average of them, from 100 points. Moreover, it may be more than 100 due to the added rewards! 
306 | * Scores may be commented. Furthermore, students could also comment on their scores and request for re-evaluation in the form of email, which will be considered at the end of the course.  
307 | 


--------------------------------------------------------------------------------
/TensorFlow-1/README.md:
--------------------------------------------------------------------------------
 1 | # TensorFlow 1.0
 2 | * GitHub: [Set up Tensorflow 1.0](https://github.com/chiphuyen/stanford-tensorflow-tutorials/tree/master/setup)  by Chip Huyen <br>
 3 | * Slide: [Overview of TensorFlow 1.0](https://docs.google.com/presentation/d/1dizKPtp9hkuTwVDzoGZdYQb_61ULSsSUvaFfDFuhIc4/edit?usp=sharing)  by Chip Huyen <br>
 4 |     * [NoteBook:](https://github.com/hhaji/Deep-Learning/blob/master/NoteBooks/Lec1.ipynb) This was adopted from the slide ([Overview of TensorFlow 1.0](https://docs.google.com/presentation/d/1dizKPtp9hkuTwVDzoGZdYQb_61ULSsSUvaFfDFuhIc4/edit?usp=sharing)) of Chip Huyen <br>
 5 | * Slide: [Operations (TensorFlow 1.0)](https://docs.google.com/presentation/d/1iO_bBL_5REuDQ7RJ2F35vH2BxAiGMocLC6t_N-6eXaE/edit#slide=id.g1bd10f151e_0_0) by Chip Huyen <br>
 6 |     * [NoteBook:](https://github.com/hhaji/Deep-Learning/blob/master/NoteBooks/Lec2.ipynb) This was adopted from the slide ([Operations](https://docs.google.com/presentation/d/1iO_bBL_5REuDQ7RJ2F35vH2BxAiGMocLC6t_N-6eXaE/edit#slide=id.g1bd10f151e_0_0)) of Chip Huyen <br>
 7 | * NoteBook: [Useful Operations in TensorFlow 1](https://github.com/hhaji/Deep-Learning/blob/master/NoteBooks/Lec3.ipynb) <br>
 8 | * Blog: [Variables Sharing in TensorFlow 1.0: Variable vs get_variable](http://stefanocappellini.com/tf-variable-vs-get_variable-sharing/) by Stefano Cappellini 
 9 |      * [NoteBook](https://github.com/StefanoCappellini/tensorflow_tips/blob/master/TF-variable-sharing.ipynb) <br>
10 | * Blog: [Tensorflow 1.0: The Confusing Parts (2)](https://jacobbuckman.com/post/tensorflow-the-confusing-parts-2/) by Jacob Buckman  
11 |      * [NoteBook](https://github.com/hhaji/Deep-Learning/blob/master/NoteBooks/The-Confusing-Parts-2.ipynb) <br>  
12 | * Slide: [Eager Execution](https://docs.google.com/presentation/d/1e1gE2JJXipWm1UJgor_y8pHcM8L8oMaCVtvQvZUBlQY/edit?usp=sharing) by Chip Huyen 
13 | * Blog: [Tensorflow Tutorial](http://cs230.stanford.edu/blog/tensorflow/)   
14 | 


--------------------------------------------------------------------------------
/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-cayman
2 | title: Deep Learning
3 | 


--------------------------------------------------------------------------------
/_layouts/default.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="{{ site.lang | default: "en-US" }}">
 3 |   <head>
 4 | 
 5 |     {% if site.google_analytics %}
 6 |       <script async src="https://www.googletagmanager.com/gtag/js?id={{ site.google_analytics }}"></script>
 7 |       <script>
 8 |         window.dataLayer = window.dataLayer || [];
 9 |         function gtag(){dataLayer.push(arguments);}
10 |         gtag('js', new Date());
11 |         gtag('config', '{{ site.google_analytics }}');
12 |       </script>
13 |     {% endif %}
14 |     <meta charset="UTF-8">
15 | 
16 | {% seo %}
17 |     <meta name="viewport" content="width=device-width, initial-scale=1">
18 |     <meta name="theme-color" content="#157878">
19 |     <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
20 |     <link rel="stylesheet" href="{{ '/assets/css/style.css?v=' | append: site.github.build_revision | relative_url }}">
21 |     
22 |   </head>
23 |   <body>
24 |     <nav>
25 | <ul>
26 | <a href="/">Home</a> &emsp; <a href="/Teaching">Teaching</a></li> &emsp; <a href="/Presentations">Presentations</a></li> &emsp; <a href="/Publications">Publications</a></li> &emsp; <a href="/Graduate Students">Graduate Students</a></li> &emsp; <a href="/Curriculum Vitae">Curriculum Vitae</a></li> &emsp; <a href="/Problems">Problems</a></li> &emsp; <a href="/Contact">Contact</a></li>
27 | </ul>
28 |     <nav>
29 |       
30 |     <header class="page-header" role="banner">
31 |       <h1 class="project-name">{{ site.title | default: site.github.repository_name }}</h1>
32 |       <h2 class="project-tagline">{{ site.description | default: site.github.project_tagline }}</h2>
33 |       {% if site.github.is_project_page %}
34 |         <a href="{{ site.github.repository_url }}" class="btn">View on GitHub</a>
35 |       {% endif %}
36 |       {% if site.show_downloads %}
37 |         <a href="{{ site.github.zip_url }}" class="btn">Download .zip</a>
38 |         <a href="{{ site.github.tar_url }}" class="btn">Download .tar.gz</a>
39 |       {% endif %}    
40 |     </header>
41 | <ul>
42 | <a href="/Deep-Learning">Course (Deep Learning):</a></li> &emsp; <a href="/Deep-Learning/Deep-Learning-Tutorials">Deep Learning Tutorials</a></li> &emsp; <a href="/Deep-Learning/TensorFlow-Tutorials">TensorFlow Tutorials</a></li> &emsp; <a href="/Deep-Learning/Graph-Neural-Networks">Graph Neural Networks</a></li> &emsp; <a href="/Deep-Learning/Projects">Projects</a></li> &emsp; <a href="/Deep-Learning/Data-Handling">Data Handling</a></li>    
43 | </ul>
44 |     <main id="content" class="main-content" role="main">
45 |       {{ content }}
46 | 
47 |       <footer class="site-footer">
48 |         {% if site.github.is_project_page %}
49 |           <span class="site-footer-owner"><a href="{{ site.github.repository_url }}">{{ site.github.repository_name }}</a> is maintained by <a href="{{ site.github.owner_url }}">{{ site.github.owner_name }}</a>.</span>
50 |         {% endif %}
51 |         <span class="site-footer-credits">This page was generated by <a href="https://pages.github.com">GitHub Pages</a>.</span>
52 |       </footer>
53 |     </main>
54 |   </body>
55 | </html>
56 | 


--------------------------------------------------------------------------------