├── .gitignore ├── README.md ├── app.py ├── app.pyc ├── codes ├── app.logs ├── employee_class.py ├── hello_click.py ├── inheritance_example.py ├── it.py ├── pro_cpu.py ├── thr.py └── thr_cpu.py ├── docs ├── app.md ├── best_practices.md ├── cliapp.md ├── decorators.md ├── flask.md ├── flaskandorm.md ├── flaskdb.md ├── generators.md ├── gil.md ├── index.md ├── internals.md ├── iterators.md ├── logging.md ├── mem.md ├── metrics.md ├── oops.md ├── package.md ├── syntax.md ├── threading.md └── tooling.md ├── hello.py ├── images └── python_interpreter_basic.png ├── mkdocs.yml ├── requirements.txt ├── static └── style.css └── templates ├── hello.html ├── login.html └── page_not_found.html /.gitignore: -------------------------------------------------------------------------------- 1 | site 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to run this tutorial. 2 | 3 | It uses mkdocs for documentation. You can follow the below steps to run it in your local system 4 | 5 | ### How to run this 6 | 7 | virtualenv venv 8 | source venv/bin/activate 9 | git clone git@github.com:chowmean/python-training-for-devops.git 10 | cd python-training-for-devops 11 | pip install -r requirements.txt 12 | mkdocs serve 13 | 14 | ### Feel free to write to me if you get any errors or you want me to improve on something. 15 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, escape, url_for, render_template, session, request, flash, jsonify 2 | import os 3 | app = Flask(__name__) 4 | app.secret_key = os.urandom(12) 5 | 6 | @app.route('/') 7 | def hello_world(): 8 | return jsonify({'name':"sdgshd"}), 201 9 | 10 | 11 | @app.route('/projects/') 12 | def projects(): 13 | return 'The project page' 14 | 15 | 16 | @app.route('/about') 17 | def about(): 18 | return 'The about page..' 19 | 20 | 21 | @app.route('/user/') 22 | def show_user_profile(username): 23 | # show the user profile for that user 24 | return render_template('hello.html', name=username) 25 | 26 | 27 | @app.route('/post/') 28 | def show_post(post_id): 29 | # show the post with the given id, the id is an integer 30 | return 'Post %d' % post_id 31 | 32 | @app.route('/path/') 33 | def show_subpath(subpath): 34 | # show the subpath after /path/ 35 | return 'Subpath %s' % subpath 36 | 37 | 38 | @app.route('/login') 39 | def home(): 40 | if not session.get('logged_in'): 41 | return render_template('login.html') 42 | else: 43 | return "Hello Boss!" 44 | 45 | 46 | @app.route('/login', methods=['POST']) 47 | def do_admin_login(): 48 | if request.form['password'] == 'password' and request.form['username'] == 'admin': 49 | session['logged_in'] = True 50 | else: 51 | flash('wrong password!') 52 | return home() 53 | 54 | 55 | @app.errorhandler(404) 56 | def page_not_found(error): 57 | return render_template('page_not_found.html'), 404 58 | 59 | # app.run() 60 | with app.test_request_context(): 61 | print(url_for('about')) 62 | print(url_for('about')) 63 | print(url_for('about', next='/')) 64 | print(url_for('show_user_profile', username='John Doe')) -------------------------------------------------------------------------------- /app.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chowmean/python-training-for-devops/6d2f9b3aba1567fe638c1d6aa66f4ca9b0a83fb0/app.pyc -------------------------------------------------------------------------------- /codes/app.logs: -------------------------------------------------------------------------------- 1 | 02-Jul-19 21:53:56 - Initilizing application. 2 | 02-Jul-19 21:53:58 - Contacting github for repos. 3 | 02-Jul-19 21:53:58 - Starting new HTTPS connection (1): api.github.com:443 4 | 02-Jul-19 21:53:59 - https://api.github.com:443 "GET /users/chowmean/repos HTTP/1.1" 200 None 5 | 02-Jul-19 21:53:59 - Github data received. 6 | 02-Jul-19 21:54:25 - Initilizing application. 7 | 02-Jul-19 21:55:47 - Initilizing application. 8 | 02-Jul-19 21:55:49 - Contacting github for repos. 9 | 02-Jul-19 21:55:49 - Starting new HTTPS connection (1): api.github.com:443 10 | 02-Jul-19 21:55:50 - https://api.github.com:443 "GET /users/chowmean/repos HTTP/1.1" 200 None 11 | 02-Jul-19 21:55:50 - Github data received. 12 | -------------------------------------------------------------------------------- /codes/employee_class.py: -------------------------------------------------------------------------------- 1 | class Employee(): 2 | 3 | __phone_number="0000000000" 4 | 5 | def __init__(self, name, age): 6 | self.name = name 7 | self.age = age 8 | 9 | def get_age(self): 10 | return self.age 11 | 12 | def set_age(self,value): 13 | self.age = value 14 | 15 | def get_name(self): 16 | return self.name 17 | 18 | def set_phone(self,value): 19 | self.__phone_number = value 20 | 21 | def get_phone(self): 22 | return self.__phone_number 23 | 24 | def set_phone(self, value): 25 | self.__phone_number = value 26 | 27 | if __name__ == '__main__': 28 | e = Employee("gaurav", 12) 29 | print(e.get_phone()) 30 | e.set_phone("89628363**") 31 | print(e.get_phone()) 32 | 33 | -------------------------------------------------------------------------------- /codes/hello_click.py: -------------------------------------------------------------------------------- 1 | import click 2 | import requests 3 | import json 4 | import logging 5 | 6 | loglevels={ 7 | "warn": logging.WARNING, 8 | "info": logging.INFO, 9 | "error": logging.ERROR, 10 | "debug": logging.DEBUG, 11 | "critical": logging.CRITICAL, 12 | } 13 | @click.command() 14 | @click.option('--username', prompt='Github username', help='Github username to get repo details.') 15 | @click.option('--log_file', prompt='Log file name', help='File to which log', default='app.logs') 16 | @click.option('--log_level', prompt='Log level', help='Log level to use', default='warn') 17 | #logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', filename=log-file, level=loglevels[log_level]) 18 | 19 | def get_projects(username): 20 | logging.debug("Contacting github for repos.") 21 | a = requests.get("https://api.github.com/users/"+username+"/repos") 22 | logging.debug("Github data received.") 23 | data = json.loads(a.content) 24 | return data 25 | 26 | if __name__ == '__main__': 27 | logging.debug("Initilizing application.") 28 | get_projects() 29 | -------------------------------------------------------------------------------- /codes/inheritance_example.py: -------------------------------------------------------------------------------- 1 | class Employee(): 2 | 3 | __phone_number="0000000000" 4 | 5 | def __init__(self, name, age): 6 | self.name = name 7 | self.age = age 8 | 9 | def get_age(self): 10 | return self.age 11 | 12 | def set_age(self,value): 13 | self.age = value 14 | 15 | def get_name(self): 16 | return self.name 17 | 18 | def set_phone(self,value): 19 | self.__phone_number = value 20 | 21 | def get_phone(self): 22 | return self.__phone_number 23 | 24 | def set_phone(self, value): 25 | self.__phone_number = value 26 | 27 | class VIP(Employee): 28 | 29 | def get_phone(self): 30 | return "************" 31 | 32 | def get_vip_access(self): 33 | return "access_granted" 34 | 35 | if __name__ == '__main__': 36 | 37 | #Simple Employee 38 | emp = Employee("gaurav", 12) 39 | print(emp.get_phone()) 40 | emp.set_phone("89628363**") 41 | print(emp.get_phone()) 42 | 43 | #Vip employee inherting employee properties 44 | vip = VIP("Venky", 54) 45 | 46 | #Overriden method hides phone number 47 | print(vip.get_phone()) 48 | 49 | #Extra added function. 50 | print(vip.get_vip_access()) 51 | 52 | #This function calls the inherited method. 53 | print(vip.get_name()) 54 | 55 | -------------------------------------------------------------------------------- /codes/it.py: -------------------------------------------------------------------------------- 1 | # class Repeater: 2 | # def __init__(self, value): 3 | # self.value = value 4 | 5 | # def __iter__(self): 6 | # return self 7 | 8 | # def __next__(self): 9 | # return self.value 10 | 11 | # a = Repeater("asdad") 12 | 13 | # for item in a: 14 | # print(item) 15 | 16 | 17 | 18 | class RepeaterNumber: 19 | def __init__(self, value): 20 | self.value = value 21 | self.index = 0 22 | self.length = len(value) 23 | 24 | def __iter__(self): 25 | return self 26 | 27 | def __next__(self): 28 | 29 | if self.index= self.length: 34 | raise StopIteration 35 | 36 | a=[1,2,3,4,5,6,7,8,9] 37 | a=RepeaterNumber(a) 38 | for item in a: 39 | print(item) 40 | 41 | a=[1,2,3,4,5,6,7,8,9] 42 | a=RepeaterNumber(a) 43 | it = a.__iter__() 44 | print(it.__next__()) 45 | print(it.__next__()) 46 | print(it.__next__()) -------------------------------------------------------------------------------- /codes/pro_cpu.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import logging 3 | import time 4 | from datetime import datetime 5 | 6 | def process_function(name): 7 | #logging.info("Thread %s: starting", name) 8 | for i in range (10000): 9 | a = (i*i) 10 | b = (i*i)+1 11 | #logging.info("Thread %s: finishing", name) 12 | 13 | if __name__ == "__main__": 14 | start_time = datetime.now() 15 | format = "%(asctime)s: %(message)s" 16 | #logging.basicConfig(format=format, level=logging.INFO, 17 | # datefmt="%H:%M:%S") 18 | 19 | #logging.info("Main : before creating thread") 20 | for i in range (100): 21 | x = multiprocessing.Process(target=process_function, args=(i, )) 22 | x.start() 23 | x.join() 24 | #y = threading.Thread(target=thread_function, args=(1,)) 25 | #logging.info("Main : before running thread") 26 | #x.start() 27 | #y.start() 28 | #logging.info("Main : wait for the thread to finish") 29 | # x.join() 30 | #logging.info("Main : all done") 31 | end_time = datetime.now() 32 | print(end_time-start_time) 33 | -------------------------------------------------------------------------------- /codes/thr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | import time 4 | 5 | def thread_function(name): 6 | logging.info("Thread %s: starting", name) 7 | time.sleep(1) 8 | logging.info("sleeping") 9 | time.sleep(1) 10 | logging.info("Thread %s: finishing", name) 11 | 12 | if __name__ == "__main__": 13 | format = "%(asctime)s: %(message)s" 14 | logging.basicConfig(format=format, level=logging.INFO, 15 | datefmt="%H:%M:%S") 16 | 17 | logging.info("Main : before creating thread") 18 | for i in range (1000): 19 | x = threading.Thread(target=thread_function, args=(i,)) 20 | x.start() 21 | #y = threading.Thread(target=thread_function, args=(1,)) 22 | logging.info("Main : before running thread") 23 | #x.start() 24 | #y.start() 25 | logging.info("Main : wait for the thread to finish") 26 | # x.join() 27 | logging.info("Main : all done") 28 | -------------------------------------------------------------------------------- /codes/thr_cpu.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | import time 4 | from datetime import datetime 5 | 6 | def thread_function(name): 7 | #logging.info("Thread %s: starting", name) 8 | for i in range (10000): 9 | a = (i*i) 10 | b = (i*i)+1 11 | #logging.info("Thread %s: finishing", name) 12 | 13 | if __name__ == "__main__": 14 | start_time = datetime.now() 15 | format = "%(asctime)s: %(message)s" 16 | #logging.basicConfig(format=format, level=logging.INFO, 17 | # datefmt="%H:%M:%S") 18 | 19 | #logging.info("Main : before creating thread") 20 | for i in range (100): 21 | x = threading.Thread(target=thread_function, args=(i,)) 22 | x.start() 23 | x.join() 24 | #y = threading.Thread(target=thread_function, args=(1,)) 25 | #logging.info("Main : before running thread") 26 | #x.start() 27 | #y.start() 28 | #logging.info("Main : wait for the thread to finish") 29 | # x.join() 30 | #logging.info("Main : all done") 31 | end_time = datetime.now() 32 | print(end_time-start_time) 33 | -------------------------------------------------------------------------------- /docs/app.md: -------------------------------------------------------------------------------- 1 | # Lets create an Application 2 | 3 | We will create an application which will have following. 4 | 5 | 1. /register endpoint which will be post and takes arguement as username, password, email, phone. 6 | 2. /login endpoint which will take username and password and sends a token if correct. 7 | 3. /users if token is correct send all the user details. 8 | 4. Proper Logging 9 | 5. Proper Monitoring 10 | 6. Proper tests 11 | 12 | -------------------------------------------------------------------------------- /docs/best_practices.md: -------------------------------------------------------------------------------- 1 | # Best practices for programming and python 2 | 3 | * Add readability to your code as much as possible. 4 | * Function name should tell what this function does. 5 | * Function should be visible in one screen. You should not scroll to view the whole function. 6 | * Try to write pure funtions instead of impure functions. [Read here about pure and impure functions](https://www.learnsteps.com/pure-impure-functions/) 7 | * One function should do one thing and that thing properly. 8 | * Logging is very important. 9 | * You should chose level of logging very wisely. Lot of logging will make your application slow. 10 | * Less logging you may miss out required info for debugging. 11 | * Failures are normal. Fail as early as possible and as loud as possible. 12 | * Metrics provides you with a way to keep track what your code is working with respect to how it was working earlier. 13 | * Try to structure your code in such a way that if you want to add something new you can do it in minimum effort. 14 | * Documentation is very important. Try to document project you are about to build in advance this will make your thought clear about what you are going to make and how you will achieve it. 15 | * Comment wherever is necessary. You comments will help the person who will take over code to understand it easily. 16 | * You program should be config driven. Don't hardcode anything in your code. 17 | * Test cases are very important. 18 | * You will have confidence in your code and less bug will go to production. 19 | * Edge cases are important to consider. 20 | * There are people who actually do test driven development. This means they write the test case first for what they want to achieve and then they write the code for it. 21 | * Always think that your software may need to interact with other software in future so write it in a way that others can interact with it. -------------------------------------------------------------------------------- /docs/cliapp.md: -------------------------------------------------------------------------------- 1 | 2 | ## Introduction 3 | 4 | In this part we will be creating a cli application using a well known framework called Click. 5 | 6 | ## What you need to start this? 7 | 8 | Basic knowledge of python syntax loops, how to use packages and how to run python program. Knowledge of cli app will be a good addition. 9 | 10 | ## Click package 11 | 12 | Click is a Python package for creating beautiful command line interfaces in a composable way with as little code as necessary. It’s the “Command Line Interface Creation Kit”. It’s highly configurable but comes with sensible defaults out of the box. 13 | 14 | Click adds functionality like help by default which is very useful in case of cli apps. Functionality like below 15 | 16 |
 17 | (venv) $chowmean@chowmean-MACH-WX9:$ python hello_click.py help
 18 | Usage: hello_click.py [OPTIONS]
 19 | Try "hello_click.py --help" for help.
 20 | 
 21 | Error: Got unexpected extra argument (help)
 22 | (venv) $chowmean@chowmean-MACH-WX9:$ python hello_click.py --help
 23 | Usage: hello_click.py [OPTIONS]
 24 | 
 25 |   Simple program that greets NAME for a total of COUNT times.
 26 | 
 27 | Options:
 28 |   --help  Show this message and exit.
 29 | (venv) $chowmean@chowmean-MACH-WX9:$ 
 30 | 
31 | 32 | ## Lets code 33 | 34 | Create a directory for your project. Then create a requirements.txt file in your directory. After that initialize an empty env for you python project and then run pip install to install the required packages. Here package will be click so put click in your requirements file. All of this is below 35 | 36 |
 37 | mkdir project_name
 38 | cd project_name && touch requirements.txt
 39 | virtualenv venv
 40 | source venv/bin/activate
 41 | echo "click" >> requirements.txt
 42 | pip install -r requirements.txt
 43 | 
44 | 45 | Our project directory is ready create a file with any name and start writing the code below. 46 | 47 | Here we will be creating a very basic cli app which says hello. 48 | 49 |
 50 | import click
 51 | @click.command()
 52 | def hello():
 53 |     """Simple program that greets NAME for a total of COUNT times."""
 54 |     click.echo('Hello')
 55 | 
 56 | if __name__ == '__main__':
 57 |     hello()
 58 | 
 59 | 
60 | Save the above snippet in file called hello.py 61 | Run this using python hello.py and you will see hello as output. 62 | 63 | ### Arguments and options 64 | 65 | Lets first distinguish between these two. 66 | 67 | *Options* are the values that you can provide to your program through CLI. 68 | 69 | *Arguments* work similarly to options but are positional. They also only support a subset of the features of options due to their syntactical nature. 70 | 71 | Arguments cannot do anything more than options so we will have a look at options in detail 72 | 73 | #### Options 74 | 75 | For taking options click provide a very easy options. Have a look at code below 76 | 77 |
 78 | import click
 79 | @click.command()
 80 | @click.option('--name', prompt='Your name',
 81 |               help='The person to greet.')
 82 | def hello(name):
 83 |     for x in range(10):
 84 |         click.echo('Hello %s!' % name)
 85 | 
 86 | if __name__ == '__main__':
 87 |     hello()
 88 | 
89 | 90 | Here you can take your name as CLI argument and then this program will print hello your_name. Run this command to run it. 91 | 92 |
 93 | python hello.py --name=conan
 94 | 
95 | 96 | Now you can divide options in two type required and optional ones. 97 | For that you only have to pass *required=True* and that option will be mandatory 98 | 99 |
100 | import click
101 | @click.command()
102 | @click.option('--name', prompt='Your name',
103 |               help='The person to greet.', required=True)
104 | @click.option('--age', prompt='Your age',
105 |               help="The person's age.", default=35)
106 | def hello(name, age):
107 |     for x in range(1):
108 |         click.echo('%s is %s years old!!!' % (name,age))
109 | 
110 | if __name__ == '__main__':
111 |     hello()
112 | 
113 | 114 | In above code now name is a mandatory option. 115 | 116 | ### Prompts 117 | 118 | Prompts are a way to take user in input at the run time. These are useful when you want to enter some password and don't want it to be recorded in history command. 119 | 120 | There are two types of prompts here. 121 | #### Input prompts. 122 | 123 | Input prompts are used for taking inputs from users. Look at the example below 124 |
125 | value = click.prompt('Please enter your nanme', type=string)
126 | 
127 | 128 | #### Confirmation prompts. 129 | Confirmation prompts are for confirming if you want to continue. 130 | 131 |
132 | import click
133 | @click.command()
134 | def hello():
135 |     if click.confirm('Do you want to continue?'):
136 |     	click.echo('user entered yes')
137 |     else:
138 |     	click.echo('user entered No')
139 | 
140 | if __name__ == '__main__':
141 |     hello()
142 | 
143 | #Examples: 144 | this example explains about hiding the prompted value(useful in case of passwords) 145 |
146 | import click
147 | @click.command()
148 | @click.option('--password', prompt=True, hide_input=True,
149 |               confirmation_prompt=True)
150 | def encrypt(password):
151 |     click.echo('Encrypting password to %s' % password.encode('rot13'))
152 | if __name__ == '__main__':
153 |     encrypt()
154 | 
155 | 156 | You can give option `abort=True`. If you want to abort on not confirming. 157 |
158 | import click
159 | @click.command()
160 | def hello():
161 |     if click.confirm('Do you want to continue?', default=True, abort=True):
162 |     	click.echo('user entered yes')
163 |     else:
164 |     	click.echo('user entered No')
165 |     print "!!!Yahhoo... user allowed me to get execute by entering Yes"	
166 | if __name__ == '__main__':
167 |     hello()
168 | 
169 | 170 | ## Click application to pull repos 171 | 172 | Here we will build a very small application to get all the public github repos of any users. 173 | 174 | Lets first plan how we will do this. So there will be only two steps. One where we ask for the username of the user to search repos for and next is getting the repo and printing it. 175 | 176 | For first part we will get username like below 177 | 178 | 179 |
180 | import click
181 | @click.command()
182 | @click.option('--username', prompt='Github username',
183 |               help='Github username to get repo details.')
184 | 
185 | def get_projects(username):
186 |     return username
187 | 
188 | if __name__ == '__main__':
189 |     get_projects()
190 | 
191 | 192 | Here we got the input from user to get the username. Now we have to get the details of its repository. 193 | 194 | For that we need to make an api call to open githib apis. For making http calls in python we use a very famous library called requests. 195 | 196 | Look at the code below to understand how requests library works. 197 | 198 |
199 | import requests
200 | a = requests.get("www.google.com")
201 | print(a.content)
202 | 
203 | 204 | As simple as this. It supports all the options that are there in http calls like headers, methods etc. We will use this library to make a call to github api. 205 | 206 | 207 | Look at the code below 208 | 209 |
210 | import click
211 | import requests
212 | @click.command()
213 | @click.option('--username', prompt='Github username',
214 |               help='Github username to get repo details.')
215 | 
216 | def get_projects(username):
217 | 	a = requests.get("https://api.github.com/users/"+username+"/repos")
218 |     print(a.content)
219 | 
220 | if __name__ == '__main__':
221 |     get_projects()
222 | 
223 | 
224 | 225 | You got the list of repository but they have lot more data and they need to be in understood by program as object. Right now they are in json format. For handling json we have a library called json in python 226 | 227 | Lets see that.. 228 | 229 |
230 | import click
231 | import requests
232 | import json
233 | @click.command()
234 | @click.option('--username', prompt='Github username',
235 |               help='Github username to get repo details.')
236 | 
237 | def get_projects(username):
238 | 	a = requests.get("https://api.github.com/users/"+username+"/repos")
239 |     data = json.loads(a.content)
240 |     for item in data:
241 |     	print item['name']
242 | 
243 | if __name__ == '__main__':
244 |     get_projects()
245 | 
246 | 
247 | 248 | This will print the names of all the repos -------------------------------------------------------------------------------- /docs/decorators.md: -------------------------------------------------------------------------------- 1 | # Decorators 2 | ###First-Class Objects 3 | First class objects in a language are handled uniformly throughout. 4 | 5 | These are the some properties of first class objects 6 | 7 | 1. To be named by variables. 8 | 2. To be passed as arguments to procedures. 9 | 3. To be returned as values of procedures. 10 | 4. To be incorporated into data structures 11 | 12 | In Python, functions are first-class objects. This means that functions can be passed around and used as arguments, just like any other object (string, int, float, list, and so on). Consider the following three functions: 13 | #####Example 1 14 |
 15 | def say_hello(name):
 16 |     return "Hello %s"% name
 17 | def be_awesome(name):
 18 |     return "Yo %s"% name
 19 | def greet_bob(greeter_func):
 20 |     return greeter_func("Bob")
 21 | 
22 | 23 | Here, say_hello() and be_awesome() are regular functions that expect a name given as a string. The greet_bob() function however, expects a function as its argument. We can, for instance, pass it the say_hello() or the be_awesome() function:. 24 | 25 | We can execute in python shell 26 | 27 |
 28 | >>> greet_bob(say_hello)
 29 | 'Hello Bob'
 30 | >>> greet_bob(be_awesome)
 31 | 'Yo Bob'
 32 | 
33 | 34 | ###Inner Functions: 35 | It’s possible to define functions inside other functions. Such functions are called inner functions. Here’s an example of a function with two inner functions: 36 |
 37 | def parent():
 38 |     print("Printing from the parent() function")
 39 |     def first_child():
 40 |         print("Printing from the first_child() function")
 41 |     def second_child():
 42 |         print("Printing from the second_child() function")
 43 |     second_child()
 44 |     first_child()
 45 | 
46 | 47 | what happens when you call parent function? 48 | 49 |
 50 | >>> parent()
 51 | Printing from the parent() function
 52 | Printing from the second_child() function
 53 | Printing from the first_child() function
 54 | 
55 | 56 | Note that the order in which the inner functions are defined does not matter. Like with any other functions, the printing only happens when the inner functions are executed. 57 | 58 | Furthermore, the inner functions are not defined until the parent function is called. They are locally scoped to parent(): they only exist inside the parent() function as local variables. Try calling first_child(). You should get an error: 59 | 60 |
 61 | >>>first_child()
 62 | Traceback (most recent call last):
 63 |   File "", line 1, in 
 64 | NameError: name 'first_child' is not defined
 65 | 
66 | 67 | Whenever you call parent(), the inner functions first_child() and second_child() are also called. But because of their local scope, they aren’t available outside of the parent() function. 68 | 69 | ###Returning Functions From Functions: 70 | Python also allows you to use functions as return values. The following example returns one of the inner functions from the outer parent() function: 71 |
 72 | def parent(num):
 73 |     def first_child():
 74 |         return "Hi, I am Emma"
 75 |     def second_child():
 76 |         return "Call me Liam"
 77 |     if num == 1:
 78 |         return first_child
 79 |     else:
 80 |         return second_child
 81 | 
82 | 83 | Note that you are returning first_child without the parentheses. Recall that this means that you are returning a reference to the function first_child. In contrast first_child() with parentheses refers to the result of evaluating the function. This can be seen in the following example: 84 | 85 |
 86 | >>> first = parent(1)
 87 | >>> second = parent(2)
 88 | 
 89 | >>> first
 90 | .first_child at 0x7f599f1e2e18>'
 91 | 
 92 | >>> second
 93 | .second_child at 0x7f599dad5268>
 94 | 
95 | 96 | You can now use first and second as if they are regular functions, even though the functions they point to can’t be accessed directly: 97 | 98 |
 99 | >>> first()
100 | 'Hi, I am Emma'
101 | 
102 | >>> second()
103 | 'Call me Liam'
104 | 
105 | 106 | ##Simple Decorators: 107 | Now that you’ve seen that functions are just like any other object in Python, you’re ready to move on and see the magical beast that is the Python decorator. Let’s start with an example 108 | 109 | Example: 110 |
111 | def my_decorator(func):
112 |     def wrapper():
113 |         print("Something is happening before the function is called.")
114 |         func()
115 |         print("Something is happening after the function is called.")
116 |     return wrapper
117 | def say_whee():
118 |     print("Whee!")
119 | say_whee = my_decorator(say_whee)
120 | 
121 | 122 | Can you guess what happens when you call say_whee()? Try it: 123 | 124 |
125 | >>> say_whee()
126 | Something is happening before the function is called.
127 | Whee!
128 | Something is happening after the function is called.
129 | 
130 | 131 | To understand what’s going on here, look back at the previous examples. We are literally just applying everything you have learned so far. 132 | 133 | The so-called decoration happens at the following line: 134 |
135 | say_whee = my_decorator(say_whee)
136 | 
137 | 138 | In effect, the name say_whee now points to the wrapper() inner function. Remember that you return wrapper as a function when you call my_decorator(say_whee): 139 | 140 |
141 | >>> say_whee
142 | .wrapper at 0x7f3c5dfd42f0>
143 | 
144 | 145 | However, wrapper() has a reference to the original say_whee() as func, and calls that function between the two calls to print(). 146 | 147 | #####Put simply: decorators wrap a function, modifying its behavior. 148 | 149 | #####Example: 150 |
151 | from datetime import datetime
152 | def not_during_the_night(func):
153 |     def wrapper():
154 |         if 7 <= datetime.now().hour < 22:
155 |             func()
156 |         else:
157 |             pass  # Hush, the neighbors are asleep
158 |     return wrapper
159 | def say_whee():
160 |     print("Whee!")
161 | say_whee = not_during_the_night(say_whee)
162 | 
163 | 164 | If you try to call say_whee() after bedtime, nothing will happen: 165 | 166 |
167 | >>> say_whee()
168 | 
169 | 170 | ###Syntactic Sugar! 171 | The way you decorated say_whee() above is a little heavy. First of all, you end up typing the name say_whee three times. 172 | 173 | Python allows you to use decorators in a simpler way with the @ symbol, sometimes called the “pie” syntax. The following example does the exact same thing as the first decorator example: 174 | 175 | ####Example: 176 |
177 | def my_decorator(func):
178 |     def wrapper():
179 |         print("Something is happening before the function is called.")
180 |         func()
181 |         print("Something is happening after the function is called.")
182 |     return wrapper
183 | 
184 | @my_decorator
185 | def say_whee():
186 |     print("Whee!")
187 | 
188 | 189 | ###Reusing Decorators: 190 | Recall that a decorator is just a regular Python function. All the usual tools for easy reusability are available. Let’s move the decorator to its own module that can be used in many other functions. 191 | 192 |
193 | def do_twice(func):
194 |     def wrapper_do_twice():
195 |         func()
196 |         func()
197 |     return wrapper_do_twice
198 | @do_twice
199 | def say_whee():
200 |     print("Whee!")
201 | 
202 | 203 | ###Decorating Functions With Arguments: 204 | Say that you have a function that accepts some arguments. Can you still decorate it? Let’s try: 205 | 206 | 207 | Example: 208 |
209 | def do_twice(func):
210 |     def wrapper_do_twice():
211 |         func()
212 |         func()
213 |     return wrapper_do_twice
214 | 
215 | @do_twice
216 | def greet(name):
217 |     print " hello %s"%name
218 | 
219 | def say_whee():
220 |     print "Whee!"
221 | 
222 | 223 | Unfortunately, running this code raises an error: 224 | 225 |
226 | >>> greet("World")
227 | Traceback (most recent call last):
228 |   File "", line 1, in 
229 | TypeError: wrapper_do_twice() takes 0 positional arguments but 1 was given
230 | 
231 | 232 | The problem is that the inner function wrapper_do_twice() does not take any arguments, but name="World" was passed to it. You could fix this by letting wrapper_do_twice() accept one argument, but then it would not work for the say_whee() function you created earlier. 233 | 234 | The solution is to use *args and **kwargs in the inner wrapper function. Then it will accept an arbitrary number of positional and keyword arguments. Rewrite decorators.py as follows: 235 | 236 |
237 | def do_twice(func):
238 |     def wrapper_do_twice(*args, **kwargs):
239 |         func(*args, **kwargs)
240 |         func(*args, **kwargs)
241 |     return wrapper_do_twice
242 | @do_twice
243 | def greet(name):
244 |     print " hello %s"%name
245 | def say_whee():
246 |     print "Whee!"
247 | 
248 | 249 | Now call functions 250 | 251 |
252 | >>> say_whee()
253 | Whee!
254 | Whee!
255 | >>> greet("World")
256 | Hello World
257 | Hello World
258 | 
259 | 260 | ###Returning Values From Decorated Functions: 261 | What happens to the return value of decorated functions? Well, that’s up to the decorator to decide. Let’s say you decorate a simple function as follows: 262 |
263 | def do_twice(func):
264 |     def wrapper_do_twice(*args, **kwargs):
265 |         func(*args, **kwargs)
266 |         func(*args, **kwargs)
267 |     return wrapper_do_twice
268 |     
269 | @do_twice
270 | def return_greeting(name):
271 |     print("Creating greeting")
272 |     return "Hi %s"%name
273 | 
274 | 275 | Call function 276 | 277 |
278 | >>> hi_adam = return_greeting("Adam")
279 | Creating greeting
280 | Creating greeting
281 | >>> print(hi_adam)
282 | None
283 | 
284 | 285 | Oops, your decorator ate the return value from the function. 286 | modify your wrapper to return something 287 | 288 |
289 | def do_twice(func):
290 |     def wrapper_do_twice(*args, **kwargs):
291 |         func(*args, **kwargs)
292 |         return func(*args, **kwargs)
293 |     return wrapper_do_twice
294 | 
295 | 296 | ###Real world use cases: 297 | 298 | 1. Timing functions 299 | 2. Registering functions 300 | 3. checking for authentication 301 | 302 | 303 | ###Examples: 304 |
305 | from flask import Flask, g, request, redirect, url_for
306 | import functools
307 | app = Flask(__name__)
308 | 
309 | def login_required(func):
310 |     """Make sure user is logged in before proceeding"""
311 |     @functools.wraps(func)
312 |     def wrapper_login_required(*args, **kwargs):
313 |         if g.user is None:
314 |             return redirect(url_for("login", next=request.url))
315 |         return func(*args, **kwargs)
316 |     return wrapper_login_required
317 | 
318 | @app.route("/secret")
319 | @login_required
320 | def secret():
321 |     ...
322 | 
323 | 324 | 325 | -------------------------------------------------------------------------------- /docs/flask.md: -------------------------------------------------------------------------------- 1 | #Flask 2 | ###A Minimal Application 3 | A minimal Flask application looks something like this: 4 | 5 |
  6 | from flask import Flask
  7 | app = Flask(__name__)
  8 | 
  9 | @app.route('/')
 10 | def get_user_data(user):
 11 |     user_data = db.get(user)
 12 |     return user_Data
 13 | 
14 | 15 | ######So what did that code do? 16 | 17 | 1. First we imported the Flask class. An instance of this class will be our WSGI application. 18 | 19 | 2. Next we create an instance of this class. The first argument is the name of the application’s module or package. If you are using a single module (as in this example), you should use __name__ because depending on if it’s started as application or imported as module the name will be different ('__main__' versus the actual import name). This is needed so that Flask knows where to look for templates, static files, and so on. For more information have a look at the Flask documentation. 20 | 21 | 3. We then use the route() decorator to tell Flask what URL should trigger our function. 22 | 23 | 4. The function is given a name which is also used to generate URLs for that particular function, and returns the message we want to display in the user’s browser. 24 | 25 | Just save it as hello.py or something similar. Make sure to not call your application flask.py because this would conflict with Flask itself. 26 | 27 | To run the application you can either use the flask command or python’s -m switch with Flask.
28 | Before you can do that you need to tell your terminal the application to work with by exporting the FLASK_APP environment variable: 29 | 30 |
 31 | $ export FLASK_APP=hello.py
 32 | $ flask run
 33 |  * Running on http://127.0.0.1:5000/
 34 |  
35 | 36 | 37 | Alternatively you can use python -m flask: 38 | 39 |
 40 | $ export FLASK_APP=hello.py
 41 | $ python -m flask run
 42 |  * Running on http://127.0.0.1:5000/
 43 | 
44 | 45 | This launches a very simple builtin server, which is good enough for testing but probably not what you want to use in production 46 | 47 | 48 | ###Externally Visible Server: 49 | If you run the server you will notice that the server is only accessible from your own computer, not from any other in the network. This is the default because in debugging mode a user of the application can execute arbitrary Python code on your computer. 50 | 51 | If you have the debugger disabled or trust the users on your network, you can make the server publicly available simply by adding --host=0.0.0.0 to the command line: 52 | 53 |
 54 | flask run --host=0.0.0.0
 55 | 
56 | 57 | This tells your operating system to listen on all public IPs. 58 | 59 | ###what happens when type flask run? 60 | 61 | ###Debug Mode: 62 | 1. server will reload itself on code changes 63 | 2. provide you with a helpful debugger 64 | 65 |
 66 | export FLASK_ENV=development
 67 | flask run
 68 | 
69 | 70 | 71 | ###Variable Rules: 72 | You can add variable sections to a URL by marking sections with . Your function then receives the as a keyword argument. Optionally, you can use a converter to specify the type of the argument like . 73 | 74 | from flask import Flask, render_template, escape 75 | app = Flask(__name__) 76 | 77 | @app.route('/user/') 78 | def show_user_profile(username): 79 | # show the user profile for that user 80 | return 'User %s' % escape(username) 81 | 82 | @app.route('/post/') 83 | def show_post(post_id): 84 | # show the post with the given id, the id is an integer 85 | return 'Post %d' % post_id 86 | 87 | @app.route('/path/') 88 | def show_subpath(subpath): 89 | # show the subpath after /path/ 90 | return 'Subpath %s' % subpath 91 | 92 | types of converters:
93 | 1. string
94 | 2. int
95 | 3. float
96 | 4. path
97 | 5. uuid
98 | 99 | 100 | ###Unique URLs / Redirection Behavior 101 | The following two rules differ in their use of a trailing slash. 102 | 103 | from flask import Flask 104 | app = Flask(__name__) 105 | 106 | @app.route('/projects/') 107 | def projects(): 108 | return 'The project page' 109 | 110 | @app.route('/about') 111 | def about(): 112 | return 'The about page' 113 | 114 | ###Rendering Templates 115 | 116 | ####Example1: 117 | Flask configures the Jinja2 template engine for you automatically
118 | we can use render_template method to render htmls. 119 | 120 | 121 | from flask import Flask, render_template 122 | app = Flask(__name__) 123 | 124 | @app.route('/hello/') 125 | @app.route('/hello/') 126 | def hello(name=None): 127 | return render_template('hello.html', name=name) 128 | 129 | 130 | hello.html 131 | 132 | 133 | Hello from Flask 134 | {% if name %} 135 | Hello {{name}} 136 | {% else %} 137 | Hello, World! 138 | {% endif %} 139 | 140 | 141 | ####Example2: 142 | Web applications use different HTTP methods when accessing URLs. You should familiarize yourself with the HTTP methods as you work with Flask. By default, a route only answers to GET requests. You can use the methods argument of the route() decorator to handle different HTTP methods. 143 | 144 | from flask import Flask, render_template 145 | app = Flask(__name__) 146 | 147 | @app.route('/login', methods=['GET', 'POST']) 148 | def login(): 149 | error = None 150 | if request.method == 'POST': 151 | if request.form['username'] != 'admin' or request.form['password'] != 'admin': 152 | error = 'Invalid Credentials. Please try again.' 153 | else: 154 | return redirect(url_for('home')) 155 | return render_template('login.html', error=error) 156 | @app.route('/home') 157 | def home(): 158 | render_template('home.html') 159 | 160 | login.html 161 | 162 | 163 | 164 | Flask Intro - login page 165 | 166 | 167 | 168 | 169 |
170 |

Please login

171 |
172 |
173 | 175 | 177 | 178 |
179 | {% if error %} 180 |

Error: {{ error }} 181 | {% endif %} 182 |

183 | 184 | 185 | 186 | home.html 187 | 188 | 189 | 190 | Home page 191 | 192 | 193 | 194 | 195 |
196 |

This is home

197 |
198 | 199 | 200 | 201 | 202 | 203 | 204 | ###Redirects: 205 | 206 | from flask import abort, redirect, url_for 207 | app = Flask(__name__) 208 | 209 | @app.route('/') 210 | def index(): 211 | return redirect(url_for('login')) 212 | 213 | @app.route('/login') 214 | def login(): 215 | return 'Hey.. You just landed on log in page' 216 | 217 | 218 | 219 | ###Errors: 220 | 221 | from flask import Flask, redirect, redirect 222 | app = Flask(__name__) 223 | 224 | @app.route('/') 225 | def index(): 226 | return redirect(redirect('login')) 227 | 228 | @app.route('/login') 229 | def login(): 230 | return 'Hey.. You just landed on log in page' 231 | 232 | @app.errorhandler(404) 233 | def page_not_found(error): 234 | return 'Hey.. you landed on mising page..
  • /
  • /login
' 235 | 236 | 237 | 238 | ###APIs with JSON: 239 | 240 | from flask import Flask, redirect, redirect 241 | app = Flask(__name__) 242 | @app.route("/data") 243 | def data(): 244 | user = get_current_user() 245 | return { 246 | "username": user["name"], 247 | "age": user['age'] 248 | } 249 | 250 | def get_current_user(): 251 | return {'name':'ram', 'age':20} 252 | 253 | ###Cookies 254 | 255 | setting cookies: 256 | 257 | from flask import Flas, make_response 258 | app = Flask(__name__) 259 | 260 | @app.route('/set_cookie') 261 | def set_cookie(): 262 | resp = make_response('hello world') 263 | username = request.args.get('username') 264 | resp.set_cookie('username', username) 265 | return resp 266 | 267 | reading cookies 268 | 269 | from flask import Flask, request 270 | app = Flask(__name__) 271 | 272 | @app.route('/get_cookie') 273 | def read_cookie(): 274 | username = request.cookies.get('username') 275 | return 'hello %'%username 276 | 277 | 278 | ###Sessions: 279 | from flask import Flask, session, redirect, url_for, escape, request 280 | app = Flask(__name__) 281 | 282 | # Set the secret key to some random bytes. Keep this really secret! 283 | app.secret_key = b'_5#y2L"F4Q8z\n\xec]/' 284 | 285 | @app.route('/') 286 | def index(): 287 | if 'username' in session: 288 | return 'Logged in as %s' % escape(session['username']) 289 | return 'You are not logged in' 290 | 291 | @app.route('/login', methods=['GET', 'POST']) 292 | def login(): 293 | if request.method == 'POST': 294 | session['username'] = request.form['username'] 295 | return redirect(url_for('index')) 296 | return ''' 297 |
298 |

299 |

300 |

301 | ''' 302 | 303 | @app.route('/logout') 304 | def logout(): 305 | # remove the username from the session if it's there 306 | session.pop('username', None) 307 | return redirect(url_for('index')) -------------------------------------------------------------------------------- /docs/flaskandorm.md: -------------------------------------------------------------------------------- 1 | # ORM 2 | 3 | ## What are orm? 4 | 5 | A simple answer is that you wrap your tables or stored procedures in classes in your programming language, so that instead of writing SQL statements to interact with your database, you use methods and properties of objects.[stackoverflow] 6 | 7 | ## SQL Alchemy in flask 8 | 9 |
 10 | from flask import Flask
 11 | from flask_sqlalchemy import SQLAlchemy
 12 | 
 13 | app = Flask(__name__)
 14 | app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/test.db'
 15 | db = SQLAlchemy(app)
 16 | 
 17 | 
 18 | class User(db.Model):
 19 |     id = db.Column(db.Integer, primary_key=True)
 20 |     username = db.Column(db.String(80), unique=True, nullable=False)
 21 |     email = db.Column(db.String(120), unique=True, nullable=False)
 22 | 
 23 |     def __repr__(self):
 24 |         return '' % self.username
 25 | 
 26 | 
 27 | admin = User(username="test1asd",email="aasdadsdadsad")
 28 | db.session.add(admin)
 29 | db.session.commit()
 30 | 
 31 | print(User.query.all())
 32 | 
33 | 34 | Lets see what is happening above. 35 | First two line from import flask and sql alchemy 36 | 37 | Next we created application and defined app.config for sql alchemy. It has to be this variable name only `app.config['SQLALCHEMY_DATABASE_URI']`. Then we intialized db instance. 38 | 39 | Next we have created User class. 40 | 41 | Then we created an instance of User and added it to the session and then commited it. 42 | 43 | This is how data is saved. 44 | 45 | Next we can query it `using the User.query.all()` . 46 | 47 | 48 | 49 | ## Flask Migrations 50 | 51 | We will use flask-migrate for keeping track of migrations. It uses alembic behind the scenes for migrations. 52 | 53 | ### Installation 54 | 55 |
 56 | pip install flask-migrate
 57 | 
58 | 59 | ### Code 60 | 61 | Have a look at the below code 62 | 63 |
 64 | from flask import Flask
 65 | from flask_sqlalchemy import SQLAlchemy
 66 | from flask_migrate import Migrate
 67 | 
 68 | app = Flask(__name__)
 69 | app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/test.db'
 70 | db = SQLAlchemy(app)
 71 | migrate = Migrate(app, db)
 72 | 
 73 | class User(db.Model):
 74 |     id = db.Column(db.Integer, primary_key=True)
 75 |     username = db.Column(db.String(80), unique=True, nullable=False)
 76 |     email = db.Column(db.String(120), unique=True, nullable=False)
 77 |     password = db.Column(db.String(120), unique=True, nullable=True)
 78 |     def __repr__(self):
 79 |         return '' % self.username
 80 | 
81 | 82 | As you can see there are 2 new line. Importing flask_migrate and initializing migrate. Thats it you have to do for now. 83 | 84 | ### How to create migration. 85 | 86 | ##### Initializing 87 | 88 |
flask db init
89 | 90 | It will generate migrations folder. 91 | 92 | ##### Generating Migration 93 | 94 | After making code changes in model. You can run the below command to generate migrtion 95 | 96 |
flask db migrate
97 | 98 | Now there will be migrations generated and you can see those in migrations/versions folder. 99 | 100 | ##### Applying migration 101 | 102 |
flask db upgrade
103 | 104 | ##### Downgrade Migration 105 | 106 |
flask db downgrade version
107 | 108 | These are very basic of how you can keep track of your database migrations. You can read more about it. There are lot of complexities involve when it comes to migration with databases. So read about it properly before using it in production. -------------------------------------------------------------------------------- /docs/flaskdb.md: -------------------------------------------------------------------------------- 1 | # Python with database[MYSQL]. 2 | 3 | We have seen how to write flask applications that can serve basic requests. The next thing that we need to learn is how to save states in application. Till now we have only seen stateless applications. 4 | 5 | We will have a look at how we can connect to mysql using python mysql connectors. 6 | 7 |
 8 | 
 9 | import mysql.connector
10 | 
11 | mydb = mysql.connector.connect(
12 |   host="localhost",
13 |   user="yourusername",
14 |   passwd="yourpassword"
15 | )
16 | 
17 | print(mydb)
18 | 
19 | 20 | This will print the connection that is created. 21 | 22 | Let try and create a table and insert some values in it. 23 | 24 |
25 | import mysql.connector
26 | 
27 | mydb = mysql.connector.connect(
28 |   host="localhost",
29 |   user="yourusername",
30 |   passwd="yourpassword",
31 |   database="testapp"
32 | )
33 | 
34 | print(mydb)
35 | mycursor = mydb.cursor()
36 | mycursor.execute("CREATE TABLE customers (name VARCHAR(255), address VARCHAR(255))")
37 | 
38 | 39 | The above code will create table customers in testapp database. 40 | 41 | Lets insert values in this database. 42 | 43 |
44 | import mysql.connector
45 |   
46 | mydb = mysql.connector.connect(
47 |   host="localhost",
48 |   user="chowmean",
49 |   passwd="chowmean",
50 |   database="testapp"
51 | )
52 | 
53 | print(mydb)
54 | mycursor = mydb.cursor()
55 | sql = "INSERT INTO customers (name, address) VALUES (%s, %s)"
56 | val = ("John", "Highway 21")
57 | mycursor.execute(sql, val)
58 | mydb.commit()
59 | 
60 | print(mycursor.rowcount, "record inserted.")
61 | 
62 | 63 | This will insert a row in table. 64 | 65 | Lets see how we can see the entry that is created right now. 66 | 67 |
68 | import mysql.connector
69 |   
70 | mydb = mysql.connector.connect(
71 |   host="localhost",
72 |   user="chowmean",
73 |   passwd="chowmean",
74 |   database="testapp"
75 | )
76 | 
77 | mycursor = mydb.cursor()
78 | 
79 | mycursor.execute("SELECT * FROM customers")
80 | 
81 | myresult = mycursor.fetchall()
82 | 
83 | for x in myresult:
84 |   print(x)
85 | 
86 | 87 | This will print the data that is extracted from mysql. -------------------------------------------------------------------------------- /docs/generators.md: -------------------------------------------------------------------------------- 1 | # Generators 2 | 3 | Generators are just like normal functions, what makes functions generators in yield statement instead of return statement. 4 | 5 | ## What is yield? 6 | 7 | Yield is a key word that is used to return the value only, the difference is it is used to induce the lazy processing capability. What is does is it returns the state of the functions at that time and save it locally so that it can be resumed from there in the next iteration. 8 | So it actually doesn't generate any value but save the function state so that it can be generated on the run time. Have a look at the code below 9 | 10 |
11 | def testgen():
12 |     for i in range(10000):
13 |         yield i
14 | 
15 | 16 | So what is happening here is we have created a generator using yeild. Yeild creates an itertor by itself so it works out of the box from with `for-in` 17 | 18 | In the above function we have created a generator that is generating list from 0,9999. 19 | 20 | 21 | ## Advantages of generators 22 | 23 | Generator have following advantages 24 | - They don't process at the same time but does lazy loading so when the value is required it gets executed and gets the data. 25 | - They help a lot in saving memory by not generating and saving things in memory all at once. 26 | 27 | 28 | ### Memory Efficient and lazy processing. 29 | Lets have a look at the code below. 30 | 31 |
32 | from sys import getsizeof
33 | 
34 | def tesgen():
35 |         for i in range(10000):
36 |                 yield i
37 | 
38 | b = [x for x in range(10000)]
39 | print(getsizeof(b))
40 | 
41 | a = tesgen()
42 | print(getsizeof(a))
43 | 
44 | 45 | If you can run this program you can see the difference between memory consumption in both the cases. Lets discuss why there is so much difference in memroy consumption here. 46 | 47 | Memory difference is because there is no acutal calculation and storing of the values in the memory. 48 | 49 | ### Infinite Sequence of data 50 | 51 | It is easy to have an infinite sequence of data with generators because of the memory advantage otherwise for infite sequence of data you need infinite RAM. 52 | 53 | 54 | ### Iterator by default 55 | 56 | Generators automatically define `__iter__` and `__next__`. Thus these can be used directly with `for-in` and has iterators. You can get the iterators using `iter` key word. 57 | 58 | ## Python generator experssion 59 | 60 |
61 | a = (x**2 for x in my_list)
62 | 
63 | Above code creates generator, what you need to do this is use `()` with your normal expression. 64 | 65 | This was introduction to generators. These are very powerful and you can use them to accomplish different tasks. Now its upto you how you will dig deeper and use it. 66 | -------------------------------------------------------------------------------- /docs/gil.md: -------------------------------------------------------------------------------- 1 | # GIL: Global Interpreter Lock 2 | 3 | Gil is a mutex lock in python which allows only one thread to execute at a time. This is why python is single threaded application. So if your program is simgle threaded python will perform as equal to any other language. But when it comes to multithreading and executing threads in parallel, it is not possible in python 4 | 5 | ## Why GIL? 6 | 7 | Python uses reference counting for memory management. What it does is it keeps track of all the variables that are created and wheather they are being used in the program now or not. If their is no reference for those variables. The python garbage collector kicks in and removes that variable from the heap. 8 | 9 | Now for this reference counting to work properly it need to be safe from race conditions. That is two threads try to change the reference count. If that happens that can lead to memory leak. This reference count can be kept safe by locking all the data structures that are shared accross the threads. 10 | 11 | Now if we lock these data structures. It means multiple locks will exist which can lead to problems like deadlock. Also it will decrease performance as aquisition and release of locks will happen quite frequently. 12 | 13 | GIL is a single lock which says that execution of any python byte code requires the interpreter lock. This prevents the above problem but makes the python single threaded. 14 | 15 | ## Why only GIL as solution? 16 | 17 | Python has been around since the days when operating systems did not have a concept of threads. Python was designed to be easy-to-use in order to make development quicker and more and more developers started using it. 18 | 19 | A lot of extensions were being written for the existing C libraries whose features were needed in Python. To prevent inconsistent changes, these C extensions required a thread-safe memory management which the GIL provided. 20 | 21 | The GIL is simple to implement and was easily added to Python. It provides a performance increase to single-threaded programs as only one lock needs to be managed. 22 | 23 | C libraries that were not thread-safe became easier to integrate. And these C extensions became one of the reasons why Python was readily adopted by different communities. 24 | 25 | 26 | ## Impacts? 27 | 28 | There are two kinds of multithreaded programs. 29 | 30 | * I/O bound program : Programs that wait for input/output from user, file, database, network etc. These sometimes have to wait alot. 31 | * CPU bound program : These programs are the one which does high computational tasks. 32 | 33 | We will see how python works in both these cases. 34 | 35 | In case of CPU bound programs the multithreading will not make any difference as the only one thread will execute. Infact you can see performance degrading for these cases. 36 | 37 | In case of IO bound process the multithreading will increase performance as IO bound process mostly wait for the input or output to be provided to them and they are in waiting state. Aquiring and releasing lock for these kind of process doesn't cause problem cause when one thread may got the output or input the others may still be waiting for the event. 38 | 39 | 40 | ## Work around? 41 | 42 | If you want to make use of all your core in pytho program instead of going for multithreading go for multiprocessing. Each process gets its own python interpretor this the GIL is applied on only that process and hence you can do lot of work in parallel. 43 | 44 | Read more about GIL [here](https://realpython.com/python-gil/) -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Python Training for devops 2 | 3 | Python is a well known programming language which is used for scripting, data science, software and system development. It is one of the most used language of these days. 4 | 5 | This training course is for devops people who want to use python in their projects and later use it in a full fledge development. 6 | 7 | In this course we will talk about OOPS, CLI applications in python, frameworks in python, GIL, memory management, basic tooling around python, etc. -------------------------------------------------------------------------------- /docs/internals.md: -------------------------------------------------------------------------------- 1 | #Python Internals 2 | ####Compiler: 3 | Definition: A compiler is a computer program that transforms (translates) source code of a programming language (the source language) into another computer language (the target language). In most cases compilers are used to transform source code into executable program, i.e. they translate code from high-level programming languages into low (or lower) level languages, mostly assembly or machine code. 4 | 5 | Ex:
6 | 1. Java
7 | 2. C++
8 | 3. C#
9 | 10 | #### Interpreter 11 | Definition: An interpreter is a computer program that executes instructions written in a programming language. Translation occurs at the same time as the program is being executed. 12 |
It can either 13 | 14 | 15 | 1. execute the source code directly or 16 | 2. translate the source code in a first step into a more efficient representation and execute this code 17 | 18 | Ex:
19 | 1. java script
20 | 2. Python 21 | ###How Python runs? 22 | Have you ever thought how the Python code is actually executed by the Python interpreter? What steps are carried out to generate the final output of your Python script? This article answers all these questions in a simplistic manner! 23 |
24 | 25 | ![Interprer Interpreter](https://i.imgur.com/iQYveIP.png) 26 | 27 | ####The interpreter 28 | Interpreter is nothing but a software which can run your Python scripts. 29 | 30 | Interestingly, it can be implemented in any programming language! 31 | 32 | 1. CPython is the default interpreter for Python which is written in C programming language. 33 | 2. Jython is another popular implementation of python interpreter written using Java programming language. 34 | 35 | ####Programmer’s view of interpreter 36 | If you have been coding in Python for sometime, you must have heard about the interpreter at least a few times.
37 | From a programmer’s perspective, an interpreter is simply a software which executes the source code line by line. 38 | 39 | ![Interprer Interpreter](https://i.imgur.com/c0PRvvI.png) 40 | 41 |
For most of the Python programmers, an interpreter is like a black box.
42 | 43 | ####Python’s view of interpreter 44 | Now, let us scan through the python interpreter and try to understand how it works. 45 | 46 | Have a look at the diagram shown below: 47 | ![Interprer Interpreter](https://i.imgur.com/PJME67T.png) 48 | 49 | I hope you didn’t get amazed to see a compiler inside an interpreter! 50 | 51 | From the figure above, it can be inferred that interpreter is made up of two parts: 52 | 53 | 1. compiler 54 | 2. virtual machine
55 | 56 | What does compiler do? 57 | 58 | Compiler compiles your source code (the statements in your file) into a format known as byte code. Compilation is simply a translation step!
59 | 60 | Byte code is a: 61 | 62 | 1. lower level, 63 | 2. platform independent, 64 | 3. efficient and 65 | 4. intermediate
66 | representation of your source code! 67 | 68 | Roughly, each of your source statements is translated into a group of byte code instructions. 69 | 70 | The process of compilation in CPython interpreter’s compiler can be divided into 4 main parts: 71 | 72 | * **Parse source code into a parse tree:**
73 | Based on grammar rules of Python programming language, the source code is converted to a parse tree. Every node of the parse tree contains a part of your code.
74 | 75 | Consider a simple arithmetic expression: 76 |
 77 |     14 + 2 * 3 - 6 / 2
 78 |     
79 | The parse tree for above expression looks like this: 80 | ![Interprer Interpreter](https://indianpythonista.files.wordpress.com/2018/01/parsetree.png) 81 | 82 | * **Transform parse tree into an Abstract Syntax Tree:**
83 | The abstract syntax tree (AST) is a high-level representation of the program structure. 84 | Each node of the tree denotes a construct occurring in the source code. The syntax is “abstract” in not representing every detail appearing in the real syntax.Consider the AST shown below for the parse tree example discussed above: 85 | ![Interprer Interpreter](https://indianpythonista.files.wordpress.com/2018/01/ast.png) 86 | * **Transform AST into a Control Flow Graph:**
87 | A control flow graph is a directed graph that models the flow of a program using basic blocks. Each block contains the bytecode representation of program code inside it. 88 | * **Byte Code generation from CFG:**
89 | CFGs are usually one step away from final code output. Code is directly generated from the basic blocks by doing a post-order depth-first search on the CFG following the edges. 90 | 91 | This byte code translation is performed to speed up the execution—byte code can be run much quicker than the original source code statements. 92 | 93 | ###What does Virtual Machine do? 94 | As soon as source code gets converted to byte code, it is fed into PVM (Python Virtual Machine).
95 | It’s just a big loop that iterates through your byte code instructions, one by one, to carry out their operations. **The PVM is the runtime engine of Python**; it’s always present as part of the Python system, and is the component that truly runs your scripts. Technically, it’s just the last step of what is called the Python interpreter. 96 | 97 | ####Lastly, here are a interesting points: 98 | 99 | 1. PyPy is an implementation of Python which does not use an interpreter! It is implemented using something called just-in-time compiler! 100 | Interestingly, it often runs faster than the standard implementation of Python, CPython. 101 | 2. Whenever a Python script is executed, the byte code is generated in memory and simply discarded when program exits. 102 | 3. But, if a Python module is imported, a .pyc file for the module is generated which contains its Byte code. 103 | Thus, when the module is imported next time, the byte code from .pyc file is used, hence skipping the compilation step! 104 | 105 | ###see compiled files: 106 |
107 |   python -m py_compile hello.py
108 | 
109 | 110 | ###Question: 111 | when we update source code of imported module, how does interpreter picks up that updated code? -------------------------------------------------------------------------------- /docs/iterators.md: -------------------------------------------------------------------------------- 1 | # Iterators 2 | 3 | Generally, iterators are used to iterate over a list of objects. We use iterators without knowing we are using it. Look at the below example 4 | 5 |
  6 | a = [1,2,4,5,6,7,8]
  7 | for i in a:
  8 | 	print(i)
  9 | 
10 | 11 | Here we have used iterator behind the scenes of `for-in` to loop over a list. 12 | 13 | ## How to create Iterators? 14 | 15 | Iterators consists of two important functions 16 | 17 |
 18 | __iter__
 19 | 
20 | 21 |
 22 | __next__
 23 | 
24 | 25 | If your class has these two functions written properly it can be used as iterator and can be used with `for-in`. 26 | 27 | ## Iterator Visualization 28 | 29 | Lets take a list and try to iterate over it with iterator. 30 | 31 |
 32 | a = [1,2,3,4,5,6,7]
 33 | it = iter(a)  #same as it = a.__iter__()
 34 | print(it.__next__())
 35 | 
36 | 37 | Everytime you print if will print the next value in list till it reaches stop iteration exception. Now lets try and create iterator classes. 38 | 39 | 40 | ## Basic Iterator Example. 41 | 42 |
 43 | class Iterator:
 44 |     def __init__(self, value):
 45 |         self.value = value
 46 | 
 47 |     def __iter__(self):
 48 |         return self
 49 | 
 50 |     def __next__(self):
 51 |         return self.value
 52 | 
 53 | a = Iterator("asdad")
 54 | 
 55 | for item in a: 
 56 | 	print(item)
 57 | 
58 | 59 | 60 | Here what we are doing is creating and object of Iterator class which has iter which returns itself and and next which returns the values. This is then used in `for-in` to demonstrate iterator working. 61 | 62 | This iterator will keep on printing whatever value you will provide it forever. It will not break. Since in `__next__` function we are just returning the value itself and not chagning anything else. Ideally it should change some value which defines when this iterator will stop. 63 | 64 | Let us now try and write an iterator which will loop over out list of numbers and print it. 65 | 66 | ## Creating List iterators 67 | 68 |
 69 | class NumberIterator:
 70 |     def __init__(self, value):
 71 |         self.value = value
 72 |         self.index = 0
 73 |         self.length = len(value)
 74 | 
 75 |     def __iter__(self):
 76 |         return self
 77 | 
 78 |     def __next__(self):
 79 |         
 80 |         if self.index= self.length:
 85 |             raise StopIteration
 86 | 
 87 | a=[1,2,3,4,5,6,7,8,9]
 88 | a=NumberIterator(a)
 89 | for item in a:
 90 | 	print(item)
 91 | 
92 | 93 | As you can see now in `__next__` function we are checking if we reached will the end of the len or not if yes we are stopping the iteration. 94 | 95 | This is how iterator for list may be implemented. 96 | 97 | Now lets try and see if manully getting an iterator will work with our class? 98 | 99 |
100 | a=[1,2,3,4,5,6,7,8,9]
101 | a=RepeaterNumber(a)
102 | it = a.__iter__()
103 | print(it.__next__())
104 | print(it.__next__())
105 | print(it.__next__())
106 | 
107 | 108 | If it works we have learned the basic of how iterator works. 109 | Iterate if you still have problems. :) -------------------------------------------------------------------------------- /docs/logging.md: -------------------------------------------------------------------------------- 1 | # Logging 2 | 3 | In this section we will have a look how we can do logging in python applications. We will also have a look what we should and and what we should not log. 4 | Logging is important but extra logging can cause a lot of issue. Issues like latency, extra disk space usage etc. 5 | 6 | For logging in python we use logging module of python. Have a look at the code below 7 | 8 |
  9 | 
 10 | import logging
 11 | 
 12 | logging.debug('This is a debug')
 13 | logging.info('This is an info')
 14 | logging.warning('This is a warning')
 15 | logging.error('This is an error')
 16 | logging.critical('This is a critical')
 17 | 
18 | 19 | It is very basic to do logging in python. 20 | 21 | ### Configurations 22 | 23 | Below are few of the basic configs that will be useful for you. 24 | 25 | level: Sets the log level 26 | 27 | filename: This specifies the file. 28 | 29 | filemode: If filename is given, the file is opened in this mode. The default is a, which means append. 30 | 31 | format: This is the format of the log message. 32 | 33 | Lets use this in our applications 34 | 35 |
 36 | import click
 37 | import requests
 38 | import json
 39 | import logging
 40 | logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', filename='app.logs', level=logging.DEBUG)
 41 | 
 42 | @click.command()
 43 | @click.option('--username', prompt='Github username',
 44 |                 help='Github username to get repo details.')
 45 | 
 46 | def get_projects(username):
 47 | 	logging.debug("Contacting github for repos.")
 48 | 	a = requests.get("https://api.github.com/users/"+username+"/repos")
 49 | 	logging.debug("Github data received.")
 50 |     data = json.loads(a.content)
 51 |     return data
 52 | 
 53 | if __name__ == '__main__':
 54 | 	logging.debug("Initilizing application.")
 55 |     get_projects()
 56 | 
57 | 58 | Now when you will run this you can see the logs in app.logs file. Now if you want these logging to be configurable you can simply get these as options like below. 59 | 60 |
 61 | import click
 62 | import requests
 63 | import json
 64 | import logging
 65 | 
 66 | log_levels={
 67 |     "warn": logging.WARNING,
 68 |     "info": logging.INFO,
 69 |     "error": logging.ERROR,
 70 |     "debug": logging.DEBUG,
 71 |     "critical": logging.CRITICAL
 72 | }
 73 | 
 74 | 
 75 | 
 76 | @click.command()
 77 | @click.option('--username', prompt='Github username', help='Github username to get repo details.')
 78 | @click.option('--logfile', prompt='Log file name', help='File to which log', default="app.logs")
 79 | @click.option('--loglevel', prompt='Log level', help='Log level to use.', default="warn")
 80 | def get_projects(username, logfile, loglevel):
 81 |     logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', filename=logfile, level=log_levels[loglevel])
 82 |     logging.debug("Contacting github for repos.")
 83 |     a = requests.get("https://api.github.com/users/"+username+"/repos")
 84 |     logging.debug("Github data received.")
 85 |     data = json.loads(a.content)
 86 |     return data
 87 | 
 88 | if __name__ == '__main__':
 89 |     get_projects()
 90 | 
91 | 92 | #Rotating Logs: 93 | Python’s logging module has lots of options. In this article, we will looks at the logging module’s ability to create Rotating Logs. Python supports two types of rotating logs: 94 | 1) Rotate logs based on size (RotatingFileHandler) 95 | 2) Rotate the logs based on some time interval (TimedRotatingFileHandler) 96 | 97 | ###The RotatingFileHandler: 98 | It rotates the logs based on the size of the log file. 99 | We can specify the size with "maxBytes" parameter. 100 | 101 | The handler will close the file and silently open a new one. If you pass in a number for the backupCount parameter, then it will append “.1”, “.2”, etcetera to the end of the log files. Let’s take a look at a simple example: 102 |
103 | import logging
104 | import time
105 |  
106 | from logging.handlers import RotatingFileHandler
107 | def create_rotating_log(path):
108 |     """
109 |     Creates a rotating log based on size 
110 |     """
111 |     logger = logging.getLogger("Rotating Log")
112 |     logger.setLevel(logging.INFO)
113 |     handler = RotatingFileHandler(path, maxBytes=20, backupCount=5)
114 |     logger.addHandler(handler) 
115 |     for i in range(10):
116 |         logger.info("This is test log line %s" % i)
117 |         time.sleep(1.5)
118 |  
119 | if \__name__ == "\__main__":
120 |     log_file = "test.log"
121 |     create_rotating_log(log_file)
122 | 
123 | 124 | ####Understanding above code in steps: 125 | 1. we create a log file with a logging level of INFO. 126 | 2. Then we set up the handler to rotate the log whenever the log file is 20 bytes in length 127 | 3. we create a loop that will create 10 lines in our log file with a sleep in between each call to log. 128 | 129 | ###The TimedRotatingFileHandler: 130 | The TimedRotatingFileHandler allows the developer to create a rotating log based on how much time has elapsed. 131 | You can set it to rotate the log on the following time conditions: 132 | 133 | 1. second (s) 134 | 2. minute (m) 135 | 3. hour (h) 136 | 4. day (d) 137 | 5. w0-w6 (weekday, 0=Monday) 138 | 6. midnight 139 | 140 | ###Example: 141 |
142 | import logging
143 | import time
144 |  
145 | from logging.handlers import TimedRotatingFileHandler
146 |  
147 | def create_timed_rotating_log(path):
148 |     logger = logging.getLogger("Rotating Log")
149 |     logger.setLevel(logging.INFO)
150 |     handler = TimedRotatingFileHandler(path,
151 |                                        when="m",
152 |                                        interval=1,
153 |                                        backupCount=5)
154 |     logger.addHandler(handler)
155 |     for i in range(6):
156 |         logger.info("This is a test!")
157 |         time.sleep(75)
158 |  
159 | if \__name__ == "\__main__":
160 |     log_file = "timed_test.log"
161 |     create_timed_rotating_log(log_file)
162 | 
163 | 164 | 165 | 166 | -------------------------------------------------------------------------------- /docs/mem.md: -------------------------------------------------------------------------------- 1 | # Memory Management 2 | 3 | This concept deals with the way python manages memory, how it assigns memory, how it cleans memory and garbage collect. 4 | 5 | An OS-specific virtual memory manager carves out a chunk of memory for the Python process. 6 | 7 | Python uses a portion of the memory for internal use and non-object memory. The other portion is dedicated to object storage (your int, dict, and the like). 8 | 9 | ## Heaps and Stack 10 | 11 | These are the two major datastructure that is being used by any programming language for running the programs. 12 | 13 | ### Heaps 14 | 15 | Heaps are used for stroing data and objects in memory that are being referenced from stacks. 16 | 17 | ### Stacks 18 | 19 | These are used for mainly function calls and variables in the a particular scope. 20 | 21 | ## Garbage Collection 22 | 23 | Python interpretor keeps the reference count of all the variables, if reference count of any variable becomes 0. That variable is garbage collected. 24 | 25 | If you want to read more about it read here 26 | [memory management in python](https://realpython.com/python-memory-management/) -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Exporting Metrics 2 | 3 | Metrics are very important to keep track of how your software is working. This will help you in keeping check on any unwanted things that went to production and impacted it. 4 | One of the most important tools for metrics is prometheus that is very widely used by companies. We will try expose metrics for prometheus using python prometheus client. 5 | 6 | ### Installation 7 | pip install prometheus_client 8 | 9 | ### Code 10 | 11 |
12 | from prometheus_client import start_http_server, Summary
13 | import random
14 | import time
15 | 
16 | # Create a metric to track time spent and requests made.
17 | REQUEST_TIME = Summary('request_processing_seconds', 'Time spent processing request')
18 | 
19 | # Decorate function with metric.
20 | @REQUEST_TIME.time()
21 | def process_request(t):
22 |     """A dummy function that takes some time."""
23 |     time.sleep(t)
24 | 
25 | if __name__ == '__main__':
26 |     # Start up the server to expose the metrics.
27 |     start_http_server(9000)
28 |     # Generate some requests.
29 |     while True:
30 |         process_request(random.random())
31 | 
32 | 33 | This will run a server on port 9000 and export metrics that prometheus will be able to consume. 34 | 35 | ### Prometheus exporter with flask. 36 | 37 | You can do the same with you application written in flask using different client present for flask. https://pypi.org/project/prometheus-flask-exporter/ 38 | 39 | Read about it create an application with metrics. Ping me if you are stuck. -------------------------------------------------------------------------------- /docs/oops.md: -------------------------------------------------------------------------------- 1 | #Concept of Object Oriented Programming 2 | 3 | 4 | ### Objects 5 | 6 | Everything in python are objects. Even if you initialize a variable with 1. 1 is a object of type int. 7 | 8 | ### Classes 9 | 10 | Classes are combination of variables and functions. Classes define a type of any object. An example of class in below 11 | 12 |
 13 | class employee():	
 14 |   
 15 |   __phone_number=""
 16 | 	
 17 | 	def __init__(name, age):
 18 | 		self.name = name
 19 | 		self.age = age
 20 | 	
 21 | 	def get_age():
 22 | 		return self.age
 23 | 	
 24 | 	def set_age(value):
 25 | 		self.age = value
 26 | 	
 27 | 	def get_name():
 28 | 		return self.name
 29 | 	
 30 | 	def set_phone(value):
 31 | 		self.__phone_number = value
 32 | 
 33 | 
34 | 35 | This is an example of class, Here we have private variable __phone_number, which cannot be ediited by anyone else but functions inside the class. __init__() is a constructor which is used to initialize the class. 36 | 37 | ### Inheritance 38 | The capability of a class to derive properties and characteristics from another class is called Inheritance. Inheritance is one of the most important feature of Object Oriented Programming. 39 | Sub Class: The class that inherits properties from another class is called Sub class or Derived Class. 40 | Super Class: The class whose properties are inherited by sub class is called Base Class or Super class. 41 | 42 |
 43 | 
 44 | class Employee():	
 45 |   
 46 | 	__phone_number="0000000000"
 47 | 	
 48 | 	def __init__(self, name, age):
 49 | 		self.name = name
 50 | 		self.age = age
 51 | 	
 52 | 	def get_age(self):
 53 | 		return self.age
 54 | 	
 55 | 	def set_age(self,value):
 56 | 		self.age = value
 57 | 	
 58 | 	def get_name(self):
 59 | 		return self.name
 60 | 	
 61 | 	def set_phone(self,value):
 62 | 		self.__phone_number = value
 63 | 
 64 | 	def get_phone(self):
 65 | 		return self.__phone_number 
 66 | 
 67 | 	def set_phone(self, value):
 68 | 		self.__phone_number = value
 69 | 
 70 | class VIP(Employee):
 71 | 
 72 | 	def get_phone(self):
 73 | 		return "************"
 74 | 
 75 | 	def get_vip_access(self):
 76 | 		return "access_granted"
 77 | 
 78 | if __name__ == '__main__':
 79 | 
 80 | 	#Simple Employee 
 81 | 	emp = Employee("gaurav", 12)
 82 | 	print(emp.get_phone())
 83 | 	emp.set_phone("89628363**")
 84 | 	print(emp.get_phone())
 85 | 
 86 | 	#Vip employee inherting employee properties
 87 | 	vip = VIP("Venky", 54)
 88 | 
 89 | 	#Overriden method hides phone number
 90 | 	print(vip.get_phone())
 91 | 
 92 | 	#Extra added function. 
 93 | 	print(vip.get_vip_access())
 94 | 
 95 | 	#This function calls the inherited method.
 96 | 	print(vip.get_name())
 97 | 	
 98 | 
 99 | 
100 | 101 | ### Encapsulation 102 | 103 | Encapsulation is a way to hide some of the variables of the class to be not accessible from outside the class. In other languages there are class members like private and protected but in python there is no such concept. Instead you can define a private class member like this 104 | 105 |
106 | 
107 | class Employee():	
108 |   
109 | 	__phone_number="0000000000"
110 | 
111 | 112 | Here __phone_number is a private class member and cannot be accessible from outside the class. You cannot access this variable like below. 113 | 114 |
115 | 
116 | a = Employee()
117 | a.__phone_number
118 | 
119 | 120 | To access these variables you have to write getter and setter function for these. Which can do the basic checks before setting the variables. 121 | 122 |
123 | 
124 | class Employee():	
125 |   
126 | 	__phone_number="0000000000"
127 | 
128 | 	def get_phone(self):
129 | 		return self.__phone_number
130 | 
131 | 	def set_phone(self, value):
132 | 		self.__phone_number = value
133 | 
134 | 
135 | #### Use of Encapsulation? 136 | 137 | Say you have a class User where you have age and password class members. Here you will not want anyone outside the class to see password. Also you want to apply some check before setting the password. This can be done using this. Look at the code below. 138 | 139 |
140 | 
141 | class User():
142 | 	
143 | 	__password=""
144 | 
145 | 	def __init__(self, name, age):
146 | 		self.name = name
147 | 		self.age = age
148 | 
149 | 	def set_password(password):
150 | 		if password_check(password):
151 | 			self.__password=password
152 | 
153 | 154 | 155 | ### Polymorphism 156 | 157 | Polymorphism means having many forms. In programming it means that same function names can be used for different types of work. 158 | 159 |
160 | def add(x, y, z = 0):  
161 |     return x + y+z 
162 |   
163 | # Driver code  
164 | print(add(2, 3)) 
165 | print(add(2, 3, 4)) 
166 | 
167 | 
168 | Here function add has two forms. One is when it takes two arguements where the third one itself is 0. Other is when it takes 3 variables. This is very basic example of polymorphism -------------------------------------------------------------------------------- /docs/package.md: -------------------------------------------------------------------------------- 1 | # Creating a python package. 2 | 3 | For creating a python file you need below steps. 4 | 5 | ### Application Structure 6 | Lets say our app structure is like below 7 | 8 |
 9 | 
10 | firstapp
11 | 	--pack.py
12 | 	--__init__.py
13 | setup.py
14 | README.md
15 | 
16 | 17 | Lets say pack.py contains below code 18 | 19 |
20 | def say_hello():
21 | 	print("hello")
22 | 
23 | 24 | ### Setup file 25 | 26 | You need to create a setup.py file which will contain information about the package. 27 | 28 | 29 |
30 | import setuptools
31 | with open("README.md", "r") as fh:
32 |     long_description = fh.read()
33 | setuptools.setup(
34 |      name='firstapp',  
35 |      version='0.1',
36 |      scripts=['firstapp/pack.py'] ,
37 |      author="chowmean",
38 |      author_email="gaurav.dev.iiitm@gmail.com",
39 |      description="Test package creation package",
40 |      long_description="long_description",
41 |      long_description_content_type="text/markdown",
42 |      url="https://gitlab.com/chowmean",
43 |      packages=setuptools.find_packages(),
44 |      classifiers=[
45 |          "Programming Language :: Python :: 3",
46 |          "License :: OSI Approved :: MIT License",
47 |          "Operating System :: OS Independent",
48 |      ],
49 |  )
50 | 
51 | 52 | ### Other files 53 | init file will be empty and README.md you can put about the project. 54 | 55 | After this you need to make sure setuptools is installed on you environment. Then run this below command. 56 | 57 | ### Build package 58 |
59 | python setup.py bdist_wheel
60 | 
61 | 62 | This will create your python package and your package will be present in /dist directory. 63 | 64 | To install this package you can simply use the below command. 65 | 66 | 67 | ### Install local package 68 |
69 | python -m pip install firstapp-0.1-py2-none-any.whl
70 | 
71 | 72 | This will install firstapp in your python environment. You can use it like below. 73 | 74 | 75 | ### Using Installed package. 76 |
77 | from firstapp import pack
78 | pack.say_hello()
79 | 
80 | 81 | 82 | This will print the Hello from the package. 83 | 84 | This was very basic of how you can create a python pakcage. The advanced thing that you can try is installing all the dependent packages of your application at the time of application installation. 85 | 86 | -------------------------------------------------------------------------------- /docs/syntax.md: -------------------------------------------------------------------------------- 1 | # About python 2 | 3 | 4 | 5 | 6 | ## Blocks in python 7 | 8 | While in most of the langauges blocks are defined using brackets like {}, (), [] but in python blocks are defined using spaces that is the piece of code that has the same spaces from the start of line belong to that contiguous block. 9 | 10 | Example: 11 | 12 |
 13 | for(int i=0;i<=20;i++){	//block start  
 14 | 	console.log(i)
 15 | }	//block end
 16 | 
17 | 18 | This is javascript example of printing number from one to 20 19 | 20 |
 21 | i=0
 22 | while i<=20: 	#block start
 23 | 	print(i)
 24 | 				#block end
 25 | 
26 | 27 | This is python style of the same. 28 | 29 | ### Why are Blocks important? 30 | 31 | Blocks are important because variables scoping depends a lot on them. Variable defined inside a block will only be accessible from inside the block only. 32 | 33 | ## Variables 34 | #### Python Variable Name Rules 35 | - Must begin with a letter (a - z, A - B) or underscore (_) 36 | - Other characters can be letters, numbers or _ 37 | - Case Sensitive 38 | - Can be any (reasonable) length 39 | - There are some reserved words which you cannot use as a variable name because Python uses them for other things. 40 | 41 | ####Good Variable Name 42 | - Choose meaningful name instead of short name. roll_no is better than rn. 43 | - Maintain the length of a variable name. Roll_no_of_a-student is too long? 44 | - Be consistent; roll_no or RollNo 45 | - Begin a variable name with an underscore(_) character for a special case. 46 | 47 | 48 | Since python in not static typed language you do not need to define the type of variables. 49 | 50 | 51 | ## Loops 52 | 53 | ### For 54 | 55 |
 56 | i=0
 57 | for i < 10:
 58 | 	print(i)
 59 | 	i++
 60 | 
61 | 62 | ### While 63 | 64 |
 65 | i=0
 66 | while i<10:
 67 | 	print(i)
 68 | 	i++
 69 | 
70 | 71 | ### Loop through objects 72 | 73 |
 74 | a = [1,2,3,4,5,6,7]
 75 | for i in a:
 76 | 	print(i)
 77 | 
78 | 79 |
 80 | 
 81 | a=[{"name":"qw"},{"name":"qwe"}]
 82 | 
 83 | for i in a:
 84 | 	print(a["name"])
 85 | 
86 | 87 | ## Functions 88 | 89 |
 90 | 
 91 | def function_add(var1, var2, var3):
 92 | 	do_some_work
 93 | 	return var1+var2+var3
 94 | 
95 | 96 | ### Default arguements in functions 97 | 98 |
 99 | def function_default_arg(var1, var2, var3=10):
100 | 	return var1+var2+var3
101 | 
102 | 103 | Here if no value is specified for var3 it will be 10 by default so you can call it like below. It also shows us a form of polymorphism which we will see later in this tutorial. -------------------------------------------------------------------------------- /docs/threading.md: -------------------------------------------------------------------------------- 1 | # Threading and multiprocessing in Python 2 | 3 | Threading module implments basic threading in python. Using threading is not an easy task. Threading wants you to write your code in a concurrent design so that it can run parallely on your CPU. 4 | 5 | Now there are two things: What is difference between concurrency and parallelism and what about these in python. 6 | 7 | ## What is difference between concurrency and parallelism. 8 | 9 | In simple word concurrency is a property of your code and parallelisn is property of your runtime. This means concurrency you have to implement in your code and if that code runs in parallel or not on CPU is parallelism. 10 | 11 | ## Concurrency and parallelism in python. 12 | 13 | You can write concurrent code in python but it will never run in parallel. The cause if GIL which doesn't let you run code in parallel in python. Though you will have the illusion of these things running in parallel if you are doing I/O intensive tasks. 14 | 15 | Now lets implement a threaded program in python 16 | 17 | ## Threading implementation 18 | 19 | We will use threading library to make our program run in parallel. 20 |
 21 | import logging
 22 | import threading
 23 | import time
 24 | from datetime import datetime
 25 | 
 26 | def function(name):
 27 |     logging.info("Thread %s: starting", name)
 28 |     for i in range (1000):
 29 |         a = (i*i)
 30 |     logging.info("Thread %s: finishing", name)
 31 | 
 32 | if __name__ == "__main__":
 33 |     format = "%(asctime)s: %(message)s"
 34 |     logging.basicConfig(format=format, level=logging.INFO,
 35 |                         datefmt="%H:%M:%S")
 36 | 
 37 |     logging.info("Main    : before creating thread")
 38 |     a = []
 39 |     for i in range (3):
 40 |         x = threading.Thread(target=function, args=(i,))
 41 |         x.start()
 42 |         a.append(x)
 43 |     for i in a:
 44 |         i.join()
 45 | 
46 | 47 | In the above code we have imported thread. Then created 3 threads and started them and after that joined them so that the execute will proceed after that only once all the threads are completed. 48 | 49 | Now let us see how we will do the same using multiprocessing. 50 | 51 | ## Multiprocessing in python 52 | 53 | Multiprocessing instead of creating threads create a new process which itself is heavier. Let use multiprocessing in above program. 54 | 55 |
 56 | import logging
 57 | import multiprocessing
 58 | import time
 59 | from datetime import datetime
 60 | 
 61 | def function(name):
 62 |     logging.info("Thread %s: starting", name)
 63 |     for i in range (1000):
 64 |         a = (i*i)
 65 |     logging.info("Thread %s: finishing", name)
 66 | 
 67 | if __name__ == "__main__":
 68 |     format = "%(asctime)s: %(message)s"
 69 |     logging.basicConfig(format=format, level=logging.INFO,
 70 |                         datefmt="%H:%M:%S")
 71 | 
 72 |     logging.info("Main    : before creating thread")
 73 |     a= []
 74 |     for i in range (3):
 75 |         x = multiprocessing.Process(target=function, args=(i, ))
 76 |         x.start()
 77 |         a.append(x)
 78 |     for i in a:
 79 |         i.join()
 80 | 
81 | 82 | ## Threading vs Multiprocessing in python 83 | 84 | Threading in python is helpful in IO bound process and Multithreading in case of CPU bound. We will see the comparision of CPU bound process to see the difference between the execution time of mutlithreaded program and multiprocess program. 85 | 86 | 87 | 88 |
 89 | #multiprocessing version
 90 | import multiprocessing
 91 | import logging
 92 | import time
 93 | from datetime import datetime
 94 | 
 95 | def process_function(name):
 96 |     for i in range (10000):
 97 |         a = (i*i)
 98 |         b = (i*i)+1
 99 | 
100 | if __name__ == "__main__":
101 |     start_time = datetime.now()
102 |     format = "%(asctime)s: %(message)s"
103 |     a = []
104 |     for i in range (100):
105 |         x = multiprocessing.Process(target=process_function, args=(i, ))
106 |         x.start()
107 |         a.append(x)
108 |     for i in a:
109 |         i.join()
110 |     end_time = datetime.now()
111 |     print(end_time-start_time)
112 | 
113 | 114 | 115 |
116 | #Multithreaded version
117 | import logging
118 | import threading
119 | import time
120 | from datetime import datetime
121 | 
122 | def thread_function(name):
123 |     for i in range (10000):
124 |         a = (i*i)
125 |         b = (i*i)+1
126 | 
127 | if __name__ == "__main__":
128 |     start_time = datetime.now()
129 |     format = "%(asctime)s: %(message)s"
130 |     a = []
131 |     for i in range (100):
132 |         x = threading.Thread(target=thread_function, args=(i,))
133 |         x.start()
134 |         a.append(x)
135 |     for i in a:
136 |         i.join()
137 |     end_time = datetime.now()
138 |     print(end_time-start_time)
139 | 
140 | 
141 | 142 | 143 | When you run this program you can see the time taken by both the program to complete the execution. Try and explain why the time difference happened. 144 | 145 | Thinking in term of context swtiches, size of thread and process and how GIL works will help you in defining why the time difference in there. 146 | 147 | ## Lets talk about context switching, weight of process and thread. 148 | 149 | ### In case of IO bound process 150 | 151 | In this scenario since CPU is taken away from the process which is waiting for an IO process. So in this scenario threads can be better since context switching for threads is lighter operation as comparision to context switching in case of process. 152 | 153 | ### In case of CPU intensice process 154 | 155 | In this scenario process will perform better when the amount of CPU task being done is huge otherwise you can see similar performance in the smaller tasks because the heavier context switches for process will compensate for the performance gain that it gets from parallel processing in CPU. 156 | 157 | Again if the number of processes are very large their will be huge number of context switches which will again cost and this cost can be make your multiprocessed program slow. 158 | -------------------------------------------------------------------------------- /docs/tooling.md: -------------------------------------------------------------------------------- 1 | To begin with python tooling, lets first see what these tools are used for. 2 | # Starting a project 3 | To start any project it is always recommended to create the project in its own environment. This means that anything that is installed for python at global level will not affect this env anf vice versa. 4 | 5 | ## How to start development? 6 | 7 | ### VirutalENV: 8 | 9 | Install virtualenv 10 | 11 | sudo apt-get install virtualenv 12 | 13 | After installation activate it. 14 | 15 | virtualenv env_name -m python3 16 | 17 | This will create an env for python 3 and you can start working inside it. Keep in mind that you have to activate the env before running your code to make it work. 18 | 19 | Activate env: 20 | 21 | sourve env_name/bin/activate 22 | 23 | Now you can install any packages that you want to use in your python program. 24 | 25 | Deactivate env: 26 | 27 | deactivate 28 | 29 | ### Pep8 formatting. 30 | 31 | Pep8 is the formatting style that defines how you should format your python program. How you should name your variables and more such conventions. 32 | Pep8 is highly recommende for anyone who want to work with opensource community. 33 | 34 | ### Dependency Management 35 | 36 | For dependency management in python we use pip. It is used for installing packages. You can have a file naming requirements.txt which will have all the packages that you need to install along with the version that you want to install. 37 | 38 | ##### How to install python package: 39 | 40 | pip install package_name 41 | 42 | ##### How to install using requirements.txt 43 | 44 | pip install -r requirements.txt 45 | 46 | ### Editor? 47 | 48 | Pycharm is very good for python but if you are power user of sublime that will be awesome. -------------------------------------------------------------------------------- /hello.py: -------------------------------------------------------------------------------- 1 | print "hello" 2 | -------------------------------------------------------------------------------- /images/python_interpreter_basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chowmean/python-training-for-devops/6d2f9b3aba1567fe638c1d6aa66f4ca9b0a83fb0/images/python_interpreter_basic.png -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Python Training for Devops. 2 | nav: 3 | - Home: index.md 4 | - Python Syntax: syntax.md 5 | - Memory Management: mem.md 6 | - Global Interpreter Lock: gil.md 7 | - Object Oriented Programming: oops.md 8 | - Basic Tooling for Python: tooling.md 9 | - Best Practices for programming and python: best_practices.md 10 | - Creating a CLI application: cliapp.md 11 | - Logging in application: logging.md 12 | - Decorators: decorators.md 13 | - Python internals: internals.md 14 | - Flask: flask.md 15 | - Python With Mysql: flaskdb.md 16 | - Flask and ORM: flaskandorm.md 17 | - Exposing metrics with python: metrics.md 18 | - Creating a python package: package.md 19 | - Lets create and application: app.md 20 | - Iterators and how to create one: iterators.md 21 | - Generators: generators.md 22 | - Threading and Multiprocessing: threading.md 23 | 24 | theme: 25 | name: material 26 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backports-abc==0.5 2 | Click==7.0 3 | futures==3.2.0 4 | htmlmin==0.1.12 5 | Jinja2==2.10.1 6 | jsmin==2.2.2 7 | livereload==2.6.1 8 | Markdown==3.1.1 9 | MarkupSafe==1.1.1 10 | mkdocs==1.0.4 11 | mkdocs-material==4.4.0 12 | mkdocs-minify-plugin==0.2.1 13 | pkg-resources==0.0.0 14 | Pygments==2.4.2 15 | pymdown-extensions==6.0 16 | PyYAML==5.1.1 17 | singledispatch==3.4.0.3 18 | six==1.12.0 19 | tornado==5.1.1 20 | flask 21 | -------------------------------------------------------------------------------- /static/style.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | *:focus { 6 | outline: none; 7 | } 8 | body { 9 | font-family: Arial; 10 | background-color: #3498DB; 11 | padding: 50px; 12 | } 13 | .login { 14 | margin: 20px auto; 15 | width: 300px; 16 | } 17 | .login-screen { 18 | background-color: #FFF; 19 | padding: 20px; 20 | border-radius: 5px 21 | } 22 | 23 | .app-title { 24 | text-align: center; 25 | color: #777; 26 | } 27 | 28 | .login-form { 29 | text-align: center; 30 | } 31 | .control-group { 32 | margin-bottom: 10px; 33 | } 34 | 35 | input { 36 | text-align: center; 37 | background-color: #ECF0F1; 38 | border: 2px solid transparent; 39 | border-radius: 3px; 40 | font-size: 16px; 41 | font-weight: 200; 42 | padding: 10px 0; 43 | width: 250px; 44 | transition: border .5s; 45 | } 46 | 47 | input:focus { 48 | border: 2px solid #3498DB; 49 | box-shadow: none; 50 | } 51 | 52 | .btn { 53 | border: 2px solid transparent; 54 | background: #3498DB; 55 | color: #ffffff; 56 | font-size: 16px; 57 | line-height: 25px; 58 | padding: 10px 0; 59 | text-decoration: none; 60 | text-shadow: none; 61 | border-radius: 3px; 62 | box-shadow: none; 63 | transition: 0.25s; 64 | display: block; 65 | width: 250px; 66 | margin: 0 auto; 67 | } 68 | 69 | .btn:hover { 70 | background-color: #2980B9; 71 | } 72 | 73 | .login-link { 74 | font-size: 12px; 75 | color: #444; 76 | display: block; 77 | margin-top: 12px; 78 | } -------------------------------------------------------------------------------- /templates/hello.html: -------------------------------------------------------------------------------- 1 | 2 | Hello from Flask 3 | {% if name %} 4 |

Hello {{ name }}!

5 | {% else %} 6 |

Hello, World!

7 | {% endif %} -------------------------------------------------------------------------------- /templates/login.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 20 |
-------------------------------------------------------------------------------- /templates/page_not_found.html: -------------------------------------------------------------------------------- 1 | not found this page --------------------------------------------------------------------------------