├── README.md └── code_snippets ├── context_manager_tutorial.py ├── descriptor.py ├── parallelism ├── part1-threading │ ├── alaki.py │ ├── multi_threaded_cpubound.py │ ├── podcast_downloader-simplethread.py │ ├── podcast_downloader-single.py │ ├── podcast_downloader-threadpool.py │ ├── requirements.txt │ ├── results.txt │ └── single_threaded_cpubound.py └── part2-multiprocessing │ ├── multi_process_cpubound.py │ ├── processpool_cpubound.py │ ├── processpool_methods.py │ └── single_threaded_cpubound.py └── timing_and_profiling.py /README.md: -------------------------------------------------------------------------------- 1 | # Once For Ever Series 2 | This repository holds codes that I wrote in once for ever article series at https://virgool.io/@GreatBahram 3 | 4 | * Parallelism 2 - Multiprocessing: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-parallelism2-bsaucpjhzy6s), source code is available at [link](https://github.com/GreatBahram/OnceForEver/tree/master/code_snippets/parallelism/part2-multiprocessing) 5 | * Parallelism 1: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-parallelism-1-iqyzjwqmks0n), source code is available at [link](https://github.com/GreatBahram/OnceForEver/tree/master/code_snippets/parallelism/part1-threading) 6 | * Context manager: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-context-manager-qqqbqxgryxk5), source code is available at [link](https://github.com/greatbahram/OnceForEver/tree/master/code_snippets/context_manager_tutorial.py) 7 | * Iterators and Generators: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-decorator-pud0dll) 8 | * Decorators: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-decorator-pud0dll) 9 | * Descriptor: Blog post [link](https://virgool.io/@GreatBahram/once-for-ever-descriptor-hgdwzld1qler), source code is available at [link](https://github.com/GreatBahram/OnceForEver/tree/master/code_snippets/descriptor.py) 10 | -------------------------------------------------------------------------------- /code_snippets/context_manager_tutorial.py: -------------------------------------------------------------------------------- 1 | # Link: https://virgool.io/@GreatBahram/once-for-ever-context-manager-qqqbqxgryxk5 2 | ####### 3 | # read a file in a pythonic way 4 | with open('names.txt', mode='rt', encoding='utf-8') as myfile: 5 | for line in myfile: 6 | print(line) 7 | 8 | ####### 9 | ## Old-fashioned way 10 | try: 11 | myfile = open('names.txt', mode='rt', encoding='utf-8') 12 | for line in myfile: 13 | print(line) 14 | finally: 15 | myfile.close() 16 | 17 | ####### 18 | # How to define a context manager 19 | class File: 20 | def __init__(self, filename, mode='rt'): 21 | self.filename = filename 22 | self.mode = mode 23 | 24 | def __enter__(self): 25 | self.open_file = open(self.filename, mode=self.mode) 26 | return self.open_file # as part 27 | 28 | def __exit__(self, *excepts): 29 | self.open_file.close() 30 | 31 | # How to use it? 32 | with File('names.txt', mode='rt') as myfile: 33 | for line in myfile: 34 | print(line) 35 | 36 | ####### 37 | # Compare with and without 'with' statement 38 | ## without 'with' statement, how ugly it is, isn't it? 39 | myfile = File('names.txt', mode='rt') 40 | myfile = myfile.__enter__() 41 | try: 42 | for line in myfile: 43 | print(line) 44 | finally: 45 | myfile.__exit__() 46 | 47 | ## Beyond PEP 8 by using 'with' statement 48 | with File('names.txt', mode='rt') as myfile: 49 | for line in myfile: 50 | print(line) 51 | 52 | ####### 53 | # Compare old and new-fashioned way to work with a databse 54 | # Old-fashioned way 55 | import sqlite3 56 | 57 | path = './database.db' 58 | 59 | connection = sqlite3.connect(path) 60 | cursor = connection.cursor() 61 | 62 | cursor.execute(''' CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT, email TEXT unique) ''') 63 | cursor.execute(''' INSERT INTO users(name, email) VALUES('Bahram Aghaei','aghaee.bahram@gmail.com')''') 64 | cursor.execute(''' SELECT * FROM users''') 65 | 66 | users = cursor.fetchall() 67 | 68 | connection.commit() 69 | 70 | # Pythonic way 71 | import sqlite3 72 | from sqlite3 import IntegrityError 73 | 74 | path = './database.db' 75 | 76 | class DBHelper: 77 | ''' Simple Database helper which supports context manager as well''' 78 | def __init__(self, dbname): 79 | self.dbname = dbname 80 | self.connection = None 81 | self.cursor = None 82 | 83 | def __enter__(self): 84 | self.connection = sqlite3.connect(path) 85 | self.cursor = self.connection.cursor() 86 | return self.cursor 87 | 88 | def __exit__(self, exctype, value, traceback): 89 | if exctype is IntegrityError: 90 | print('That e-mail is taken, try another one.') 91 | else: 92 | self.connection.commit() 93 | self.connection.close() 94 | return True 95 | 96 | with DBHelper(path) as dbhelper: 97 | dbhelper.execute(''' CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT, email TEXT unique) ''') 98 | dbhelper.execute(''' INSERT INTO users(name, email) VALUES('Bahram Aghaei','aghaee.bahram@gmail.com')''') 99 | dbhelper.execute(''' SELECT * FROM users''') 100 | users = dbhelper.fetchall() 101 | print(users) 102 | 103 | ####### 104 | # Timer context manager 105 | import time 106 | 107 | 108 | class Timer: 109 | def __init__(self, title): 110 | self.title = title 111 | 112 | def __enter__(self): 113 | self.start_time = time.time() 114 | 115 | def __exit__(self, exctype, value, traceback): 116 | self.end_time = time.time() 117 | self.elapsed_time = self.end_time - self.start_time 118 | print(f'{self.title} took {self.elapsed_time}') 119 | print(f'type is {exctype}') 120 | print(f'value is {value}') 121 | print(f'traceback is {traceback}') 122 | return True 123 | 124 | with Timer('Alaki'): 125 | time.sleep(2) 126 | raise TypeError('Context manager is a great thing') 127 | print('After exception') 128 | 129 | ####### 130 | # Contextlib 131 | 132 | ## Example1: ope_file context manager by using contextlib decorator 133 | from contextlib import contextmanager 134 | 135 | @contextmanager 136 | def open_file(filename, mode='rt'): 137 | myfile = open(filename, mode=mode) 138 | yield myfile 139 | myfile.close() 140 | 141 | with open_file('names.txt') as myfile: 142 | for line in myfile: 143 | print(line) 144 | 145 | 146 | ## Example2: dbhelper context manager by using contextlib decorator 147 | import sqlite3 148 | 149 | from contextlib import contextmanager 150 | from sqlite3 import IntegrityError 151 | 152 | path = './database.db' 153 | 154 | @contextmanager 155 | def dbhelper(dbname): 156 | connection = sqlite3.connect(dbname) 157 | cursor = connection.cursor() 158 | try: 159 | yield cursor 160 | except IntegrityError as e: 161 | msg = 'That e-mail is taken, try another one.' 162 | # raise 163 | finally: 164 | connection.commit() 165 | connection.close() 166 | if msg: 167 | print(msg) 168 | 169 | with dbhelper(path) as db: 170 | db.execute(''' INSERT INTO users(name, email) VALUES('Bahram Aghaei','aghaee.bahram@gmail.com')''') 171 | 172 | ## Example 3 - How to use contextlib's closing context manager 173 | from contextlib import closing 174 | 175 | with closing(open('names.txt', mode='at')) as fp: 176 | fp.write('contextlib.closing') 177 | 178 | fp.write('I cannot write here') 179 | 180 | ## Example 4: How to use contextlib's suppress context manager 181 | ### what is the problem? 182 | import os 183 | 184 | from contextlib import suppress 185 | 186 | filename = 'alakikhan.png' 187 | 188 | try: 189 | os.remove(filename) 190 | except FileNotFoundError: 191 | pass 192 | 193 | ### How to solve it? 194 | import os 195 | 196 | from contextlib import suppress 197 | 198 | filename = 'alakikhan.png' 199 | 200 | with suppress(FileNotFoundError): 201 | os.remove(filename) 202 | -------------------------------------------------------------------------------- /code_snippets/descriptor.py: -------------------------------------------------------------------------------- 1 | class Clock: 2 | def __init__(self, hour, minute): 3 | self.hour = hour 4 | self.minute = minute 5 | 6 | 7 | c = Clock(12, 24) 8 | print(c.hour) 9 | c.hour = 13 10 | c.hour 11 | del c.hour 12 | 13 | 14 | class Book: 15 | def __init__(self, name, price): 16 | self.name = name 17 | self.price = price 18 | 19 | 20 | interestng_book = Book("Atomic Habits", 16.20) 21 | 22 | # Access instance and class attributes 23 | interestng_book.__dict__ 24 | 25 | vars(interestng_book) 26 | 27 | vars(Book) 28 | 29 | 30 | from abc import ABC, abstractmethod 31 | 32 | 33 | class Validator(ABC): 34 | def __init__(self, initial): 35 | self.initial = initial 36 | 37 | def __set_name__(self, instance, name): 38 | self.name = name 39 | 40 | def __get__(self, instance, obj_type): 41 | return vars(instance).get(self.name, self.initial) 42 | 43 | def __set__(self, instance, value): 44 | self.validate(value) 45 | vars(instance)[self.name] = value 46 | 47 | @abstractmethod 48 | def validate(self, value): 49 | """Validate the given value.""" 50 | 51 | 52 | class NonNegative(Validator): 53 | def validate(self, value): 54 | if value < 1: 55 | raise ValueError("Positive number required") 56 | 57 | 58 | class cached_property: 59 | def __init__(self, func): 60 | self.func = func 61 | 62 | def __set_name__(self, obj_type, name): 63 | self.name = name 64 | 65 | def __get__(self, instance, obj_type): 66 | print("Computing...") 67 | value = self.func(instance) 68 | vars(instance)[self.name] = value 69 | return value 70 | 71 | 72 | from datetime import datetime 73 | 74 | 75 | class Person: 76 | def __init__(self, name, born): 77 | self.name = name 78 | self.born = born 79 | 80 | @cached_property 81 | def age(self): 82 | current_year = datetime.now().year 83 | return current_year - self.born 84 | 85 | 86 | j = Person("Jamie", 1980) 87 | j.age 88 | j.age 89 | del j.age 90 | j.age 91 | j.age 92 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/alaki.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import threading 4 | import time 5 | 6 | 7 | def long_function(name): 8 | """ just wasting time""" 9 | time.sleep(5) 10 | print(name) 11 | 12 | def main(): 13 | 14 | print('Start of the program.') 15 | 16 | # instanciate the Thread class 17 | thread = threading.Thread(target=long_function, args=('bahram', )) 18 | 19 | # run the thread 20 | thread.run() 21 | 22 | # wait until the thread is finished 23 | thread.join() 24 | 25 | print('End of the program.') 26 | 27 | if __name__ == '__main__': 28 | main() 29 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/multi_threaded_cpubound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import threading 3 | import time 4 | 5 | COUNT = 85_000_000 6 | 7 | def countdown(number): 8 | while number > 0: 9 | number -= 1 10 | 11 | if __name__ == '__main__': 12 | print('Start of program.') 13 | 14 | start_time = time.time() 15 | 16 | thread1 = threading.Thread(target=countdown, args=(COUNT//2,)) 17 | thread2 = threading.Thread(target=countdown, args=(COUNT//2,)) 18 | 19 | thread1.start() 20 | thread2.start() 21 | 22 | thread1.join() 23 | thread2.join() 24 | 25 | end_time = time.time() 26 | 27 | print('End of program.') 28 | 29 | print(f'Countdown took {int(end_time - start_time)} seconds') 30 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/podcast_downloader-simplethread.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | import requests 5 | 6 | from bs4 import BeautifulSoup 7 | 8 | main_url = 'https://talkpython.fm' 9 | 10 | def get_episodes_links(): 11 | """ Gets all episodes links""" 12 | episodes_url = main_url + '/episodes/all' 13 | with requests.get(episodes_url) as request: 14 | data = request.text 15 | soup = BeautifulSoup(data, 'lxml') 16 | links = [main_url + link.attrs['href'] for link in soup.select('.table-hover a')] 17 | return links 18 | 19 | def get_audio_link(link): 20 | """ return audio link for a given link""" 21 | with requests.get(link) as request: 22 | data = request.text 23 | soup = BeautifulSoup(data, 'lxml') 24 | audio_link = soup.select_one('.subscribe-btn') 25 | #print(f'Getting audio link for {Path(audio_link.attrs["href"]).name}...') 26 | return main_url + audio_link.attrs['href'] 27 | 28 | def download_link(directory, link): 29 | """ download and save the audio """ 30 | print(f'Downloaded {link}') 31 | 32 | def get_bunch_audio_links(links, result): 33 | output = [] 34 | for link in links: 35 | if link: 36 | output.append(get_audio_link(link)) 37 | result.append(output) 38 | 39 | def main(): 40 | num_of_threads = 2 41 | 42 | print('Start of program.') 43 | 44 | start_time = time.time() 45 | audiolists = [] 46 | links = get_episodes_links() 47 | 48 | partition = len(links) // num_of_threads 49 | 50 | downloadThreads = [] 51 | 52 | # Create and start the Thread objects. 53 | for i in range(0, len(links), partition): 54 | downloadThread = threading.Thread(target=get_bunch_audio_links, args=(links[i: i+partition], audiolists)) 55 | downloadThreads.append(downloadThread) 56 | downloadThread.start() 57 | 58 | # Wait for all threads to end. 59 | for thread in downloadThreads: 60 | thread.join() 61 | 62 | end_time = time.time() 63 | 64 | print('End of program.') 65 | print(f'Download links took {int(end_time - start_time)} seconds') 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/podcast_downloader-single.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | 4 | import requests 5 | 6 | from bs4 import BeautifulSoup 7 | 8 | main_url = 'https://talkpython.fm' 9 | 10 | def get_episodes_links(): 11 | """ Gets all episodes links""" 12 | episodes_url = main_url + '/episodes/all' 13 | with requests.get(episodes_url) as request: 14 | data = request.text 15 | soup = BeautifulSoup(data, 'lxml') 16 | links = [main_url + link.attrs['href'] for link in soup.select('.table-hover a')] 17 | return links 18 | 19 | def get_audio_link(link): 20 | """ return audio link for a given link""" 21 | with requests.get(link) as request: 22 | data = request.text 23 | soup = BeautifulSoup(data, 'lxml') 24 | audio_link = soup.select_one('.subscribe-btn') 25 | #print(f'Getting audio link for {Path(audio_link.attrs["href"]).name}...') 26 | return main_url + audio_link.attrs['href'] 27 | 28 | def download_link(directory, link): 29 | """ download and save the audio """ 30 | print(f'Downloaded {link}') 31 | 32 | def main(): 33 | print('Start of program.') 34 | start_time = time.time() 35 | audiolists = [] 36 | 37 | links = get_episodes_links() 38 | 39 | for episode_link in reversed(links): 40 | audiolists.append(get_audio_link(episode_link)) 41 | 42 | end_time = time.time() 43 | print('End of program.') 44 | print(f'Download links took {int(end_time - start_time)} seconds') 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/podcast_downloader-threadpool.py: -------------------------------------------------------------------------------- 1 | import time 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | import requests 5 | 6 | from bs4 import BeautifulSoup 7 | 8 | main_url = 'https://talkpython.fm' 9 | 10 | def get_episodes_links(): 11 | """ Gets all episodes links""" 12 | episodes_url = main_url + '/episodes/all' 13 | with requests.get(episodes_url) as request: 14 | data = request.text 15 | soup = BeautifulSoup(data, 'lxml') 16 | links = [main_url + link.attrs['href'] for link in soup.select('.table-hover a')] 17 | return links 18 | 19 | def get_audio_link(link): 20 | """ return audio link for a given link""" 21 | with requests.get(link) as request: 22 | data = request.text 23 | soup = BeautifulSoup(data, 'lxml') 24 | audio_link = soup.select_one('.subscribe-btn') 25 | #print(f'Getting audio link for {Path(audio_link.attrs["href"]).name}...') 26 | return main_url + audio_link.attrs['href'] 27 | 28 | def download_link(directory, link): 29 | """ download and save the audio """ 30 | print(f'Downloaded {link}') 31 | 32 | def main(): 33 | num_of_threads = 10 34 | 35 | print('Start of program.') 36 | 37 | start_time = time.time() 38 | audiolists = [] 39 | links = get_episodes_links() 40 | 41 | with ThreadPoolExecutor(max_workers=num_of_threads) as pool: 42 | audiolists = pool.map(get_audio_link, links) 43 | 44 | end_time = time.time() 45 | 46 | print('End of program.') 47 | print(f'Download links took {int(end_time - start_time)} seconds') 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.6.3 2 | bs4==0.0.1 3 | certifi==2018.10.15 4 | chardet==3.0.4 5 | idna==2.7 6 | lxml==4.6.3 7 | requests==2.20.0 8 | urllib3==1.26.5 9 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/results.txt: -------------------------------------------------------------------------------- 1 | 2 | Multthread example output 3 | 4 | num_of_threads | time(s) 5 | -------------------------- 6 | 2 75, 74 7 | 3 49, 49 8 | 5 31, 31 9 | 10 17, 17 10 | 30 10, 10 11 | 60 10, 10 12 | 100 10, 10 13 | 150 10, 10 14 | 185 10, 10 15 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part1-threading/single_threaded_cpubound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import time 3 | 4 | COUNT = 85_000_000 5 | 6 | def countdown(number): 7 | while number > 0: 8 | number -= 1 9 | 10 | if __name__ == '__main__': 11 | print('Start of program.') 12 | 13 | start_time = time.time() 14 | 15 | countdown(COUNT) 16 | 17 | end_time = time.time() 18 | 19 | print('End of program.') 20 | 21 | print(f'Countdown took {int(end_time - start_time)} seconds') 22 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part2-multiprocessing/multi_process_cpubound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import multiprocessing 3 | import time 4 | 5 | COUNT = 85_000_000 6 | 7 | def countdown(number): 8 | while number > 0: 9 | number -= 1 10 | 11 | if __name__ == '__main__': 12 | print('Start of program.') 13 | 14 | start_time = time.time() 15 | 16 | process1 = multiprocessing.Process(target=countdown, args=(COUNT//2,)) 17 | process2 = multiprocessing.Process(target=countdown, args=(COUNT//2,)) 18 | 19 | process1.start() 20 | process2.start() 21 | 22 | process1.join() 23 | process2.join() 24 | 25 | end_time = time.time() 26 | 27 | print('End of program.') 28 | 29 | print(f'Countdown took {int(end_time - start_time)} seconds') 30 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part2-multiprocessing/processpool_cpubound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import time 3 | from multiprocessing import Pool, cpu_count 4 | 5 | COUNT = 85_000_000 6 | 7 | def countdown(number): 8 | while number > 0: 9 | number -= 1 10 | 11 | if __name__ == '__main__': 12 | print('Start of program.') 13 | 14 | start_time = time.time() 15 | num_of_process = cpu_count() 16 | 17 | jobs = [COUNT//2, COUNT//2] 18 | 19 | with Pool(processes=num_of_process) as pool: 20 | pool.map(countdown, jobs) 21 | 22 | end_time = time.time() 23 | 24 | print('End of program.') 25 | 26 | print(f'Countdown took {int(end_time - start_time)} seconds') 27 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part2-multiprocessing/processpool_methods.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from multiprocessing import Pool 3 | 4 | jobs = [ 5 | (1, 'Mat', 'Fraser'), 6 | (2, 'Rich', 'Froning'), 7 | (3, 'Brooke', 'Ence'), 8 | (4, 'Tia', 'Toomey'), 9 | ] 10 | 11 | 12 | def simple_func(args): 13 | number, fname, lname = args 14 | return (number, fname) 15 | 16 | 17 | def main(): 18 | # map method example 19 | with Pool() as pool: 20 | results = pool.map(simple_func, jobs) 21 | for result in results: 22 | print(result[0], result[1]) 23 | 24 | # imap method example 25 | with Pool() as pool: 26 | for result in pool.imap(simple_func, jobs): 27 | print(result[0], result[1]) 28 | 29 | 30 | # imap_unordered method example 31 | with Pool() as pool: 32 | for result in pool.imap_unordered(simple_func, jobs): 33 | print(result[0], result[1]) 34 | 35 | # starmap method example, take note there is no need to unpack the arguments 36 | def custom_func(number, fname, lname): 37 | return (number, fname) 38 | 39 | with Pool() as pool: 40 | results = pool.starmap(custom_func, jobs) 41 | for result in results: 42 | print(result[0], result[1]) 43 | 44 | if __name__ == '__main__': 45 | main() 46 | 47 | -------------------------------------------------------------------------------- /code_snippets/parallelism/part2-multiprocessing/single_threaded_cpubound.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import time 3 | 4 | COUNT = 85_000_000 5 | 6 | def countdown(number): 7 | while number > 0: 8 | number -= 1 9 | 10 | if __name__ == '__main__': 11 | print('Start of program.') 12 | 13 | start_time = time.time() 14 | 15 | countdown(COUNT) 16 | 17 | end_time = time.time() 18 | 19 | print('End of program.') 20 | 21 | print(f'Countdown took {int(end_time - start_time)} seconds') 22 | -------------------------------------------------------------------------------- /code_snippets/timing_and_profiling.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | large_a = list(range(1000)) 5 | large_b = list(range(1000)) 6 | 7 | 8 | def approach_1(x: List[int], w: List[int]) -> int: 9 | """Generate index; access items; add them.""" 10 | z = 0.0 11 | for i in range(len(x)): 12 | z += x[i] * w[i] 13 | return z 14 | 15 | 16 | def approach_2(x: List[int], w: List[int]) -> int: 17 | """Using functional operators.""" 18 | import operator 19 | 20 | return sum(map(operator.mul, x, w)) 21 | 22 | 23 | import numpy as np 24 | large_a, large_b = np.arange(1000), np.arange(1000) 25 | 26 | def approach_3(x: List[int], w: List[int]) -> int: 27 | """They say NumPy is fast!""" 28 | 29 | return np.dot(x, w) 30 | 31 | 32 | ######################################################## 33 | 34 | import time 35 | 36 | start_time = time.perf_counter() 37 | 38 | total = 0 39 | 40 | for _ in range(100_000_000): 41 | total += 1 42 | 43 | elapsed_time = time.perf_counter() - start_time 44 | 45 | print("It took {:0.1f} seconds.".format(elapsed_time)) 46 | 47 | ######################################################## 48 | 49 | import timeit 50 | setup = "import numpy as np" 51 | stmt = "np.sum(np.arange(1000))" 52 | timeit.timeit(stmt=stmt, setup=setup, number=1_000) 53 | timeit.timeit(stmt=stmt, setup=setup, number=10_000) 54 | # the return value is in seconds. 55 | 56 | ######################################################## 57 | 58 | topic = "timing and profiling" 59 | timeit.timeit("print(topic)", number=1) 60 | 61 | # Fix the problem using globals 62 | timeit.timeit("print(topic)", number=1, globals=globals()) 63 | 64 | # Fix the problem using import statment 65 | timeit.timeit("print(topic)", setup="from __main__ import topic", number=1) 66 | 67 | ######################################################## 68 | 69 | import timeit 70 | 71 | timeit.timeit("approach_1(large_a, large_b)", globals=globals(), number=10_000) 72 | timeit.timeit("approach_2(large_a, large_b)", globals=globals(), number=10_000) 73 | 74 | setup = """ 75 | import numpy as np 76 | 77 | large_a, large_b = np.arange(1000), np.arange(1000) 78 | 79 | from __main__ import approach_3 80 | """ 81 | 82 | timeit.timeit("approach_3(large_a, large_b)", setup=setup, number=10_000) 83 | 84 | ######################################################## 85 | 86 | %timeit approach_1(large_a, large_b) 87 | 88 | %timeit approach_2(large_a, large_b) 89 | 90 | import numpy as np 91 | 92 | large_a, large_b = np.arange(1000), np.arange(1000) 93 | 94 | 95 | def approach_3(x: List[int], w: List[int]) -> int: 96 | """They say NumPy is fast!""" 97 | return np.dot(x, w) 98 | 99 | %timeit approach_3(large_a, large_b) 100 | 101 | ######################################################## 102 | 103 | import time 104 | 105 | 106 | def an_expensive_function(times: int): 107 | time.sleep(times) 108 | 109 | 110 | def main(): 111 | print("Start") 112 | an_expensive_function(1) 113 | an_expensive_function(3) 114 | an_expensive_function(1) 115 | print("End") 116 | 117 | 118 | if __name__ == "__main__": 119 | main() 120 | 121 | 122 | python3 -m cProfile -o output.prof profile_this.py 123 | 124 | pip install snakeviz 125 | 126 | snakeviz output.prof 127 | 128 | --------------------------------------------------------------------------------