├── .gitignore ├── LICENSE ├── README.md ├── assets ├── BBFHQEPWWXJM.jpeg └── course-one-final-project.png ├── automating-real-world-tasks-with-python ├── week-one │ └── README.md ├── week-three │ ├── README.md │ ├── car_sales.json │ ├── emails.py │ └── tmp │ │ ├── cars.pdf │ │ ├── example.png │ │ └── report.pdf └── week-two │ └── README.md ├── configuration-management-and-the-cloud ├── week-four │ └── README.md ├── week-one │ └── README.md ├── week-three │ └── README.md └── week-two │ ├── README.md │ ├── ntp.pp │ └── tools.pp ├── crash-course-on-python ├── week-four │ ├── README.md │ └── course-one-week-four.py ├── week-one │ ├── README.md │ └── course-one-week-one.py ├── week-six │ ├── README.md │ └── course-one-week-six.py ├── week-three │ ├── README.md │ └── course-one-week-three.py └── week-two │ ├── README.md │ └── course-one-week-two.py ├── introduction-to-git-and-github ├── week-four │ └── README.md ├── week-three │ └── README.md └── week-two │ └── README.md ├── troubleshooting-and-debugging-technique ├── week-four │ └── README.md ├── week-one │ ├── README.md │ ├── binary-search.py │ ├── find_item.py │ └── linear-search.py ├── week-three │ └── README.md └── week-two │ ├── README.md │ └── thumbnail_generator.py └── using-python-to-interact-with-the-operating-system ├── week-five ├── README.md ├── charfreq.py ├── rearrange.py ├── rearrange_test.py └── validations.py ├── week-four ├── README.md ├── check_cron.py ├── course-two-week-four.py ├── create_file.py ├── hello.py ├── myapp.py ├── parameters.py ├── seconds.py └── variables.py ├── week-one ├── README.md ├── health_checks.py └── network.py ├── week-seven └── final.sh ├── week-six ├── README.md ├── check-localhost.sh ├── fruits.sh ├── gather-information.sh ├── old_website │ ├── about.HTM │ ├── contact.HTM │ ├── footer.HTM │ ├── index.HTM │ └── rename.sh ├── random-exit.py ├── retry.sh ├── toploglines.sh └── while.sh ├── week-three ├── README.md └── course-two-week-three.py └── week-two ├── README.md ├── course-two-week-two.py ├── csv_file.txt ├── generate_report.py ├── software.csv └── spider.txt /.gitignore: -------------------------------------------------------------------------------- 1 | Source: https://github.com/github/gitignore/blob/master/Python.gitignore 2 | 3 | # User generated 4 | ENV/ 5 | .vscode 6 | .idea 7 | .DS_Store 8 | 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | .pytest_cache/ 15 | 16 | # C extensions 17 | *.so 18 | 19 | # Distribution / packaging 20 | .Python 21 | build/ 22 | develop-eggs/ 23 | dist/ 24 | downloads/ 25 | eggs/ 26 | .eggs/ 27 | lib/ 28 | lib64/ 29 | parts/ 30 | sdist/ 31 | var/ 32 | wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .coverage 52 | .coverage.* 53 | .cache 54 | nosetests.xml 55 | coverage.xml 56 | *.cover 57 | .hypothesis/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | .static_storage/ 66 | .media/ 67 | local_settings.py 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # celery beat schedule file 89 | celerybeat-schedule 90 | 91 | # SageMath parsed files 92 | *.sage.py 93 | 94 | # Environments 95 | .env 96 | .venv 97 | env/ 98 | venv/ 99 | ENV/ 100 | env.bak/ 101 | venv.bak/ 102 | 103 | # Spyder project settings 104 | .spyderproject 105 | .spyproject 106 | 107 | # Rope project settings 108 | .ropeproject 109 | 110 | # mkdocs documentation 111 | /site 112 | 113 | # mypy 114 | .mypy_cache/ 115 | 116 | node_modules/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Kyungrae Kim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google IT Automation With Python 2 | 3 | Author: Kyungrae Kim 4 | Status: Completed 5 | 6 | This repository is created to keep track of [Google IT Automation With Python](https://www.coursera.org/professional-certificates/google-it-automation) provided by Coursera. This program is developed by Google and designed to teach how to program with Python and how to use Python to automate common system administration tasks. Also, the course will teach how to use Git and GitHub, troubleshoot and debug complex problems, and apply automation at scale by using configuration management and the Cloud. 7 | 8 | ![specialization-certificate](https://github.com/jeremymaya/google-it-automation-with-python/blob/master/assets/BBFHQEPWWXJM.jpeg) 9 | 10 | --- 11 | 12 | ## Table of Contents 13 | 14 | 1. [Crash Course On Python](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/crash-course-on-python) 15 | 2. [Using Python to Interact with the Operating System](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/using-python-to-interact-with-the-operating-system) 16 | 3. [Introduction to Git and GitHub](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/introduction-to-git-and-github) 17 | 4. [Troubleshooting and Debugging Techniques](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/troubleshooting-and-debugging-technique) 18 | 5. [Configuration Management and the Cloud](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/configuration-management-and-the-cloud) 19 | 6. [Automating Real-World Tasks with Python](https://github.com/jeremymaya/google-it-automation-with-python/tree/master/automating-real-world-tasks-with-python) 20 | 21 | --- 22 | 23 | ## Credit 24 | 25 | * [Grow With Google - A new certificate to help people grow careers in IT](https://www.blog.google/outreach-initiatives/grow-with-google/new-certificate-help-people-grow-careers/) 26 | * [Coursera - Google IT Automation with Python Professional Certificate](https://www.coursera.org/professional-certificates/google-it-automation#courses) 27 | -------------------------------------------------------------------------------- /assets/BBFHQEPWWXJM.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/assets/BBFHQEPWWXJM.jpeg -------------------------------------------------------------------------------- /assets/course-one-final-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/assets/course-one-final-project.png -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-one/README.md: -------------------------------------------------------------------------------- 1 | # Automating Real-World Tasks with Python - Week 1 2 | 3 | ## Learning Objectives 4 | 5 | --- 6 | 7 | ## Application Programming Interfaces (APIs) 8 | 9 | --- 10 | 11 | ## Manipulating Images 12 | 13 | --- 14 | 15 | ## Credit 16 | 17 | * [Coursera - Automating Real World Tasks Python Week 1](https://www.coursera.org/learn/automating-real-world-tasks-python/home/week/1) 18 | -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Automating Real-World Tasks with Python - Week 3 2 | 3 | ## Learning Objectives 4 | 5 | Look into a different aspect of automation: automating the generation of nicely formatted output from scripts, like sending emails by: 6 | 7 | * Learning how to create the contents of the email or the PDF 8 | * Learning how to attach a file to an email 9 | * Learning how to send the email to an SMTP server 10 | 11 | --- 12 | 13 | ## Sending Emails from Python 14 | 15 | ### Introduction to Python Email Library 16 | 17 | Instead of constructing emails message based on the **Simple Mail Transfer Protocol (SMTP)** and **Multipurpose Internet Mail Extensions (MIME)** standards, we can use Python's EmailMessage module to easily construct email messages. 18 | 19 | Email object created using EmailMessage module consists of **email header fields** and **message body**. 20 | 21 | **Email header fields** are **key-value pair** labels that instructs email clients and servers to route and display the email. Email header fields include: 22 | 23 | * From 24 | * To 25 | * Subject 26 | 27 | Email object's message body is set using the the ```set_content()``` methods. The ```set_content()``` method adds the following additional headers for encoding: 28 | 29 | * Content-Type 30 | * Content-Transfer-Encoding 31 | 32 | ```Python 33 | from email.message import EmailMessage 34 | 35 | # creates an empty email message 36 | message = EmailMessage() 37 | 38 | sender = "me@example.com" 39 | recipient = "you@example.com" 40 | 41 | # adds the sender, recipient, and subject to the message 42 | # From, To, and Subject form email header fields 43 | message['From'] = sender 44 | message['To'] = recipient 45 | message['Subject'] = 'Greetings from {} to {}!'.format(sender, recipient) 46 | 47 | # adds message by using set_content method 48 | body = """Hey there! 49 | 50 | I'm learning to send emails using Python!""" 51 | message.set_content(body) 52 | ``` 53 | 54 | ### Adding Attachments 55 | 56 | The attachment for email needs to be labeled with a MIME type and subtype in order for the recipient to understand what to do with an attachment. We can use Python's mimetypes module to determine attachment type and subtype. 57 | 58 | ```Python 59 | import os.path, mimetypes 60 | attachment_path = "/tmp/example.png" 61 | 62 | # uses the Python mimetypes module to determine attachment type and subtype 63 | mime_type, _ = mimetypes.guess_type(attachment_path) 64 | print(mime_type) 65 | 66 | # splits mime_type string into the MIME type and subtype 67 | mime_type, mime_subtype = mime_type.split('/', 1) 68 | 69 | # opens and adds the attachment the message 70 | with open(attachment_path, 'rb') as ap: 71 | message.add_attachment(ap.read(), 72 | maintype=mime_type, 73 | subtype=mime_subtype, 74 | filename=os.path.basename(attachment_path)) 75 | ``` 76 | 77 | ### Sending the Email Through an SMTP Server 78 | 79 | Computers use the Simple Mail Transfer Protocol (SMTP) to send emails which is a protocol that specifies how computers can deliver email to each other. We can use Python's smtplib module to send email messages. 80 | 81 | For more information, read [Sending Emails With Python](https://realpython.com/python-send-email/). 82 | 83 | ```Python 84 | import smtplib 85 | 86 | # creates an object that will represent a mail server which handles sending messages to the server 87 | mail_server = smtplib.SMTP('localhost') 88 | 89 | # establishes a secrue connection to the remote server using SSL/TLS protocol 90 | mail_server = smtplib.SMTP_SSL('smtp.example.com') 91 | 92 | # sets the debug level on the SMTP object 93 | mail_server.set_debuglevel(1) 94 | 95 | import getpass 96 | 97 | # uses the getpass module to get password - mail_pass variable is still a string 98 | mail_pass = getpass.getpass('Password? ') 99 | 100 | # authenticates to the email server using the SMTP object's login method 101 | mail_server.login(sender, mail_pass) 102 | mail_server.send_message(message) 103 | mail_server.quit() 104 | ``` 105 | 106 | --- 107 | 108 | ## Generating PDFs from Python 109 | 110 | --- 111 | 112 | ## Credit 113 | 114 | * [Coursera - Automating Real World Tasks Python Week 3](https://www.coursera.org/learn/automating-real-world-tasks-python/home/week/3) 115 | -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-three/emails.py: -------------------------------------------------------------------------------- 1 | from email.message import EmailMessage 2 | import os, smtplib, ssl, getpass, mimetypes 3 | 4 | def generate_email_with(): 5 | # creates an empty email message 6 | message = EmailMessage() 7 | 8 | sender = "jeremiah.kim.spam@gmail.com" 9 | recipient = "jeremiah.kim.spam@gmail.com" 10 | 11 | # adds the sender, recipient, and subject to the message 12 | # From, To, and Subject form email header fields 13 | message['From'] = sender 14 | message['To'] = recipient 15 | message['Subject'] = 'Greetings from {} to {}!'.format(sender, recipient) 16 | 17 | # adds message by using set_content method 18 | body = """Hey there! 19 | 20 | I'm learning to send emails using Python!""" 21 | message.set_content(body) 22 | 23 | add_attachment(message) 24 | 25 | return sender, message 26 | 27 | def add_attachment(message): 28 | attachment_path = os.curdir + ("/tmp/example.png") 29 | 30 | # uses the Python mimetypes module to determine attachment type and subtype 31 | mime_type, _ = mimetypes.guess_type(attachment_path) 32 | print(mime_type) 33 | 34 | # splits mime_type string into the MIME type and subtype 35 | mime_type, mime_subtype = mime_type.split('/', 1) 36 | 37 | # opens and adds the attachment the message 38 | with open(attachment_path, 'rb') as ap: 39 | message.add_attachment(ap.read(), 40 | maintype=mime_type, 41 | subtype=mime_subtype, 42 | filename=os.path.basename(attachment_path)) 43 | 44 | def send_email_through_smtp(sender, message): 45 | # https://realpython.com/python-send-email/#option-2-setting-up-a-local-smtp-server 46 | # https://support.google.com/mail/answer/7126229?hl=e 47 | 48 | port = 465 # For SSL 49 | smtp_server = "smtp.gmail.com" 50 | 51 | context = ssl.create_default_context() 52 | with smtplib.SMTP_SSL(smtp_server, port, context=context) as server: 53 | # server.set_debuglevel(1) 54 | mail_pass = getpass.getpass('Password? ') 55 | server.login(sender, mail_pass) 56 | server.send_message(message) 57 | 58 | sender, message = generate_email_with() 59 | send_email_through_smtp(sender, message) 60 | -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-three/tmp/cars.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/automating-real-world-tasks-with-python/week-three/tmp/cars.pdf -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-three/tmp/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/automating-real-world-tasks-with-python/week-three/tmp/example.png -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-three/tmp/report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/automating-real-world-tasks-with-python/week-three/tmp/report.pdf -------------------------------------------------------------------------------- /automating-real-world-tasks-with-python/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Automating Real-World Tasks with Python - Week 2 2 | 3 | ## Learning Objectives 4 | 5 | Look into different tools that can be really useful in today's IT world by: 6 | 7 | * Learning how you can use different text formats to store data in text files, retrieve it, and even transmit it over the internet 8 | * Learning how we can get our code to interact with services running on different computers using a module called Python Requests 9 | 10 | --- 11 | 12 | ## Web Applications and Services 13 | 14 | ### Web Applications and Services (sub) 15 | 16 | A **web application** is an application that you interact with over HTTP. Most of the time when you’re using a website on the Internet, you’re interacting with a web application. 17 | 18 | A **web service** is a web applications that have an API. 19 | 20 | * Instead of browsing to a web page to type and click around, you can use your program to send a message known as an **API call** to the web service's API endpoints. 21 | 22 | ### Data Serialization 23 | 24 | **Data serialization** is the process of taking an in-memory data structure, like a Python object, and turning it into something that can be stored on disk or transmitted across a network. (ex: CSV file) 25 | 26 | Turning the serialized object back into an in-memory object is called **deserialization**. 27 | 28 | A web service's API endpoint can take messages in a specific format, containing specific data such as serialized data. 29 | 30 | ### Data Serialization Formats 31 | 32 | There are different data serialization formats: 33 | 34 | * JavaScript Object Notation (JSON) 35 | * Yet Another Markup Language (YAML) 36 | * Python pickle 37 | * Protocol Buffers 38 | * eXtensible Markup Language (XML) 39 | 40 | JSON (JavaScript Object Notation) is the serialization format that is used widely today in IT. 41 | 42 | Below code uses the ```json.dump()``` function to serialize the people object into a JSON file. 43 | 44 | ```Python 45 | import json 46 | 47 | with open('people.json', 'w') as people_json: 48 | json.dump(people, people_json, indent=2) 49 | ``` 50 | 51 | Below is output of the ```json.dump()``` 52 | 53 | ```JSON 54 | [ 55 | { 56 | "name": "Sabrina Green", 57 | "username": "sgreen", 58 | "phone": { 59 | "office": "802-867-5309", 60 | "cell": "802-867-5310" 61 | }, 62 | "department": "IT Infrastructure", 63 | "role": "Systems Administrator" 64 | }, 65 | { 66 | "name": "Eli Jones", 67 | "username": "ejones", 68 | "phone": { 69 | "office": "684-348-1127" 70 | }, 71 | "department": "IT Infrastructure", 72 | "role": "IT Specialist" 73 | }, 74 | ] 75 | ``` 76 | 77 | ### More About JSON 78 | 79 | Characateristics of JSON incliudes: 80 | 81 | * JSON is human-readable 82 | * JSON elements are always comma-delimited 83 | 84 | JSON supports a few elements of different data types such as: 85 | 86 | * Strings 87 | * Numbers 88 | * Objects 89 | * Key-Value pair 90 | * Arrays - arrays can contain strings, numbers, objects, or other arrays 91 | 92 | ```JSON 93 | [ 94 | "apple", 95 | "banana", 96 | 12345, 97 | 67890, 98 | { 99 | "name": "Sabrina Green", 100 | "username": "sgreen", 101 | "phone": { 102 | "office": "802-867-5309", 103 | "cell": "802-867-5310" 104 | }, 105 | "department": "IT Infrastructure", 106 | "role": "Systems Administrator" 107 | } 108 | ] 109 | ``` 110 | 111 | --- 112 | 113 | ## Python Requests 114 | 115 | ### The Python Requests Library 116 | 117 | HTTP (HyperText Transfer Protocol) is the protocol of the world-wide web 118 | 119 | --- 120 | 121 | ## Credit 122 | 123 | * [Coursera - Automating Real World Tasks Python Week 2](https://www.coursera.org/learn/automating-real-world-tasks-python/home/week/2) 124 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-four/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 4 2 | 3 | ## Learning Objectives 4 | 5 | --- 6 | 7 | ## Credit 8 | 9 | * [Coursera - Configuration Management Cloud Week 4](https://www.coursera.org/learn/configuration-management-cloud/home/week/4) 10 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-one/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 1 2 | 3 | ## Learning Objectives 4 | 5 | Look into how automation can be applied to manage fleets of computers by: 6 | 7 | * Learn how to use **Puppet**, the current industry standard for configuration management 8 | * Learn about the benefits and challenges of moving services to the Cloud 9 | * Learn the best practices for handling hundreds of virtual machines running in the Cloud 10 | 11 | --- 12 | 13 | ## Introduction to Automation at Scale 14 | 15 | ### What is scale 16 | 17 | Being able to **scale** means keep achieving larger impacts with the same amount of effort. A scalable system is a **flexible**. 18 | 19 | **Automation** is an essential tool for keeping up with the infrastructure needs of a growing business. 20 | 21 | ### What is configuration management 22 | 23 | **Unmanaged configuration** manually deploys the installation and configuring a computer. 24 | 25 | **Managed configuration** uses a **configuration management system** to handle all of the configuration of the devices in the **nodes** which aims to solve the scaling problem. 26 | 27 | * Typically you'll define a set of rules that have to be applied to the nodes you want to manage and then have a process that ensures that those settings are true on each of the nodes 28 | 29 | Configuration management system allows a way to make changes to a system or group of systems in a **systematic**, **repeatable way**. 30 | 31 | ### What is infrastructure as code 32 | 33 | **Infrastructure as Code** or IaC is the paradigm of storing all the configuration for the managed devices in version controlled files. This is then combined with automatic tooling to actually get the nodes provisioned and managed. 34 | 35 | The principals of Infrastructure as Code are commonly applied in __cloud computing environments__, where machines are treated like __interchangeable resources__, instead of individual computers. 36 | 37 | IaC allows the followings: 38 | 39 | * Makes the deployment consistent 40 | * Applies the benefits of the version control system to the infrastructure 41 | * Run automated tests on the files 42 | 43 | In a complex or large environment, treating your IT Infrastructure as Code can help you deploy a flexible scalable system. A configuration management system can help you manage that code by providing a platform to maintain and provision that infrastructure in an automated way. 44 | 45 | Managing your Infrastructure as Code it means that the fleet of nodes are **consistent, versioned, reliable, and repeatable**. 46 | 47 | --- 48 | 49 | ## Introduction to Puppet 50 | 51 | ### What is Puppet 52 | 53 | Puppet is the current industry standard for managing the configuration of computers in a fleet of machines. Puppet is: 54 | 55 | * Cross-platform 56 | * Open source project 57 | 58 | Puppet is typically deploy using a client-server architecture. 59 | 60 | * The client is known as the Puppet agent 61 | * Te service is known as the Puppet master 62 | 63 | When using this model, 64 | 65 | 1. The agent connects to the master and sends a bunch of facts that describe the computer to the master 66 | 2. The master then processes this information, generates the list of rules that need to be applied on the device 67 | 3. The master sends this list back to the agent 68 | 4. The agent is then in charge of making any necessary changes on the computer 69 | 70 | Below example says that the package 'sudo' should be present on every computer where the rule gets applied. 71 | 72 | ```puppet 73 | class sudo { 74 | package { 'sudo': 75 | ensure => present, 76 | } 77 | } 78 | ``` 79 | 80 | Tasks Puppet can accomplish includes 81 | 82 | * Install packages 83 | * Add, remove, or modify configuration files stored in the system 84 | * Change registry entries on Windows 85 | * Enable, disable, start, or stop the services 86 | * Configure crone jobs 87 | * Schedule tasks 88 | * Add, remove, or modify Users and Groups 89 | * Execute external commands 90 | 91 | ### Puppet Resources 92 | 93 | **Resources** are the basic unit for modeling the configuration that we want to manage in Puppet. 94 | 95 | * Each resource specifies one configuration that we're trying to manage, like a service, a package, or a file 96 | 97 | Below example is s a simple rule that ensures that etc/sysctl.d exists and is a directory. 98 | 99 | ```puppet 100 | class sysctl { 101 | # resource type: file 102 | # resource title: '/etc/sysctl.d' 103 | file { '/etc/sysctl.d': 104 | # resource attributes 105 | ensure => directory, 106 | } 107 | } 108 | ``` 109 | 110 | Below examples uses a file resource to configure the contents of etc/timezone, a file, which is used in some Linux distributions to determine the time zone of the computer. 111 | 112 | ```puppet 113 | class timezone { 114 | # resource type: file 115 | # resource title: '/etc/timezone' 116 | file { '/etc/timezone': 117 | # resource attributes 118 | # this will be a file instead of a directory or a symlink 119 | # the contents of the file will be the UTC time zone 120 | # the contents of the file will be replaced even if the file already exists 121 | ensure => file, 122 | content => "UTC\n", 123 | replace => true, 124 | } 125 | } 126 | ``` 127 | 128 | When we declare a resource in our puppet rules. We're defining the desired state of that resource in the system. The puppet agent then turns the desired state into reality using providers. 129 | 130 | ### Puppet Classes 131 | 132 | **Classes** in Puppets are used to collect the resources that are needed to achieve a goal in a single place. 133 | 134 | Below example groups all of the resources related to NTP in the same class to make changes in the future eaiser. 135 | 136 | ```puppet 137 | # a class with three resources related to the Network Time Protocol, or NTP 138 | # rules make sure that the NTP package is always upgraded to the latest version 139 | class ntp { 140 | package { 'ntp': 141 | ensure => latest, 142 | } 143 | # contents of the file will be based on the source attribute 144 | file { '/etc/ntp.conf': 145 | source => 'puppet:///modules/ntp/ntp.conf', 146 | replace => true, 147 | } 148 | # enable and run the NTP service 149 | service { 'ntp': 150 | enable => true, 151 | ensure => running, 152 | } 153 | } 154 | ``` 155 | 156 | --- 157 | 158 | ## The Building Blocks of Configuration Management 159 | 160 | ### What are domain-specific languages 161 | 162 | These resources are the building blocks of Puppet rules, but we can do much more complex operations using Puppet's domain specific language or DSL. 163 | 164 | a domain specific language is a programming language that's more limited in scope 165 | 166 | In the case of Puppet, the DSL is limited to operations related to when and how to apply configuration management rules to our devices. 167 | 168 | On top of the basic resource types that we already checked out, Puppet's DSL includes variables, conditional statements, and functions. 169 | 170 | et's talk a bit about Puppet facts. Facts are variables that represent the characteristics of the system. When the Puppet agent runs, it calls a program called factor which analyzes the current system, storing the information it gathers in these facts. Once it's done, it sends the values for these facts to the server, which uses them to calculate the rules that should be applied 171 | 172 | Let's check out an example of a piece of Puppet code that makes use of one of the built-in facts. This piece of code is using the is-virtual fact together with a conditional statement to decide whether the smartmontools package should be installed or purged. This package is used for monitoring the state of hard drives using smart. So it's useful to have it installed in physical machines, but it doesn't make much sense to install it in our virtual machines. 173 | 174 | First, facts is a variable. All variable names are preceded by a dollar sign in Puppet's DSL. In particular, the facts variable is what's known as a hash in the Puppet DSL, which is equivalent to a dictionary in Python. This means that we can access the different elements in the hash using their keys. In this case, we're accessing the value associated to the is virtual key. Second, we see how we can write a conditional statement using if else, enclosing each block of the conditional with curly braces. Finally, each conditional block contains a package resource. We've seen resources before, but we haven't looked at the syntax in detail. So let's do that now. Every resource starts with the type of resource being defined. In this case, package and the contents of the resource are then enclosed in curly braces. Inside the resource definition, the first line contains the title followed by a colon. Any lines after that are attributes that are being set. We use equals greater than to assign values to the attributes and then each attribute ends with a comma 175 | 176 | ```puppet 177 | if $facts['is_virtual']{ 178 | package{ 'smartmontools': 179 | ensure => purged, 180 | } 181 | } 182 | else { 183 | package{ 'smartmontools': 184 | ensure => installed, 185 | } 186 | } 187 | ``` 188 | 189 | ### The Driving Principles of Configuration Management 190 | 191 | Unlike Python or C which are called procedural languages, Puppet is a **declarative language** because the desired state is declared rather than writing the steps to get there. 192 | 193 | There are three important principles of congifuration management 194 | 195 | 1. Idempotentcy 196 | 2. Test and repair paradigm 197 | 3. Stateless 198 | 199 | In configuration management, operations should be **idempotent**. In this context, an idempotent action can be performed over and over again without changing the system after the first time the action was performed, and with no unintended side effects Idempotency is a valuable property of any piece of automation. If a script is idempotent, it means that it can fail halfway through its task and be run again without problematic consequences 200 | 201 | * Most Puppet resources provide idempotent actions 202 | * exec resource is NOT an idempotent actions though - exec modifies the system each time it's executed 203 | * This can be worked around by using the onlyif attribute 204 | 205 | ```puppet 206 | exec {'move example file': 207 | command => 'mv /home/user/example.txt /home/user/Desktop', 208 | onlyif => 'test -e /home/user/example.txt', 209 | } 210 | ``` 211 | 212 | Another important aspect of how configuration management works is the **test and repair paradigm**. This means that actions are taken only when they are necessary to achieve a goal. 213 | 214 | Finally, another important characteristic is **stateless**, this means that there's no state being kept between runs of the agent. 215 | 216 | --- 217 | 218 | ## Credit 219 | 220 | * [Coursera - Configuration Management Cloud Week 1](https://www.coursera.org/learn/configuration-management-cloud/home/week/1) 221 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 3 2 | 3 | ## Learning Objectives 4 | 5 | Dive into the details of the different Cloud services, when it makes sense to use them, and how to get the most out of the cloud deployments by: 6 | 7 | * Learn how cloud deployments can help us quickly scale our services 8 | * Learn the differences between when running IT infrastructure on-premise versus running it in the cloud 9 | * Learn how we can use a variety of different tools to manage instances running in the cloud 10 | 11 | --- 12 | 13 | ## Cloud Computing 14 | 15 | ### Cloud Services Overview 16 | 17 | A service is running in the Cloud means that the service is running somewhere else either in a data center or in other remote servers that can be __reached over via Internet__. 18 | 19 | Cloud providers typically offer different service types such as: 20 | 21 | * **Software as a Service** or SaaS, is when a Cloud provider delivers an entire application or program to the customer 22 | * **Platform as a Service** or PaaS, is when a Cloud provider offers a preconfigured platform to the customer 23 | * **Infrastructure as a Service** or IaaS, is when a Cloud provider supplies only the bare-bones computing experience 24 | 25 | When setting up Cloud resources, **regions** need to be considered. A **region** is a geographical location containing a number of data centers, regions contain **zones** and zones can contain one or more physical **data centers**. 26 | 27 | There are multiple factors that is detremiend based on the selected region, 28 | 29 | * Latency 30 | * Legal or political factors 31 | * Other services as dependencies that is needed to run the service 32 | 33 | ### Scaling in the Cloud 34 | 35 | capacity is how much the service can deliver. 36 | 37 | queries per second or QPS 38 | 39 | This capacity change is called scaling, 40 | 41 | * Upscaling when the capacity is being increased 42 | * Downscaling when the capacity is being decreased 43 | 44 | There are a couple of different ways that we can scale our service in the Cloud, 45 | 46 | * Horizontal scaling vs. Vertical scaling 47 | * Horizontally scaling scales the capacity by adding more nodes into the pool (eg. add more servers) 48 | * Vertically scaling scales the capacity by making the nodes bigger (eg. upgrade memories, CPU or disk space) 49 | * Automatic scaling vs. Manual scaling 50 | * Automatic scaling uses metrics to automatically increase or decrease the capacity of the system which is controleld by the Cloud provider 51 | * Manual scaling are controlled by humans instead of software 52 | 53 | ### Evaluating the Cloud 54 | 55 | In the case of cloud solutions, IT team is giving up some of its control to the cloud provider. Therefore, it's important to know what kind of support is available and select the one that fits the needs. 56 | 57 | Treat the servers executing the workloads as a **commodity** and always use reasonable judgment to protect the machines that we deploy, whether that's on physical server is running on-premise or on virtual machines in the Cloud. 58 | 59 | ### Migrating to the Cloud 60 | 61 | **IaaS** is especially useful to administrators using a **lift and shift strategy**. 62 | 63 | When migrating to the Cloud, the process is somewhat similar. But instead of moving the physical server in the back of a truck, you migrate your physical servers running on-premise to a virtual machine running in the Cloud. In this case, you're shifting from one way of running your servers to another. The key thing to note with both approaches, is that **the servers core configurations stay the same**. 64 | 65 | **PaaS** is well-suited for when you have a specific infrastructure requirement, but you don't want to be involved in the day-to-day management of the platform 66 | 67 | **Containers** are applications that are packaged together with their configuration and dependencies. 68 | 69 | There are different types of Cloud services: 70 | 71 | * Public Cloud is services provided by the public Cloud providers 72 | * Private Cloud is when one company owns the services and the rest of the infrastructure 73 | * Hybrid Cloud is a mixture of both public and private Clouds 74 | * Multi-Cloud is a mixture of public and/or private Clouds across vendors 75 | 76 | --- 77 | 78 | ## Managing Instances in the Cloud 79 | 80 | There are different Cloud providers each with some specific advantages depending on what we are trying to achieve. But usually Cloud service providers implement a console to manage the services. 81 | 82 | Regardless of the service proivder, following paramaters needs to be set when creating a VM running in the Cloud 83 | 84 | * Name of the instance 85 | * Region and zone where the instance will be running 86 | * CPU, memory and boot disk options for the VM 87 | 88 | Cloud service proivders also porvides the command line interface, which allows for us to specify what we want once, and then use the same parameters many times. __Using the command line interface lets us create, modify, and even delete virtual machines from our scripts.__ 89 | 90 | **Reference images** store the contents of a machine in a reusable format, while templating is the process of capturing all of the system configuration to let us create VM in a repeatable way. That exact format of the reference image will depend on the vendor. But often, the result is a file called a **disk image**. A disk image is a snapshot of a virtual machine's disk at a given point in time. 91 | 92 | --- 93 | 94 | ## Automatic Cloud Deployments 95 | 96 | --- 97 | 98 | ## Credit 99 | 100 | * [Coursera - Configuration Management Cloud Week 3](https://www.coursera.org/learn/configuration-management-cloud/home/week/3) 101 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 2 2 | 3 | ## Learning Objectives 4 | 5 | Dive deeper into baisc congfiguration management concepts and Puppet by: 6 | 7 | * Learn how to install Puppet on your computer and how to use a simple test setup to check your rules work as expected 8 | * Learn how to configure the typical client-server set-up with Puppet clients connecting and authenticating to the Puppet server to get the rules that they should apply 9 | * Learn how to use testing techniques and releasing best practices to safely deploy changes to clients of our configuration management system 10 | 11 | --- 12 | 13 | ## Deploying Puppet 14 | 15 | ### Applying Rules Locally 16 | 17 | The **manifest** is a file with .pp extention where we'll store the rules that we want to apply. 18 | 19 | Checkout tools.pp file as an example. 20 | 21 | The **catalog** is the list of rules that are generated for one specific computer once the server has evaluated all variables, conditionals, and functions. 22 | 23 | ### Managing Resource Relationships 24 | 25 | Checkout ntp.pp file as an example. 26 | 27 | ### Organizing Your Puppet Modules 28 | 29 | In puppet, manifests are organized into **modules**. A module is a collection of manifests and associated data. 30 | 31 | * Manifest directory which stores manifest and init.pp where it defines a class with the same name as the created module 32 | * Templates directory which stores any files that stores rules to be used 33 | 34 | --- 35 | 36 | ## Deploying Puppet to Clients 37 | 38 | ### Puppet Nodes 39 | 40 | In Puppet terminology, a **node** is any system where a Puppet agent can run. To apply different rules to different systems is to use separate node definitions. 41 | 42 | Below example is the default node with two classes, the sudo class and the ntp class. 43 | 44 | ```puppet 45 | node default { 46 | class { 'sudo': } 47 | # sets an additional servers parameter which can be used to get the network time 48 | class { 'ntp': 49 | servers => [ 'ntp1.example.com', 'ntp2.example.com' ] 50 | } 51 | } 52 | ``` 53 | 54 | Below example shows how to apply some settings to only some specific nodes by adding more node definitions. 55 | 56 | ```puppet 57 | node default { 58 | class { 'sudo': } 59 | # sets an additional servers parameter which can be used to get the network time 60 | class { 'ntp': 61 | servers => [ 'ntp1.example.com', 'ntp2.example.com' ] 62 | } 63 | class { 'apache': } 64 | } 65 | ``` 66 | 67 | Below example shows how specific nodes in the fleet are identified by their FQDNs, or fully qualified domain names. 68 | 69 | ```puppet 70 | node webserver.example.com { 71 | class { 'sudo': } 72 | # sets an additional servers parameter which can be used to get the network time 73 | class { 'ntp': 74 | servers => [ 'ntp1.example.com', 'ntp2.example.com' ] 75 | } 76 | class { 'apache': } 77 | } 78 | ``` 79 | 80 | ### Puppet's Certificate Infrastructure 81 | 82 | Puppet uses __public key infrastructure (PKI)__, __secure sockets layer (SSL)__, to establish secure connections between the server and the clients. 83 | 84 | __Puppet comes with its own certificate authority__, which can be used to create certificates for each clients. 85 | 86 | why do we care so much about the identity of the nodes? There's a bunch of reasons. 87 | 88 | One of the reason why identity of the nodes matter is that the Puppet rules can sometimes include confidential information. 89 | 90 | * Automatic sign all requests feature is available in Puppet, it should be limited to test deployment and never used for real computers being used by real users 91 | 92 | --- 93 | 94 | ## Updating Deploymnets 95 | 96 | ### Modifying and Testing Manifests 97 | 98 | There are few ways of test changes made on Puppet 99 | 100 | 1. puppet parser validate command with noop paramter 101 | * Checks that the syntax of the manifests is correct 102 | * noop parameter which comes from no operations makes puppet simulate what it would do without actually doing it 103 | 2. Seperate test machines that are used only for testing out changes 104 | 3. Automated testing via R-Spec 105 | 106 | Below example sets the facts involved different values and checks that the catalog ends up stating the expected 107 | 108 | ```puppet 109 | describe 'gksu', :type => :class do 110 | let (:facts) { { 'is_virtual' => 'false' } } 111 | it { should contain_package('gksu').with_ensure('latest') } 112 | end 113 | ``` 114 | 115 | ### Safely Rolling out Changes and Validating Them 116 | 117 | Even if you've tested the change on your computer or on a test computer and it worked just fine, __it doesn't mean that the change will work correctly on all machines running in production__. 118 | 119 | In an infrastructure context, **production** is the parts of the infrastructure where a service is executed and served to its users. 120 | 121 | In order to roll out changes safely, 122 | 123 | 1. Always run them through a test environment first 124 | 2. Push changes in in batches 125 | 3. Make the change to be small and self-contained 126 | 127 | --- 128 | 129 | ## Credit 130 | 131 | * [Coursera - Configuration Management Cloud Week 2](https://www.coursera.org/learn/configuration-management-cloud/home/week/2) 132 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-two/ntp.pp: -------------------------------------------------------------------------------- 1 | # a class with three resources related to the Network Time Protocol, or NTP 2 | # rules make sure that the NTP package is always upgraded to the latest version 3 | class ntp { 4 | # write resource types in lowercase when declaring them 5 | package { 'ntp': 6 | ensure => latest, 7 | } 8 | # contents of the file will be based on the source attribute 9 | # write resource types in lowercase when declaring them 10 | file { '/etc/ntp.conf': 11 | source => 'puppet:///modules/ntp/ntp.conf', 12 | replace => true, 13 | # declares relationship 14 | # configuration file requires the NTP package 15 | # write resource types in capitalize when referring to them from another resource's attributes 16 | require => Package['ntp'], 17 | # notifies the NTP service if the configuration file changes 18 | # the service will get reloaded with the new settings if changes are made to the contents of the configuration file 19 | notify => Service['ntp'], 20 | } 21 | # enable and run the NTP service 22 | # write resource types in lowercase when declaring them 23 | service { 'ntp': 24 | enable => true, 25 | ensure => running, 26 | # service requires the configuration file 27 | require => File['/etc/ntp.conf'], 28 | } 29 | } 30 | 31 | include ntp 32 | 33 | # Run the rules using 'sudo puppet apply -v ntp.pp' 34 | -------------------------------------------------------------------------------- /configuration-management-and-the-cloud/week-two/tools.pp: -------------------------------------------------------------------------------- 1 | # This example uses Puppet to make sure that htop is installed on each computer for debugging 2 | 3 | # htop package is a tool similar to top that shows some extra information 4 | package { 'htop': 5 | # ensures that the package is present on a computer 6 | ensure => present, 7 | } 8 | 9 | # Run the rules using 'sudo puppet apply -v tools.pp' 10 | # The -v flag tells Puppet that we want to get verbose output 11 | -------------------------------------------------------------------------------- /crash-course-on-python/week-four/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 4 2 | 3 | ## Learning Objectives 4 | * Understand the difference between strings, lists, and dictionaries 5 | * Manipulate strings in your code 6 | * Create and use lists 7 | * Create and use dictionaries 8 | 9 | --- 10 | 11 | ## String 12 | ### What is a string? 13 | A **string** is a data type in Python that's used to represent a piece of text. 14 | 15 | Strings can be concatenated to build longer strings by using the plus sign or multiplied by a number, which multiplies the content of the string that many times. 16 | 17 | ### The Parts of a String 18 | The **string indexing** operation accesses the character in a given position or index using square brackets and the number of the position specified 19 | * string[0] 20 | * To access the last character of a string, use negative indexes 21 | * A **slice** is the portion of a string that can contain more than one character (ex. string[1:4]) 22 | 23 | ### Creating New Strings 24 | Strings in Python are immutable, meaning __they can't be modified__. 25 | 26 | ### Formatting Strings 27 | String operations 28 | * len(string) Returns the length of the string 29 | * for character in string Iterates over each character in the string 30 | * if substring in string Checks whether the substring is part of the string 31 | * string[i] Accesses the character at index i of the string, starting at zero 32 | * string[i:j] Accesses the substring starting at index i, ending at index j-1. If i is omitted, it's 0 by default. If j is omitted, it's len(string) by default. 33 | 34 | String methods 35 | * string.lower() / string.upper() Returns a copy of the string with all lower / upper case characters 36 | * string.lstrip() / string.rstrip() / string.strip() Returns a copy of the string without left / right / left or right whitespace 37 | * string.count(substring) Returns the number of times substring is present in the string 38 | * string.isnumeric() Returns True if there are only numeric characters in the string. If not, returns False. 39 | * string.isalpha() Returns True if there are only alphabetic characters in the string. If not, returns False. 40 | * string.split() / string.split(delimiter) Returns a list of substrings that were separated by whitespace / delimiter 41 | * string.replace(old, new) Returns a new string where all occurrences of old have been replaced by new. 42 | * delimiter.join(list of strings) Returns a new string with all the strings joined by the delimiter 43 | 44 | --- 45 | 46 | ## Lists 47 | ### What is a list? 48 | Square brackets are used to indicate where the list starts and ends in Python. 49 | 50 | In Python, strings and lists are both examples of **sequences**. There are other sequences too, and they all share operations like: 51 | * Iterating over them using for-loops 52 | * Indexing using the len function to know the length of the sequence 53 | * Using plus to concatenate two sequences 54 | * Using in to verify if the sequence contains an element 55 | 56 | ### Modifying the Contents of a List 57 | One of the ways that lists and strings are different is that lists are mutable. Which is another fancy word to say that they can change. This means we can add, remove, or modify elements in a list. 58 | * The append method adds a new element at the end of the list. 59 | * The insert method inserts an element in a different position, instead of at the end, by taking an index as the first parameter and an element as the second parameter. 60 | * The remove method removes an element 61 | * The pop method can also remove an element and receives an index 62 | 63 | ### Lists and Tuples 64 | * **Strings** are sequences of characters and are **immutable**. 65 | * **Lists** are sequences of elements of any type and are **mutable**. 66 | * **Tuples** are sequences of elements of any type that are **immutable**. 67 | * When using tuples the position of the elements inside the tuple have meaning. 68 | * Tuples can be **unpacked** whichwe can turn a tuple of elements into separate variables. 69 | 70 | Examples of tuples include: 71 | * When a function returns more than one value, it's returning a tuple. 72 | * Store a filename and it's size 73 | * Store the name and email address of a person 74 | * Store a date and time and the general health of the system at any point in time 75 | 76 | ### Iterating over Lists and Tuples 77 | Use **enumerate** method when iterate over lists or tuples. 78 | 79 | Using range method works BUT it's more idiomatic in Python to iterate through the elements of the list directly or using enumerate when we need the indexes. 80 | 81 | ### List Comprehension 82 | Since creating lists based on sequences is such a common task, Python provides a technique called **list comprehension**, which allows it to be done in just one line. __List comprehensions let us create new lists based on sequences or ranges.__ 83 | 84 | --- 85 | 86 | ## Dictionaries 87 | ### What is a dictionary? 88 | **Dictionaries** take the form of pairs of keys and values to store data. 89 | * Dictionaries are mutable 90 | * Unlike lists, elements cannot be accessed using their position 91 | 92 | #### Definition 93 | x = {key1:value1, key2:value2} 94 | 95 | #### Operations 96 | * len(dictionary) - Returns the number of items in the dictionary 97 | * for key in dictionary - Iterates over each key in the dictionary 98 | * for key, value in dictionary.items() - Iterates over each key,value pair in the dictionary 99 | * if key in dictionary - Checks whether the key is in the dictionary 100 | * dictionary[key] - Accesses the item with key key of the dictionary 101 | * dictionary[key] = value - Sets the value associated with key 102 | * del dictionary[key] - Removes the item with key key from the dictionary 103 | 104 | #### Methods 105 | * dict.get(key, default) - Returns the element corresponding to key, or default if it's not present 106 | * dict.keys() - Returns a sequence containing the keys in the dictionary 107 | * dict.values() - Returns a sequence containing the values in the dictionary 108 | * dict.update(other_dictionary) - Updates the dictionary with the items coming from the other dictionary. Existing entries will be replaced; new entries will be added. 109 | * dict.clear() - Removes all the items of the dictionary 110 | --- 111 | 112 | ## Credit 113 | * [Coursera - Python Crash Course Week 4](https://www.coursera.org/learn/python-crash-course/home/week/4) 114 | * [Python Documentation - Common string operations](https://docs.python.org/3/library/string.html) 115 | * [Python Documentation - String Methods](https://docs.python.org/3/library/stdtypes.html#string-methods) 116 | * [Python Documentation - Dictionary](https://docs.python.org/3/library/stdtypes.html#mapping-types-dict) -------------------------------------------------------------------------------- /crash-course-on-python/week-four/course-one-week-four.py: -------------------------------------------------------------------------------- 1 | # Strings 2 | # What is a string? 3 | """ 4 | Modify the double_word function so that it returns the same word repeated twice, followed by the length of the new doubled word. For example, double_word("hello") should return hellohello10. 5 | """ 6 | def double_word(word): 7 | word *= 2 8 | count = 0 9 | for letter in word: 10 | count += 1 11 | return word + str(count) 12 | 13 | print(double_word("hello")) # Should return hellohello10 14 | print(double_word("abc")) # Should return abcabc6 15 | print(double_word("")) # Should return 0 16 | # The Parts of a String 17 | """ 18 | Modify the first_and_last function so that it returns True if the first letter of the string is the same as the last letter of the string, False if they’re different. Remember that you can access characters using message[0] or message[-1]. Be careful how you handle the empty string, which should return True since nothing is equal to nothing. 19 | """ 20 | def first_and_last(message): 21 | if message == "" or message[0] == message[-1]: 22 | return True 23 | return False 24 | 25 | print(first_and_last("else")) 26 | print(first_and_last("tree")) 27 | print(first_and_last("")) 28 | 29 | # Creating New Strings 30 | """ 31 | Using the index method, find out the position of "x" in "supercalifragilisticexpialidocious". 32 | """ 33 | word = "supercalifragilisticexpialidocious" 34 | print(word.index("x")) 35 | 36 | # More String Methods 37 | """ 38 | Fill in the gaps in the initials function so that it returns the initials of the words contained in the phrase received, in upper case. For example: "Universal Serial Bus" should return "USB"; "local area network" should return "LAN”. 39 | """ 40 | def initials(phrase): 41 | words = phrase.split() 42 | result = "" 43 | for word in words: 44 | result += word[0] 45 | return result.upper() 46 | 47 | print(initials("Universal Serial Bus")) # Should be: USB 48 | print(initials("local area network")) # Should be: LAN 49 | print(initials("Operating system")) # Should be: OS 50 | 51 | # Formatting Strings 52 | """ 53 | Modify the student_grade function using the format method, so that it returns the phrase "X received Y% on the exam". For example, student_grade("Reed", 80) should return "Reed received 80% on the exam". 54 | """ 55 | def student_grade(name, grade): 56 | return "{name} received {grade}% on the exam".format(name=name, grade=grade) 57 | 58 | print(student_grade("Reed", 80)) 59 | print(student_grade("Paige", 92)) 60 | print(student_grade("Jesse", 85)) 61 | 62 | # Lists 63 | # What is a list? 64 | """ 65 | The group_list function accepts a group name and a list of members, and returns a string with the format: group_name: member1, member2, … For example, group_list("g", ["a","b","c"]) returns "g: a, b, c". Fill in the gaps in this function to do that. 66 | """ 67 | def group_list(group, users): 68 | members = group + ": " + ", ".join(users) 69 | return members 70 | 71 | print(group_list("Marketing", ["Mike", "Karen", "Jake", "Tasha"])) # Should be "Marketing: Mike, Karen, Jake, Tasha" 72 | print(group_list("Engineering", ["Kim", "Jay", "Tom"])) # Should be "Engineering: Kim, Jay, Tom" 73 | print(group_list("Users", "")) # Should be "Users:" 74 | 75 | # Modifying the Contents of a List 76 | """ 77 | The skip_elements function returns every other element from the list, starting from the first. Complete this function to do that. 78 | """ 79 | def skip_elements(elements): 80 | new_elements = [] 81 | for element in elements: 82 | if elements.index(element) % 2 == 0: 83 | new_elements.append(element) 84 | return new_elements 85 | 86 | print(skip_elements(["a", "b", "c", "d", "e", "f", "g"])) # Should be ['a', 'c', 'e', 'g'] 87 | print(skip_elements(['Orange', 'Pineapple', 'Strawberry', 'Kiwi', 'Peach'])) # Should be ['Orange', 'Strawberry', 'Peach'] 88 | print(skip_elements([])) # Should be [] 89 | 90 | # Lists and Tuples 91 | """ 92 | The guest_list function reads in a list of tuples with the name, age, and profession of each party guest, and prints the sentence "Guest is X years old and works as __." for each one. For example, guest_list(('Ken', 30, "Chef"), ("Pat", 35, 'Lawyer'), ('Amanda', 25, "Engineer")) should print out: 93 | 94 | Ken is 30 years old and works as Chef. 95 | 96 | Pat is 35 years old and works as Lawyer. 97 | 98 | Amanda is 25 years old and works as Engineer. 99 | 100 | Fill in the gaps in this function to do that. 101 | """ 102 | def guest_list(guests): 103 | for guest in guests: 104 | name, age, job = guest 105 | print("{} is {} years old and works as {}".format(name, age, job)) 106 | 107 | guest_list([('Ken', 30, "Chef"), ("Pat", 35, 'Lawyer'), ('Amanda', 25, "Engineer")]) 108 | 109 | # Iterating over Lists and Tuples 110 | """ 111 | Complete the skip_elements function to return every other element from the list, this time using the enumerate function to check if an element is on an even position or an odd position. 112 | """ 113 | def skip_elements_enumerate(elements): 114 | new_elements = [] 115 | for index, element in enumerate(elements): 116 | if(index % 2 == 0): 117 | new_elements.append(element) 118 | return new_elements 119 | 120 | print(skip_elements_enumerate(["a", "b", "c", "d", "e", "f", "g"])) # Should be ['a', 'c', 'e', 'g'] 121 | print(skip_elements_enumerate(['Orange', 'Pineapple', 'Strawberry', 'Kiwi', 'Peach'])) # Should be ['Orange', 'Strawberry', 'Peach'] 122 | 123 | # List Comprehension 124 | """ 125 | Complete the skip_elements function to return every other element from the list, this time using a list comprehension to generate the new list based on the previous one, where elements in odd positions are skipped. 126 | """ 127 | def skip_elements_list_comprehension(elements): 128 | return [element for element in elements if elements.index(element) % 2 == 0] 129 | 130 | print(skip_elements_list_comprehension(["a", "b", "c", "d", "e", "f", "g"])) # Should be ['a', 'c', 'e', 'g'] 131 | print(skip_elements_list_comprehension(['Orange', 'Pineapple', 'Strawberry', 'Kiwi', 'Peach'])) # Should be ['Orange', 'Strawberry', 'Peach'] 132 | 133 | # Dictionaries 134 | # What is a dictionary? 135 | """ 136 | The "toc" dictionary represents the table of contents for a book. Fill in the blanks to do the following: 1) Add an entry for Epilogue on page 39. 2) Change the page number for Chapter 3 to 24. 3) Display the new dictionary contents. 4) Display True if there is Chapter 5, False if there isn't. 137 | """ 138 | toc = {"Introduction":1, "Chapter 1":4, "Chapter 2":11, "Chapter 3":25, "Chapter 4":30} 139 | toc["Epilogue"] = 39 # Epilogue starts on page 39 140 | toc["Chapter 3"] = 24 # Chapter 3 now starts on page 24 141 | print(toc) # What are the current contents? 142 | print("Chapter 5" in toc) # Is there a Chapter 5? 143 | 144 | # Iterating over the Contents of a Dictionary 145 | """ 146 | Complete the code to iterate through the keys and values of the cool_beasts dictionary. Remember that the items method returns a tuple of key, value for each element in the dictionary. 147 | """ 148 | cool_beasts = {"octopuses":"tentacles", "dolphins":"fins", "rhinos":"horns"} 149 | for beast, feature in cool_beasts.items(): 150 | print("{} have {}".format(beast, feature)) 151 | 152 | # Dictionaries vs. Lists 153 | """ 154 | In Python, a dictionary can only hold a single value for a given key. To workaround this, our single value can be a list containing multiple values. Here we have a dictionary called "wardrobe" with items of clothing and their colors. Fill in the blanks to print a line for each item of clothing with each color, for example: "red shirt", "blue shirt", and so on. 155 | """ 156 | wardrobe = {"shirt":["red","blue","white"], "jeans":["blue","black"]} 157 | for cloth in wardrobe: 158 | for color in wardrobe[cloth]: 159 | print("{} {}".format(color, cloth)) 160 | -------------------------------------------------------------------------------- /crash-course-on-python/week-one/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 1 2 | 3 | ## Learning Objectives 4 | * Understand what Python is 5 | * Understand what scripting is 6 | * Understand why Python is relevant to IT 7 | * Understand the role of automation in IT 8 | * Run your first Python script 9 | 10 | --- 11 | 12 | ## Introduction to Programming 13 | ### What is programming? 14 | At a basic level, a computer program is a recipe of instructions that tells your computer what to do. When you write a program, you create a step by step recipe of what needs to be done to complete a task and when your computer executes the program it reads what you wrote and follows your instructions to the letter. 15 | 16 | In a human language, __**syntax** is the rules for how a sentence is constructed__ while __**semantics** refers to the actual meaning of the statements__. 17 | In a programming language, __the **syntax** is the rules for how each instruction is written__ and __the **semantics** is the effects the instructions have__. 18 | 19 | | | Syntax | Semantic | 20 | |:-|:-|:-| 21 | | Human Language | the rules for how a sentence is constructed | refers to the actual meaning of the statements | 22 | | Programming Language | the rules for how each instruction is written | the effects the instructions have | 23 | 24 | What's the difference between a script and a program? 25 | * Scripts as programs with a short development cycle that can be created and deployed rapidly. In other words, a script is a program that is short, simple, and can be written very quickly. 26 | 27 | ### What is automation? 28 | Automation is the process of replacing a manual step with one that happens automatically. 29 | 30 | The benefits of autmation includes: 31 | * Allowing people to concentrate on more complex, creative, or difficult tasks 32 | * Consistency 33 | 34 | However, some tasks that may require a degree of creativity or flexibility that automatic systems can't provide or for more complicated or less frequently executed tasks creating the automation may actually be more effort or cost than it's worth. 35 | 36 | ### Getting Computers to Work for You 37 | Tasks performed by a computer that need to be done multiple times with little variation are really well suited for automation. 38 | 39 | Automation: 40 | * Helps avoid the possibility of human errors 41 | * Reduces the time it takes to do perfrom tasks 42 | 43 | --- 44 | 45 | ## Introduction To Python 46 | ### What is Python? 47 | Python is a general purpose scripting language. 48 | 49 | Benefits of using Python includes programming in Python usually feels similar to using a human language. 50 | 51 | In programming, an interpreter is the program that reads and executes code. The Python interpreter is the program that reads what is in the recipe and translates it into instructions for your computer to follow. 52 | 53 | ### Other Languages 54 | There are different programming languages. Also, there are platform-specific scripting languages like PowerShell which is used on Windows, and Bash which is used on Linux. Each programming languages has its advantages and disadvantages. 55 | 56 | --- 57 | 58 | ## Hello World 59 | ### Hello, World! 60 | * Functions in Python are pieces of code that performs a unit of work 61 | * Kewwords in PyThon are reserved words tha are used to construct instruction 62 | * The print function in Python outputs messages to the screen. 63 | * The print function wraps text in quotation marks indicates that the text is considered a string, which means it's text that will be manipulated by our script. 64 | ### First Programming Concepts Cheat Sheet 65 | * [First Programming Concepts Cheat Sheet](https://www.coursera.org/learn/python-crash-course/supplement/nonTo/first-programming-concepts-cheat-sheet) 66 | 67 | --- 68 | 69 | ## Credit 70 | * [Coursera - Python Crash Course Week 1](https://www.coursera.org/learn/python-crash-course/home/week/1) -------------------------------------------------------------------------------- /crash-course-on-python/week-one/course-one-week-one.py: -------------------------------------------------------------------------------- 1 | # Introduction to Python 2 | # What is Python - Let's make friends! 3 | friends = ['Taylor', 'Alex', 'Pat', 'Eli'] 4 | for friend in friends: 5 | print("Hi " + friend) 6 | 7 | # Other Languages - Prints "Hello, World!" 10 times 8 | # Notice the similarities and differences between different scripting languages 9 | """ 10 | Bash: 11 | for i in {1...10}; do 12 | echo Hello, World 13 | 14 | PowerShell: 15 | for ($i; $i -le 10; $i++) { 16 | Write-Host "Hello, World!" 17 | } 18 | """ 19 | for i in range(10): 20 | print("Hello, World!") 21 | 22 | # Pratice Quiz- Fill in the correct Python comman to put "My first Python program" onto the screen. 23 | print("My first Python program") 24 | 25 | # Hello World 26 | # Hello, World! - Write a Python script that outputs "I'm programming in Python!" 27 | print("I'm programming in Python!") 28 | 29 | # Getting Information from the User - Change the values of color and thing to have the computer output a different statement than the initial one 30 | name = "Green" 31 | thing = "Hope" 32 | print(name + " is the color of " + thing) 33 | 34 | name = "Blue" 35 | thing = "Sky" 36 | print(name + " is the color of " + thing) 37 | 38 | # Python Can Be Your Calculator 39 | # Addtion = 9 40 | print(4+5) 41 | # Multiplication = 63 42 | print (9*7) 43 | # Division = -0.25 44 | print(-1/4) 45 | # Division - Repeating or periodic numbers = 0.3333333333333333 46 | print(1/3) 47 | # Order of operations (PEMDAS) 48 | print(((2050/5)-32)/9) 49 | # 2 to the power of 10 50 | print(2**10) 51 | # Use Python to calcualte (((1 + 2) * 3) / 4)^5 52 | print((((1+2)*3)/4)**5) 53 | 54 | # Practice Quiz - Calculate the Golden ratio 55 | ratio = ((1+(5**(1/2)))/2) 56 | print(ratio) -------------------------------------------------------------------------------- /crash-course-on-python/week-six/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 6 2 | 3 | ## Writing a Script from the Group Up 4 | ### Problem Statement 5 | Create a daily report that tracks the use of machines. Specifically, create report that generates which users are currently connected to which machines. 6 | 7 | ### Criteria 8 | #### Input 9 | An instance of the event class which will contains the following: 10 | * The date when the event happened 11 | * The name of the machine where it happened 12 | * The user involved 13 | * The event type, which includes the login and logout event type 14 | * The event types are strings and the ones we care about are login and logout 15 | 16 | The class event has the following attributes: 17 | * date 18 | * user 19 | * machine 20 | * type 21 | #### Output 22 | Print the machine name followed by all the current users separated by commas. 23 | 24 | ### Apprach 25 | 1. Sort the list of events chronologically. 26 | 2. Store the data in a dictionary of sets 27 | * We need keep track of who's logged into which machine 28 | * Set provides O(n) search 29 | 3. Print the dictionary 30 | 31 | ## Final Project 32 | ### Problem Statement 33 | Create a "word cloud" from a text by procssing the text and return a dictionary that outputs the frequency of each words. 34 | 35 | ### Criteria 36 | #### Input 37 | * Processing the text 38 | * Remove punctuation 39 | * Ignore case and words that do not contain all alphabets 40 | * Count the frequencies 41 | * Ignore uninteresting or irrelevant words 42 | 43 | #### Output 44 | A dictionary is the output of the calculate_frequencies function. The wordcloud module will then generate the image from your dictionary. 45 | 46 | ### Apprach 47 | 1. Browse and upload a text to be processed 48 | 2. Remove puntuations and normalize the letters 49 | 3. Create a set to store the uninteresting words 50 | 4. Create a dictionary that uses a word as a key and frequencies of the word as a value 51 | 5. Use WordCloud framework to generate a image of world cloud 52 | 53 | ### Visual (The Sun Also Rises by Ernest Hemingway) 54 | ![course-one-final-project](https://github.com/jeremymaya/google-it-automation-with-python/blob/master/assets/course-one-final-project.png) 55 | 56 | --- 57 | 58 | ## Credit 59 | * [Coursera - Python Crash Course Week 6](https://www.coursera.org/learn/python-crash-course/home/week/6) 60 | * [Guthenberg Project - The Sun Also Rises by Ernest Hemingway](https://gutenberg.ca/ebooks/hemingwaye-sunalsorises/hemingwaye-sunalsorises-00-h.html) 61 | -------------------------------------------------------------------------------- /crash-course-on-python/week-six/course-one-week-six.py: -------------------------------------------------------------------------------- 1 | class Event: 2 | def __init__(self, event_date, event_type, machine_name, user): 3 | self.date = event_date 4 | self.type = event_type 5 | self.machine = machine_name 6 | self.user = user 7 | 8 | def get_event_date(event): 9 | return event.date 10 | 11 | def current_users(events): 12 | events.sort(key=get_event_date) 13 | machines = {} 14 | for event in events: 15 | if event.machine not in machines: 16 | machines[event.machine] = set() 17 | if event.type == "login": 18 | machines[event.machine].add(event.user) 19 | elif event.type == "logout" and machines[event.machine] == event.user: 20 | machines[event.machine].remove(event.user) 21 | return machines 22 | 23 | def generate_report(machines): 24 | for machine, users in machines.items(): 25 | if len(users) > 0: 26 | user_list = ", ".join(users) 27 | print("{}: {}".format(machine, user_list)) 28 | 29 | events = [ 30 | Event('2020-01-21 12:45:56', 'login', 'myworkstation.local', 'jordan'), 31 | Event('2020-01-22 15:53:42', 'logout', 'webserver.local', 'jordan'), 32 | Event('2020-01-21 18:53:21', 'login', 'webserver.local', 'lane'), 33 | Event('2020-01-22 10:25:34', 'logout', 'myworkstation.local', 'jordan'), 34 | Event('2020-01-21 08:20:01', 'login', 'webserver.local', 'jordan'), 35 | Event('2020-01-23 11:24:35', 'logout', 'mailserver.local', 'chris'), 36 | ] 37 | 38 | users = current_users(events) 39 | print(users) -------------------------------------------------------------------------------- /crash-course-on-python/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 3 2 | 3 | ## Learning Objectives 4 | * Understand the role and value of loops 5 | * Understand the differences between while and for loops 6 | * Write code that uses while and for loops 7 | * Understand recursion and why it may be useful in scripting 8 | 9 | --- 10 | 11 | ## While Loops 12 | ### What is a while loop? 13 | While Loops instruct computer to continuously execute the code based on the value of a condition. The difference between if statements and while loop is that the body of the block can be executed multiple times instead of just once. 14 | 15 | **Initialization** is to give an initial value to a variable. 16 | 17 | ### Why initializing variables matters 18 | When we forget to initialize the variable two different things can happen: 19 | 1. A name error when we forget to initialize a variable 20 | 2. When we forget to initialize variables with the right value 21 | 22 | ### Infinite loops and how to break them 23 | While loops: 24 | * Use the condition to check when to exit 25 | * The body of the while loop needs to make sure that the condition being checked will change 26 | 27 | An **infinite loop**, a loop that keeps executing and never stops. 28 | 29 | **Break** keyword can stop infinite loops but also to stop a loop early if the code has already achieved what's needed. 30 | 31 | --- 32 | 33 | ## for Loops 34 | ### What is for loop? 35 | For loops iterate over a sequence of values. The for loop can iterate over a sequence of values of any type, not just a range of numbers. 36 | 37 | The range function, range(): 38 | 1. A range of numbers will start with the value 0 by default. 39 | 2. The list of numbers generated will be one less than the given value 40 | 3. The range function takes two parameters to the function instead of one to specify the first element of the list to generate 41 | 4. The range function takes a third parameter to change the size of each step (increments) 42 | 43 | ### Nested for Loops 44 | **Nested for loops** are two for loops, one inside the other. 45 | 46 | However, use caution when using nested for loops since the longer the list your code needs to iterate through, the longer it takes computer to complete the task. 47 | 48 | ### Common Errors 49 | The interpreter will refuse to iterate over a single element. This can be mitigated by: 50 | 1. Using range() 51 | 2. Make the loop iterate over a list with the single element 52 | 53 | Since strings are iterable, above solution may be needed. 54 | 55 | for loops are best when you want to iterate over a known sequence of elements but when you want to operate while a certain condition is true, while loops are the best choice. 56 | 57 | --- 58 | 59 | ## Recursion 60 | ### What is recursion? 61 | **Recursion** is the repeated application of the same procedure to a smaller problem. It tackles complex problems by reducing the problem to a simpler one by a function call itself until it reaches the **base case**. 62 | 63 | In Python, a recursive function can be called up to 1,000 times. 64 | 65 | --- 66 | 67 | 68 | ## Credit 69 | * [Coursera - Python Crash Course Week 3](https://www.coursera.org/learn/python-crash-course/home/week/3) -------------------------------------------------------------------------------- /crash-course-on-python/week-three/course-one-week-three.py: -------------------------------------------------------------------------------- 1 | # While Loops 2 | # What is while loop and more examples 3 | x = 0 4 | while x < 5: 5 | print("Not there yet, x=" + str(x)) 6 | x = x + 1 7 | 8 | print("x=" + str(x)) 9 | 10 | def attempts(n): 11 | x = 1 12 | while x <= n: 13 | print("Attempt " + str(x)) 14 | x += 1 15 | print("Done") 16 | 17 | attempts(5) 18 | # Why initializing variables matters 19 | """ 20 | In this code, there's an initialization problem that's causing our function to behave incorrectly. Can you find the problem and fix it? 21 | """ 22 | def count_down(start_number): 23 | current = start_number 24 | while (current > 0): 25 | print(current) 26 | current -= 1 27 | print("Zero!") 28 | 29 | count_down(3) 30 | 31 | # Infinite loops and how to break them 32 | """ 33 | The following code contains a mistake that can trigger an infinite loop, can you figure out how to fix it? 34 | """ 35 | def smallest_prime_factor(x): 36 | """Returns the smallest prime number that is a divisor of x""" 37 | # Start checking with 2, then move up one by one 38 | n = 2 39 | while n <= x: 40 | if x % n == 0: 41 | return n 42 | n += 1 43 | 44 | print(smallest_prime_factor(12)) # should be 2 45 | print(smallest_prime_factor(15)) # should be 3 46 | 47 | # for Loops 48 | # What is a for loop? 49 | """ 50 | Fill in the gaps of the sum_squares function, so that it returns the sum of all the squares of numbers between 0 and x (not included). Remember that you can use the range(x) function to generate a sequence of numbers from 0 to x (not included). 51 | """ 52 | def square(n): 53 | return n*n 54 | 55 | def sum_squares(x): 56 | sum = 0 57 | for n in range(x): 58 | sum += square(n) 59 | return sum 60 | 61 | print(sum_squares(10)) # Should be 285 62 | 63 | """ 64 | In math, the factorial of a number is defined as the product of an integer and all the integers below it. For example, the factorial of four (4!) is equal to 1*2*3*4=24. Fill in the blanks to make the factorial function return the right number. 65 | """ 66 | def factorial(n): 67 | result = 1 68 | for i in range(1, n + 1): 69 | result *= i 70 | return result 71 | 72 | print(factorial(4)) # should return 24 73 | print(factorial(5)) # should return 120 74 | 75 | # Recursion 76 | # What is recursion? 77 | """ 78 | The function sum_positive_numbers should return the sum of all positive numbers between the number n received and 1. For example, when n is 3 it should return 1+2+3=6, and when n is 5 it should return 1+2+3+4+5=15. Fill in the gaps to make this work: 79 | """ 80 | def sum_positive_numbers(n): 81 | # The base case is n being smaller than 1 82 | if n < 1: 83 | return n 84 | 85 | # The recursive case is adding this number to 86 | # the sum of the numbers smaller than this one. 87 | return n + sum_positive_numbers(n - 1) 88 | 89 | print(sum_positive_numbers(3)) # Should be 6 90 | print(sum_positive_numbers(5)) # Should be 15 91 | -------------------------------------------------------------------------------- /crash-course-on-python/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Crash Course of Python - Week 2 2 | 3 | ## Learning Objectives 4 | * Write scripts that use variables of different data types 5 | * Understand how flow control works in Python 6 | * Implement branches with conditional expressions 7 | * Encapsulate code into functions 8 | 9 | --- 10 | 11 | ## Expression and Variables 12 | ### Data Types 13 | There are different data tpyes in Python. Some of the examples include: 14 | * String which represents text wrapped in quotation marks to be manipulated by a script 15 | * Integer which represents whole numbers without a fraction (ex. 1) 16 | * Float which represents real numbers like a number with a fractional part (ex. 2.5) 17 | 18 | Generally, computer doesn't know how to mix different data types - it results "TypeError". 19 | 20 | If you're ever not 100 percent sure what data types a certain value is, Python gives you a handy way to find out. You can use the type function, to have the computer tell you the type 21 | 22 | ### Variables 23 | * **Variables** are names that we give to certain values in our programs - variables are like containers for data. 24 | * Values can be of any data type; numbers, strings or even the results of operations. 25 | * Variables are important in programming because they let you perform operations on data that may change 26 | * **Assignment** is the process of storing a value inside a variable. 27 | * A value is assigned to a variable by using the equal sign in the form of variable equals value 28 | * An **expression** is a combination of numbers, symbols or other variables that produce a result when evaluated. 29 | 30 | Below are some restrictions when naming variables in Python 31 | 1. Key words or functions that Python reserves for its own cannot be used as variable names 32 | 2. Variable names can't have any spaces and they must start with either a letter or an underscore 33 | 3. Variable names can only be made up of letters, numbers and underscores 34 | 4. Python variables are case sensitive 35 | 36 | ### Expressions, Numbers, and Type Conversion 37 | * **Implicit conversion** takes plance when the interpreter automatically converts one data type into another. 38 | * **Explicit conversion** takes place when a function is called to convert one data type into antother 39 | 40 | --- 41 | 42 | ## Functions 43 | ### Defining Functions 44 | To define a function in Python, 45 | 1. Use the **def** keyword to define a function. The name of the function is what comes after the keyword. 46 | 2. The parameters of the function are written between parentheses after the name. 47 | 3. The body of the function must be to the right of the definition. 48 | Example: 49 | ```Python 50 | def greeting(name): 51 | print("Hello, " + name) 52 | ``` 53 | 54 | ### Reutrning Values 55 | The return statement allows us to: 56 | * Combine calls to functions 57 | * Perform more complex operations 58 | * Makes the code more reusable 59 | * **In Python, it can return more than one value** 60 | 61 | * **Double slash operator (//)** is called **floor division**. A floor division divides a number and takes the integer part of the division as the result. 62 | * **None** is a very special data type in Python used to indicate that things are empty or that they return nothing. 63 | 64 | ### Code Style 65 | A few principles in mind to create good well styled code are: 66 | 1. Code to be self-documenting as possible 67 | * Self-documenting code is written in a way that's readable and doesn't conceal its intent. 68 | 2. Leave a comment to add a bit of explanatory texts to the code 69 | * In Python, comments are indicated by the hash character 70 | 71 | --- 72 | 73 | ## Conditionals 74 | ### Compare Things 75 | Equality operators allows us to take the result of the expressions and use them to make decisions 76 | * a == b: a is equal to b 77 | * a != b: a is different than b 78 | * a < b: a is smaller than b 79 | * a <= b: a is smaller or equal to b 80 | * a > b: a is bigger than b 81 | * a >= b: a is bigger or equal to b 82 | 83 | Logical operators allow connecting multiple statements together and perform more complex comparisons 84 | * a and b: True if both a and b are True. False otherwise. 85 | * a or b: True if either a or b or both are True. False if both are False. 86 | * not a: True if a is False, False if a is True. 87 | 88 | **In Python uppercase letters are alphabetically sorted before lowercase letters** 89 | 90 | ### Branching with if Statements 91 | **Branching** is the ability of a program to alter its execution sequence. 92 | 93 | if block vs. defined function 94 | | Similarities | Differences | 95 | |:-|:-| 96 | | * The keyword, either def or if, indicates the start of a special block 97 | * Colon is used at the end of the first line 98 | * The body of the function or the if block is indented to the right | * The body of the if block will only execute when the condition evaluates to true; otherwise, it skipped | 99 | 100 | ### else Statements 101 | * The **modulo operator** is represented by the percentage sign (%) and __returns the remainder of the integer division between two numbers__. 102 | * **The integer division is an operation between integers that yields two results which are both integers, the quotient and the remainder.* 103 | * When a return statement is executed, the function exits so that the code that follows doesn't get executed. 104 | 105 | ### elif Statements 106 | The main difference between elif and if statements is that an elif block can be only written as as a companion to an if block. 107 | 108 | --- 109 | 110 | ## Credit 111 | * [Coursera - Python Crash Course Week 2](https://www.coursera.org/learn/python-crash-course/home/week/2) -------------------------------------------------------------------------------- /crash-course-on-python/week-two/course-one-week-two.py: -------------------------------------------------------------------------------- 1 | # Expression and Variables 2 | # Data Types 3 | print(type("a")) 4 | # 5 | print(type(2)) 6 | # 7 | print(type(2.5)) 8 | # 9 | 10 | # Variables - Calculate the area of a triangle of base 5, height 3 and output the result 11 | base = 5 12 | height = 3 13 | area = (base * height) / 2 14 | 15 | print(area) 16 | 17 | # Expressions, Numbers, and Type Conversion 18 | """ 19 | In this scenario, we have a directory with 5 files in it. Each file has a different size: 2048, 4357, 97658, 125, and 8. Calculate the average file size by having Python add all the values for you, and then set the amount variable to the amount of files. Finally, output a message saying "The average size is: " followed by the resulting number. 20 | """ 21 | sum = 2048 + 4357 + 97658 + 125 + 8 22 | amount = 5 23 | average = sum/amount 24 | 25 | print("The average size is: " + str(average)) 26 | 27 | # Functions 28 | # Defining Functions 29 | """ 30 | Flesh out the body of the print_seconds function so that it prints the total amount of seconds given the hours, minutes, and seconds function parameters. Remember that there are 3600 seconds in an hour and 60 seconds in a minute. 31 | """ 32 | def print_seconds(hours, minutes, seconds): 33 | print((hours * 60 * 60) + (minutes * 60) + seconds) 34 | 35 | print_seconds(1,2,3) 36 | 37 | # Returning Values 38 | """ 39 | Use the get_seconds function to work out the amount of seconds in 2 hours and 30 minutes, then add this number to the amount of seconds in 45 minutes and 15 seconds. Then print the result. 40 | """ 41 | def get_seconds(hours, minutes, seconds): 42 | return 3600*hours + 60*minutes + seconds 43 | 44 | amount_a = get_seconds(2, 30, 0) 45 | amount_b = get_seconds(0, 45, 15) 46 | result = amount_a + amount_b 47 | print(result) 48 | 49 | # Example of returning multiple values from a function 50 | def convert_seconds(seconds): 51 | hours = seconds // 3600 52 | minutes = (seconds - hours * 3600) // 60 53 | remaining_seconds = seconds - hours * 3600 - minutes * 60 54 | return hours, minutes, remaining_seconds 55 | 56 | hours, minutes, seconds = convert_seconds(5000) 57 | print(hours, minutes, seconds) 58 | 59 | # The Principles of Code Reuse 60 | """ 61 | In this code, identify the repeated pattern and replace it with a function called print_monthly_expense, that receives the name of the month and the hours as parameters. Adapt the rest of the code so that the result is the same. 62 | """ 63 | june_hours = 243 64 | june_cost = june_hours * 0.65 65 | print("In June we spent: " + str(june_cost)) 66 | 67 | july_hours = 325 68 | july_cost = july_hours * 0.65 69 | print("In July we spent: " + str(july_cost)) 70 | 71 | august_hours = 298 72 | august_cost = august_hours * 0.65 73 | print("In August we spent: " + str(august_cost)) 74 | 75 | def print_monthly_expense(month, hours): 76 | print("In " + month + " we spent: " + str(hours * 0.65)) 77 | 78 | print_monthly_expense("June", 243) 79 | print_monthly_expense("July", 325) 80 | print_monthly_expense("August", 298) 81 | 82 | # Coding Style 83 | """ 84 | This function to calculate the area of a rectangle is not very readable. Can you refactor it, and then call the function to calculate the area with base of 5 and height of 6? Tip: a function that calculates the area of a rectangle should probably be called rectangle_area, and if it's receiving base and height, that's what the parameters should be called. 85 | """ 86 | def f1(x, y): 87 | z = x*y # the area is base*height 88 | print("The area is " + str(z)) 89 | 90 | def rectangle_area(base, height): 91 | area = base*height 92 | print("The area is " + str(area)) 93 | 94 | rectangle_area(5, 6) 95 | 96 | # Condtionals 97 | # Equality Operators 98 | print(10 > 1) # True 99 | print("cat" == "dog") # False 100 | print(1 != 2) # True 101 | # print(1 < "1") # TypeError 102 | print(1 == "1") # False 103 | # Logical Operators 104 | print("Yellow" > "Cyan" and "Brown" > "Magenta") # False 105 | print(25 > 50 or 1 != 2) # True 106 | print(not 42 == "Answer") # True 107 | 108 | # Branching with if Statements 109 | """ 110 | The is_positive function should return True if the number received is positive, otherwise it returns None. Can you fill in the gaps to make that happen? 111 | """ 112 | def is_positive(number): 113 | if number > 0: 114 | return True 115 | 116 | # else Statements 117 | """ 118 | The is_positive function should return True if the number received is positive and False if it isn't. Can you fill in the gaps to make that happen? 119 | """ 120 | def is_positive_two(number): 121 | if number > 0: 122 | return True 123 | else: 124 | return False 125 | 126 | 127 | # elif Statements 128 | """ 129 | The number_group function should return "Positive" if the number received is positive, "Negative" if it's negative, and "Zero" if it's 0. Can you fill in the gaps to make that happen? 130 | """ 131 | def number_group(number): 132 | if number > 0: 133 | return "Positive" 134 | elif number < 0: 135 | return "Negative" 136 | else: 137 | return "Zero" 138 | 139 | print(number_group(10)) #Should be Positive 140 | print(number_group(0)) #Should be Zero 141 | print(number_group(-5)) #Should be Negative -------------------------------------------------------------------------------- /introduction-to-git-and-github/week-four/README.md: -------------------------------------------------------------------------------- 1 | # Introduction to Git and GitHub - Week 4 2 | 3 | ## Collaboration 4 | 5 | ### Pull Requests 6 | 7 | **Forking** is a way of creating a copy of the given repository so that it belongs to our user. 8 | 9 | **Pull request** is a commit or series of commits that you send to the owner of the repository so that they incorporate it into their tree. 10 | 11 | When collaborating on projects hosted on GitHub, the typical workflows is: 12 | 13 | 1. Create a fork of the repo 14 | 2. Work on that local fork 15 | 16 | --- 17 | 18 | ## Code Review 19 | 20 | ### Managing Collaboration 21 | 22 | **Code review** means going through someone else's code, documentation or configuration and checking that it all makes sense and follows the expected patterns. 23 | 24 | The goal of a code review is: 25 | 26 | * To improve the project by making sure that changes are high quality 27 | * Helps make the contents are easy to understand 28 | * The style is consistent with the overall project 29 | * Remind us about any important cases 30 | 31 | ### Continuous Integration 32 | 33 | **Continuous integration** is a system building and testing code every time there's a change. 34 | 35 | **Continuous deployment** means the new code is deployed often. The goal is to avoid roll outs with a lot of changes between two versions of a project and instead do incremental updates with only a few changes at a time. 36 | 37 | Some of concepts that needs to be dealt with when creating CICD includes: 38 | 39 | 1. Pipelines, which specifies the steps that need to run to get the desired result 40 | 2. Artifacts, which is the name used to describe any files that are generated as part of the pipeline 41 | 42 | --- 43 | 44 | ## Managing Projects 45 | 46 | When collaborating: 47 | 48 | * Documenting any work is important 49 | * As a project maintainer, reply promptly to pull requests and don't let them stagnate 50 | * Understand any accpeted changes 51 | * Use an issue tracker to coordinating who does what and when 52 | * Have a way of communicating and coordinating between contributors 53 | 54 | --- 55 | 56 | ## Credit 57 | 58 | * [Coursera - Introduction Git Github Week 4](https://www.coursera.org/learn/introduction-git-github/home/week/4) 59 | -------------------------------------------------------------------------------- /introduction-to-git-and-github/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Introduction to Git and GitHub - Week 3 2 | 3 | ## Introduction to GitHub 4 | 5 | ### What is Github 6 | 7 | GitHub is a web-based Git repository hosting service whle Git is a distributed version control system. 8 | 9 | * Distributed means that each developer has a copy of the whole repository on their local machine. 10 | 11 | For real configuration and development work, more secure and private Git server needs to be used, and limit the people authorized to work on it. 12 | 13 | ### Git Commands to Interact with Remote Repository 14 | 15 | * git clone URL: Git clone is used to clone a remote repository into a local workspace 16 | * git push: Git push is used to push commits from your local repo to a remote repo 17 | * git pull: Git pull is used to fetch the newest updates from a remote repository 18 | 19 | --- 20 | 21 | ## Using a Remote Repository 22 | 23 | ### What is a remote 24 | 25 | Remote repositories allows 26 | 27 | * Developers contribute to a project from their own workstations making changes to local copies of the project independently of one another 28 | * Developers to use git commands to pull code from a remote repository or push code into one when they need to share their changes 29 | 30 | ### Git Remote Cheatsheet 31 | 32 | * git remote: Lists remote repos 33 | * git remote -v: List remote repos verbously 34 | * git remote show \: Describes a single remote repo 35 | * git remote update: Fetches the most up-to-date objects 36 | * git fetch: Downloads specific objects 37 | * git branch -r: Lists remote branches; can be combined with other branch arguments to manage remote branches 38 | 39 | --- 40 | 41 | ## Solving Conflicts 42 | 43 | ### Rebasing Your Changes 44 | 45 | Rebasing means changing the base commit that's used for our branch. 46 | 47 | The problem with three way merges is that because of the split history, it's hard for us to debug when an issue is found in our code, and we need to understand where the problem was introduced. By changing the base where our commits split from the branch history, we can replay the new commits on top of the new base. This allows Git to do a fast forward merge and keep history linear. 48 | 49 | Run the command git rebase, followed by the branch that we want to set as the new base. 50 | 51 | ### Best Practices for Collaboration 52 | 53 | * Always synchronize your branches before starting any work on your own 54 | * Starting from the most recent version and you minimize the chances of conflicts or the need for rebasing 55 | * Avoid having very large changes that modify a lot of different things 56 | * Push your changes often and pull before doing any work to reduce the chances of getting conflict 57 | * Have the latest version of the project in the master branch and a stable version of the project on a separate branch. 58 | * Rebasing can help a lot with identifying bugs, but use it with caution 59 | * Whenever we do a rebase, we're rewriting the history of our branch 60 | * Do not rebase changes that have been pushed to remote repos 61 | * Have good commit messages 62 | 63 | --- 64 | 65 | ## Credit 66 | 67 | * [Coursera - Introduction Git Github Week 3](https://www.coursera.org/learn/introduction-git-github/home/week/3) 68 | -------------------------------------------------------------------------------- /introduction-to-git-and-github/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Introduction to Git and GitHub - Week 2 2 | 3 | ## Advanced Git interaction 4 | 5 | ### Git Commands Cheatsheet 6 | 7 | * git commit -a: Stages files automatically 8 | * git log -p: Produces patch text 9 | * git show: Shows various objects 10 | * git diff: Is similar to the Linux `diff` command, and can show the differences in various commits 11 | * git diff --staged: An alias to --cached, this will show all staged files compared to the named commit 12 | * git add -p: Allows a user to interactively review patches to add to the current commit 13 | * git mv: Similar to the Linux `mv` command, this moves a file 14 | * git rm: Similar to the Linux `rm` command, this deletes, or removes a file 15 | 16 | --- 17 | 18 | ## Undoing Things 19 | 20 | ### Git Revert Cheatsheet 21 | 22 | * git checkout: Used to switch branches 23 | * git reset: Resets the repo by throwing away some changes 24 | * git commit --amend: Make changes to commits after-the-fact, which can be useful for making notes about a given commit 25 | * git revert: Makes a new commit which effectively rolls back a previous commit. It’s a bit like an undo command 26 | 27 | ### Ammeding Commits 28 | 29 | Avoid amending commits that have already been made public. 30 | 31 | ### Identifying a Commit 32 | 33 | The commit ID is the 40 character long string after the word commit generated by SHA1 algorithm. Git doesn't use these hashes for security. Instead, they're used to guarantee the consistency of our repository. 34 | 35 | --- 36 | 37 | ## Branching and Merging 38 | 39 | ### Git Branching and Merging Cheatsheet 40 | 41 | * git branch: Used to manage branches 42 | * git branch \: Creates the branch 43 | * git branch -d \: Deletes the branch 44 | * git branch -D \: Forcibly deletes the branch 45 | * git checkout \: Switches to a branch. 46 | * git checkout -b \: Creates a new branch and switches to it. 47 | * git merge \: Merge joins branches together. 48 | * git merge --abort: If there are merge conflicts (meaning files are incompatible), --abort can be used to abort the merge action. 49 | * git log --graph --oneline: This shows a summarized view of the commit history for a repo. 50 | 51 | ### What is a branch 52 | 53 | A branch is a pointer to a particular commit. The default branch that Git creates for you when a new repository initialized is called master. 54 | 55 | ### Working with Branches 56 | 57 | The working directory and commit history in git will be changed to reflect the snapshot of the project in that branch when switching branches. 58 | 59 | ### Merging 60 | 61 | Merging is the term that Git uses for combining branch data and history together. Git uses two different algorithms to perform a merge, fast-forward and three-way merge. 62 | 63 | * Fast-forward merge occurs when all the commits in the checked out branch are also in the branch that's being merged. If this is the case, we can say that the commit history of both branches doesn't diverge. 64 | * Three-way merge is performed when the history of the merging branches has diverged in some way, and there isn't a nice linear path to combine them via fast-forwarding. This happens when a commit is made on one branch after the point when both branches split. 65 | 66 | --- 67 | 68 | ## Credit 69 | 70 | * [Coursera - Introduction Git Github Week 2](https://www.coursera.org/learn/introduction-git-github/home/week/2) 71 | -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-four/README.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting and Debugging Techniques - Week 4 2 | 3 | ## Managing Computer Resources 4 | 5 | --- 6 | 7 | ## Managing Our Time 8 | 9 | --- 10 | 11 | ## Making Our Future Lives Easier 12 | 13 | --- 14 | 15 | ## Credit 16 | 17 | * [Coursera - Troubleshooting Debugging Techniques Week 4](https://www.coursera.org/learn/troubleshooting-debugging-techniques/home/week/4) 18 | -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-one/README.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting and Debugging Techniques - Week 1 2 | 3 | ## Introduction to Debugging 4 | 5 | ### What is debugging 6 | 7 | Problems may caused by the any of the following: 8 | 9 | * Hardware 10 | * The operating system 11 | * Applications running on the computer 12 | * The environment and configuration of the software 13 | 14 | **Troubleshooting** is the process of identifying, analyzing, and solving problems while **Debugging** is the process of identifying, analyzing, and removing bugs in a system. 15 | 16 | But generally, we say troubleshooting when we're fixing problems in the system running the application, and debugging when we're fixing the bugs in the actual code of the application. 17 | 18 | **Debuggers** let us follow the code line by line, inspect changes in variable assignments, interrupt the program when a specific condition is met, and more. 19 | 20 | There are different tools for debugging: 21 | 22 | * tcpdump and Wireshark: shows ongoing network connections, and help with analyzing the traffic going over the cables 23 | * ps, top, or free: shows the number and types of resources used in the system 24 | * strace: shows the system calls made by a program 25 | * ltrace: shows the library calls made by the software 26 | 27 | ### Problem Solving Steps 28 | 29 | 1. Gathering as much information from any of the existing documentation 30 | * The current state of things 31 | * What the issue is 32 | * When it happens 33 | * What the consequences are 34 | * Reproduction case, which is a clear description of how and when the problem appears 35 | 2. Find the root cause of the problem 36 | 3. Perform the necessary remediation 37 | * Workaround can be used when we could understand the problem just enough and try our users get back to work quickly 38 | * Ultimately, prevent the problem from occurring to save time in the future 39 | 40 | Also document the debugging process for the future referece. 41 | 42 | --- 43 | 44 | ## Understanding the Problem 45 | 46 | ### It Doesn't Work 47 | 48 | There are some common questions that we can ask a user that simply report something doesn't work: 49 | 50 | * What were you trying to do? 51 | * What steps did you follow? 52 | * What was the expected result? 53 | * What was the actual result? 54 | 55 | When debugging a problem, we want to consider the simplest explanations first and avoid jumping into complex or time-consuming solutions unless we really have to. 56 | 57 | After having a basic idea of what the problem is, figure out the root cause by applying a process of elimination, starting with the simplest explanations first and testing those until you can isolate the root cause. 58 | 59 | ### Creating a Reproduction Case 60 | 61 | A **reproduction case** is a way to verify if the problem is present or not. When trying to create a reproduction case, we want to find the actions that reproduce the issue, and we want these to be as simple as possible. The smaller the change in the environment and the shorter the list of steps to follow, the better. 62 | 63 | When debugging, the first step is to **read the logs**. Which logs to read, will depend on the operating system and the application that you're trying to debug. 64 | 65 | * On Linux, read system logs like /var/log/syslog and user-specific logs like the.xsession-errors file located in the user's home directory 66 | * On MacOs, on top of the system logs, go through the logs stored in the library logs directory 67 | * On Windows, use the Event Viewer tool to go through the event logs 68 | 69 | ### Finding the Root Cause 70 | 71 | Understanding the root cause is essential for performing the long-term remediation. 72 | 73 | Whenever possible, we should check our hypothesis in a __test environment__, instead of the production environment that our users are working with. 74 | 75 | ### Dealing with Intermitent Issues 76 | 77 | When dealing with intermitent issues, 78 | 79 | 1. Get more involved in what's going on, so that you understand when the issue happens and when it doesn't 80 | * Usually invovles going through logs or enabling debugging information 81 | 2. Look at different sources of information, like the load on the computer, the processes running at the same time, the usage of the network, and so on 82 | 83 | If a problem goes away by turning it off and on again, there's almost certainly a bug in the software, and the bug probably has to do with not managing resources correctly. 84 | 85 | --- 86 | 87 | ## Binary Searching a Problem 88 | 89 | ### What is binary search 90 | 91 | **Linear search** works but the longer the list, the longer it can take to search. 92 | 93 | If the list is **sorted**, we can use an alternative algorithm for searching called **binary search**. 94 | 95 | * It may take more time to sort an unsorted list to perform binary search 96 | * It can still make sense to do it if we're going to search through it several times 97 | * It doesn't make sense to sort the list and then use binary search to only find one element. In that case, using linear search is simpler and faster. 98 | 99 | ### Applying Binary Search in Troubleshooting 100 | 101 | In troubleshooting, we can apply **bisecting** to go through and test a long list of hypotheses. When doing this, the list of elements contains all the possible causes of the problem and we keep reducing the problem by half until only one option is left. 102 | 103 | When using Git for version control, we can use a __Git command called **bisect**__. Bisect receives two points in time in the Git history and repeatedly lets us try the code at the middle point between them until we find the commit that caused the breakage. 104 | 105 | ### Finding Invalid Data 106 | 107 | wc command - counts characters, words, and lines in a file 108 | head command - prints the first lines in the file, and the tail command to print the last lines 109 | 110 | --- 111 | 112 | ## Credit 113 | 114 | * [Coursera - Troubleshooting Debugging Techniques Week 1](https://www.coursera.org/learn/troubleshooting-debugging-techniques/home/week/1) 115 | -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-one/binary-search.py: -------------------------------------------------------------------------------- 1 | def binary_search(list, key): 2 | """Returns the position of key in the list if found, -1 otherwise. 3 | 4 | List must be sorted. 5 | """ 6 | left = 0 7 | right = len(list) - 1 8 | while left <= right: 9 | middle = (left + right) // 2 10 | 11 | if list[middle] == key: 12 | return middle 13 | if list[middle] > key: 14 | right = middle - 1 15 | if list[middle] < key: 16 | left = middle + 1 17 | return -1 -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-one/find_item.py: -------------------------------------------------------------------------------- 1 | def find_item(list, item): 2 | #Returns True if the item is in the list, False if not. 3 | if len(list) == 0: 4 | return False 5 | 6 | # In order to use binary search, list needs to be sorted 7 | list.sort() 8 | 9 | #Is the item in the center of the list? 10 | middle = len(list)//2 11 | if list[middle] == item: 12 | return True 13 | 14 | #Is the item in the first half of the list? 15 | if item < list[middle]: 16 | #Call the function with the first half of the list 17 | return find_item(list[:middle], item) 18 | else: 19 | #Call the function with the second half of the list 20 | return find_item(list[middle+1:], item) 21 | 22 | return False 23 | 24 | #Do not edit below this line - This code helps check your work! 25 | list_of_names = ["Parker", "Drew", "Cameron", "Logan", "Alex", "Chris", "Terry", "Jamie", "Jordan", "Taylor"] 26 | 27 | print(find_item(list_of_names, "Alex")) # True 28 | print(find_item(list_of_names, "Andrew")) # False 29 | print(find_item(list_of_names, "Drew")) # True 30 | print(find_item(list_of_names, "Jared")) # False -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-one/linear-search.py: -------------------------------------------------------------------------------- 1 | def linear_search(list, key): 2 | """If key is in the list returns its position in the list, 3 | otherwise returns -1.""" 4 | for i, item in enumerate(list): 5 | if item == key: 6 | return i 7 | return -1 -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting and Debugging Techniques - Week 3 2 | 3 | ## Why Programs Crash 4 | 5 | ### Systems That Crash 6 | 7 | Reduce the scope of the problem by starting with the actions that are easier and faster to check. 8 | 9 | 1. Try looking at the logs to see if there's any error that may point to what's happening 10 | 2. Try moving away the local configuration for the program and using the default configuration/reinstall the application 11 | 3. Try checking if it is some hardware component failure 12 | 13 | ### What to do when you can't fix the program 14 | 15 | Use **Wrappers** when the expected output and input formats don't match. A **Wrapper** is a function or program that provides a compatibility layer between two functions or programs so they can work well together. 16 | 17 | Run the application inside a **virtual machine** or maybe a **container** if there's another application that requires a different version of the same library or you can't change a certain configuration setting because it's required to access a different service. 18 | 19 | Deploy a **watchdog** when we can't find a way to stop an application from crashing but we can make sure that if it crashes it starts back again. This is a process that checks whether a program is running and when it's not starts the program again. To implement this, write a script that stays running in the background and periodically checks if the other program is running. 20 | 21 | Share good reproduction case and answer the questions wgeb reporting a bug by answering following questions. 22 | 23 | * What were you trying to do? 24 | * What were the steps you followed? 25 | * What did you expect to happen? 26 | * What was the actual outcome? 27 | 28 | ### Internal Server Error 29 | 30 | When a webpage on a Web server isn't working, 31 | 32 | 1. Try looking at logs 33 | * On Linux systems, logs are located in Bar log 34 | * Use the date command to check the current date 35 | 2. Use the netstat command which can give us information about our network connections depending on the flags passed 36 | * This command accesses different sockets that are restricted to route the administrator user on Linux - call it with sudo 37 | * Run commands as root, and then pass different flags with netstat 38 | * Use -n to print numerical addresses instead of resolving host names 39 | * Use L to only check out the sockets that are listening for connection 40 | * Use P to print the process ID and name to which each socket belongs 41 | * Since we are interested in port 80, connect the output to a grep command checking for colon 80 - the Web server is usually running on port 80, the default web serving port 42 | 43 | ## Code that Crashes 44 | 45 | ### Accessing Invalid Memory 46 | 47 | One common reason a program crashes is it's trying to access invalid memory. Accessing invalid memory means that the process tried to access a portion of the system's memory that wasn't assigned to it. 48 | 49 | In low-level languages like C or C++, the variables that store memory addresses are called pointers and the programmer needs to take care of requesting the memory that the program is going to use and then giving that memory back once it's not needed anymore. 50 | 51 | When a program tries to read or write to a memory address outside of the valid range, OS will raise an error like **segmentation fault** or **general protection fault**. 52 | 53 | Common programming errors that lead to segmentation faults or segfaults include, 54 | 55 | * Forgetting to initialize a variable 56 | * Trying to access a list element outside of the valid range 57 | * Trying to use a portion of memory after having given it back 58 | * Trying to write more data than the requested portion of memory can hold 59 | 60 | The **debugger** can give you a lot more detail on what the application is doing and why the memories invalid. For this to be possible, we'll need our program to be compiled with debugging symbols. This means that on top of the information that the computer uses to execute the program, the executable binary needs to include extra information needed for debugging, like the names of the variables and functions being used. 61 | 62 | Linux distributions like Debian or Ubuntu ships separate packages with the debugging symbols for all the packages in the distribution. Microsoft compilers can also generate debugging symbols in a separate PDB file. 63 | 64 | **Valgrind** can help when trying to understand problems related to handling invalid memory. Valgrind is a very powerful tool that can tell us if the code is doing any invalid operations no matter if it crashes are not. Valgrind lets us know if the code is accessing variables before initializing them. 65 | 66 | Valgrind is available on Linux and Mac OS, and Dr. Memory is a similar tool that can be used on both Windows and Linux. 67 | 68 | ### Unhandled Errors and Exceptions 69 | 70 | When a program comes across an unexpected condition that isn't correctly handled in the code, it will trigger errors or exceptions. 71 | 72 | In Python, following errors could occurs, 73 | 74 | * Index error if we tried to access an element after the end of a list 75 | * Type error or an attribute error if we try to take an action on a variable that wasn't properly initialized 76 | * Division by zero error if we tried to divide by zero 77 | 78 | When these failures happen, the interpreter that's running the program will print the following: 79 | 80 | * Type of error 81 | * Line that caused the failure 82 | * **Traceback** which shows the lines of the different functions that were being executed when the problem happened 83 | 84 | To find out where things are going wrong, we use **debugging tools** available for the application's language 85 | 86 | For a Python, program we can use the **BDB interactive debugger** which lets us do all the typical debugging actions like executing lines of code one-by-one or looking at how the variables change values. 87 | 88 | When we're trying to understand what's up with a misbehaving function on top of using debuggers, it's 89 | 90 | **print f debugging** is a common practice to trying to understand what's up with a misbehaving function on top of using debuggers. It adds statements that print data related to the codes execution to show the contents of variables, the return values of functions or metadata like the length of a list or size of a file. Taking a step further, the best approach is to add the messages in a way that can be easily enabled or disabled depending on whether we want the debug info or not. 91 | 92 | In Python, use the **logging module** which lets us set how comprehensive we want our code to be. 93 | 94 | ### Fixing Someone Else's Code 95 | 96 | Writing good comments is one of those good habits that pays off when trying to understand code written by others and also your past self. 97 | 98 | Another thing that can help to understand someone else's code is reading the tests associated to the code. 99 | 100 | ### Debugging a Segmentation Fault 101 | 102 | When an application crashes, it's useful to have a **core file** of the crash. 103 | 104 | **Core files** store all the information related to the crash so that we or someone else can debug what's going on. It's like taking a snapshot of the crash when it happens to analyze it later. 105 | 106 | In order to enable OS to generate core file, 107 | 108 | 1. Run the ulimit command 109 | 2. Use the -c flat for core files 110 | 3. use the unlimited to state that we want core files of any size 111 | 112 | For the exercise show in the video, it debugs **off-by-one error** by 113 | 114 | 1. Generated a core file and check the file using LS-L command 115 | 2. Use gdb-c core to give it a core file and then example to tell it where the executable that crashed is located 116 | 3. Use the backtrace command to look at the full backtrace of the crash 117 | 4. Use the list command that shows the lines around the current one to get more contexts for the code that failed 118 | 5. Print the contents of the first element argv 0, and then the second element argv 1 119 | * Zero is never a valid pointer 120 | 121 | ## Handling Bigger Incidents 122 | 123 | ### Crashes in Complex Systems 124 | 125 | Below is troubleshoot strategies that can be used when handling crashes in complex systems, 126 | 127 | 1. Roll back 128 | * It is the best strategy to use when the new changes are supsected to be causing the issue 129 | * It restores the service back to health if it was the cause 130 | * It helps eliminate new change as a possible cause if doing the rollback doesn't help 131 | 2. Logs with useful information 132 | 3. Monitoring of what the service is doing and version control for qucik roll back if needed 133 | 4. Deploy new machines if needed 134 | 135 | ### Communication and Documentation During Incidents 136 | 137 | Forgetting to dcoument could: 138 | 139 | * Risk forgetting some important details 140 | * Wasting a lot of valuable time when the same issue is revisited 141 | 142 | Good document should contain the following: 143 | 144 | * Root cause 145 | * How you diagnose the problem and found that root cause 146 | * What you did to fix the issue and what needs to be done to prevent the problem from happening again 147 | 148 | ### Writing Effective Postmortems 149 | 150 | **Postmortems** are documents that describe details of incidence to help us learn from our mistakes. The goal of postmortems is to learn from what happened to prevent the same issue from happening again. 151 | 152 | In general postmortem should inclde include: 153 | 154 | * Details of what caused the issue 155 | * What the impact of the issue was 156 | * How it got diagnosed 157 | * Short-term remediation you applied 158 | * Long-term remediation you recommend 159 | 160 | --- 161 | 162 | ## Credit 163 | 164 | * [Coursera - Troubleshooting Debugging Techniques Week 3](https://www.coursera.org/learn/troubleshooting-debugging-techniques/home/week/3) 165 | -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting and Debugging Techniques - Week 2 2 | 3 | ## Lerning Objectives 4 | 5 | Rundown of the different reasons that can make things run slowly by: 6 | 7 | * Looking into what causes slow scripts, slow computers, or slow systems 8 | * Looking into what tools are there to help identify the most common causes of slowness, and apply solutions to improve the overall performance 9 | 10 | ## Understanding Slowness 11 | 12 | ### Why is my computer slow 13 | 14 | The general strategy for addressing slowness is to **identify the bottleneck** for addressing slowness in device, script, or system to run slowly. Some of the reaons could be: 15 | 16 | * Running out of the CPU time 17 | * Time spent reading data from disk waiting for data transmitted over the network 18 | * Moving data from disk to RAM 19 | 20 | Treat each part of the computer (CPU, memory, disk, newtwork bandwidth) as **finite resources**. 21 | 22 | Identifying the bottleneck allows us to manage the available resources more effectively. In order to find what is causing the bottleneck, **monitor** the usage of resources to know which of them are being exhausted. 23 | 24 | We can use the following commands on Linux to monitor the usage of resources: 25 | 26 | * ```top``` command on shows 27 | * Which currently running processes are using the most CPU time 28 | * Which currently running processes are using the most memory 29 | * Other load information related to the current state of the computer such as how many processes are running and how the CPU time or memory is being used 30 | * ```iotop``` command shows which processes are currently using the most disk IO usage and swap memory usage 31 | * ```iftop``` command shows which processes are currently using the most network bandwidth 32 | 33 | Therefore, steps to diagnose what's causing the computer to run slow would be: 34 | 35 | 1. Open one of above tools to check what's going on 36 | 2. Try to understand which resources the bottleneck and why 37 | 3. Plan how you're going to solve the issue 38 | 39 | ### How Computer Use Resources 40 | 41 | When thinking about making things faster, it's important to understand the different speeds of the parts involved when accessing data/variables: 42 | 43 | ```(slowest) data from the newtok < data from disk < data from memory < data from CPU's inernal memory (fastest)``` 44 | 45 | Create a **cache** when the same files are read from disk repeatedly by putting the same information directly into the process memory and avoid loading it from disk every time. A cache stores data in a form that's faster to access than its original form. 46 | 47 | Examples of caches in IT includes: 48 | 49 | * Web proxy - stores websites, images, or videos that are accessed often by users behind the proxy 50 | * DNS - implements a local cache for the websites they resolve so they don't need to query from the internet every time someone asks for their IP address 51 | 52 | The operating system performs **swap** when there is not enough memory - the operating system will put the parts of the memory that aren't currently in use onto the hard drive in a space called swap. Reading and writing from disk is much slower than reading and writing from RAM. So when the swapped out memory is requested by an application, it will take a while to load it back. 53 | 54 | So what do you do if you find that your machine is slow because it's spending a lot of time swapping? 55 | 56 | There are three possible reasons computer slows down due to constatnt swapping: 57 | 58 | 1. There are too many open applications 59 | * Solve by closing the ones that aren't needed 60 | 2. The available memory is too small for the amount that computer is using 61 | * Solve by add more RAM to the computer 62 | 3. One of the running programs may have a **memory leak**, causing it to take all the available memory 63 | * A memory leak means that memory which is no longer needed is not getting released 64 | 65 | ### Possible Causes for Slowness 66 | 67 | Below are some of the possible causes for slowness: 68 | 69 | * Too many applications are configured to start on a boot and the computer slows at start up 70 | * Solve by disabling applications that aren't really needed on a startup 71 | * Computer slows after days of running and the problem solved by a reboot - an application keeps some state while running and takes the memory 72 | * Solve by modifying the code to frees up the memory after using it 73 | * If it can't be fixed, fix by scheduling a restart 74 | * Computer slows after days of running and the problem isn't solved by a reboot - the files that an application handles grown too large 75 | * Solve by modifying the code to hanlde the large file 76 | * If it can't be fixed, try to **reduce** the size of the files involved - **sharding** 77 | * Only a subset of computer is slow 78 | * Check if there is any difference in configuration 79 | * Computer slows due to a lot of reads and writes on the network-mounted file system 80 | * Solve by creating a cache 81 | * Hardware failures 82 | * Malicious software 83 | 84 | ### Slow Web Server 85 | 86 | ```Client complains about a web site loading slow``` 87 | 88 | First, reproduce the case and quantify the slowness with a tool called ```ab``` which stands for **Apache Benchmark** tool. ```ab``` checks if a website is behaving as expected or not by making a bunch of requests and summarize the results once it's done. 89 | 90 | Below command gets the average timing of 500 requests, and then pass site.example.com for the measurement: 91 | 92 | ```Shell 93 | ab -n 500 site.example.com 94 | ``` 95 | 96 | Next, connect to the web server with ```ssh``` 97 | 98 | ```Shell 99 | ssh SERVERNAME 100 | ``` 101 | 102 | Start by looking at the output of ```top``` and see if there's possible casue. 103 | 104 | For this scenario, 105 | 106 | * ```top``` output shows that the load average is around 30 107 | * The load average on Linux shows how much time the processor is busy at a given minute 108 | * For a single core proccessor with any number above 1 and a dual core processor with any number 2 is being **overloaded** 109 | * 30 means that there were more processes waiting for processor time than the processor had to give 110 | * There are multiple ```ffmpeg``` processes running which uses all of the available CPU 111 | * ```ffmpeg``` program is used for video transcoding and a CPU intensive process 112 | 113 | At this point, one thing we can try is to **change the processes priorities** so that the web server takes precedence which can be done with the following commands: 114 | 115 | * ```nice``` command for starting a process with a different priority 116 | * ```renice``` command for changing the priority of a process that's already running 117 | 118 | Since there are multiple ffmpeg proccesses running, let's change priority of all of ```ffmpeg``` at once instead of running ```renice``` one by one with the following shell script: 119 | 120 | ```Shell 121 | for pid in $(pidof ffmpeg); do renice 19; $pid; done 122 | ``` 123 | 124 | Above shell script: 125 | 126 | 1. Iterates over the output of the ```pidof``` command with a for loop 127 | * ```pidof``` command receives the process name and returns all the process IDs that have that name 128 | 2. Calls ```renice``` for each of the process IDs 129 | * ```renice``` takes the new priority which is 19, the lowest possible priority, as the first argument, and the process ID to change as the second one 130 | 131 | Check if the problem has been solved by running ```ab```. **For this scenario, the website is still slow.** 132 | 133 | Transcoding processes are CPU intensive, and running them in **parallel** is overloading the computer. So one thing we could do is, modify the program to run the transcoding one at a time instead of all at the same time. 134 | 135 | To do that, we'll need to find out how these processes got started: 136 | 137 | 1. Look at the output of the ```ps``` command to get some more information about the processes 138 | 2. Call ```ps ax``` which shows us all the running processes on the computer, and connect the output of the command to ```less``` to be able to scroll through it 139 | 140 | ```Shell 141 | ps ax | less 142 | 143 | #search for ffmpeg from the output of ps using / which is the search key when using less 144 | /ffmpeg 145 | ``` 146 | 147 | Upon search, it shows that 148 | 149 | * There are multiple ffmpeg processes that are converting videos from the webm format to the mp4 format in static 150 | * We don't know where these videos are on the hard drive 151 | 152 | Use ```locate``` command to locate the file. 153 | 154 | ```Shell 155 | locate static/001.webm 156 | 157 | #locate command returns the following 158 | /srv/deploy_videos/static/001.webm 159 | ``` 160 | 161 | When there are multiple files, use ```grep``` command to check if any of the files contain a call to a searching term - in this case ```ffmpeg``` instead of checking them one by one. 162 | 163 | ```Shell 164 | grep ffmpeg * 165 | ``` 166 | 167 | The output of grep shows that ```deploy.sh``` script starts the ```ffmpeg``` processes in parallel using a tool called ```Daemonize``` that runs each program separately as if it were a daemon - address the issue. 168 | 169 | **However, modifying the script does NOT chnage the processes that are already running.** 170 | 171 | **Also we want to stop these processes but not cancel them completely**, as doing so would mean that the videos being converted right now will be incomplete. 172 | 173 | To avoid canceling the proccesses, use the ```killall``` command with the ```-STOP``` flag which sends a stop signal but doesn't kill the processes completely. 174 | 175 | ```Shell 176 | killall -STOP ffmpeg 177 | ``` 178 | 179 | After stopping them all, we now want to run these processes **one at a time**. This can be done by sending the ```CONT``` signal one by one after each proccess completes the task. This can be automated with the similar for loop used earlier. 180 | 181 | ```Shell 182 | for pid in $(pidof ffmpeg); do while kill -CONT $pid; do sleep 1; done; done; 183 | ``` 184 | 185 | Check if the problem has been solved by running ```ab```. **For this scenario, the problem is fixed at this point.** 186 | 187 | ## Slow Code 188 | 189 | ### Writing Efficient Code 190 | 191 | As a rule, aim first to write code that is to avoid bugs 192 | 193 | * Readable 194 | * Easy to maintain 195 | * Easy to understand 196 | 197 | If the code is not fast enough, tru to optimize by eliminating **expensive actions**, which are those that take a long time to complete. 198 | 199 | * Trying to optimize every second out of a script is probably not worth the effort though 200 | 201 | The first step is to keep in mind that we can't really make our computer go faster. If we want our code to finish faster, we need to make our computer do less work, and to do this, we'll have to 202 | 203 | 1. Avoid doing work that isn't really needed 204 | 2. Avoid calculating data repeatedly by using the right data structures and storing the data 205 | 3. Reorganize the code so that the computer can stay busy while waiting for information from slow sources like disk or over the network 206 | 207 | Use a **profiler** tool to measure the resources that the code is using for better understanding of what's going on. It allows us to see which functions are called by our program, how many times each function was called and how much time are programs spent on each of them. 208 | 209 | Because of how profilers work, they are specific to each programming language 210 | 211 | * C program uses gprof to analyze 212 | * Python uses c-Profile module to analyze 213 | 214 | ### Using the Right Data Structures 215 | 216 | Using an appropriate data structures can help us avoid unnecessary expensive operations and create efficient scripts. 217 | 218 | **Lists** are sequences of elements that can 219 | 220 | * Add, remove, or modify the elements in them 221 | * Iterate through the whole list to operate on each of the elements 222 | 223 | When adding or removing elements, 224 | 225 | * Adding or removing elements **at the end** is fast 226 | * Adding or removing elements in the middle can be slow because all the elements that follow need to be repositioned 227 | 228 | When accesing or finding an element, 229 | 230 | * Accessing an element in a specific position in the list is fast 231 | * Acceessing an element in an unknown position requires going through the whole list 232 | * This can be super slow if the list is long 233 | 234 | **Dictionary** store key value pairs that can 235 | 236 | * Add data by associating a value to a key 237 | * Retrieve a value by looking up a specific key 238 | 239 | When accesing or finding an element, 240 | 241 | * Looking up keys is very fast O(n) 242 | 243 | In summary, 244 | 245 | * If you need to access elements by position or will always iterate through all the elements, use a list 246 | * If we need to look up the elements using a key, use a dictionary 247 | 248 | Another thing that we might want to think twice about is creating copies of the structures that we have in **memory**. 249 | 250 | ### Expensive Loops 251 | 252 | When using a loop, avoid doing expensive actions inside of the loop. If an expensive operation is performed inside a loop, you multiply the time it takes to do the expensive operation by the amount of times you repeat the loop. 253 | 254 | * Make sure that the list of elements that you're iterating through is only as long as you really need it to be 255 | * Break out of the loop once the data is found 256 | 257 | --- 258 | 259 | ## When Slowness Problems Get Complex 260 | 261 | ### Parallelizing Operations 262 | 263 | Operatiing in **parallel** is one way to run scripts more efficiently. 264 | 265 | * **Concurrency** is dedicated to how we write programs that do operations in parallel 266 | 267 | When data needs to shared while using the OS to split the work and the processes, use **threads**. 268 | 269 | * These processes don't share any memory by default 270 | 271 | __Threads let us run parallel tasks inside a process__. This allows threads to share some of the memory with other threads in the same process. 272 | 273 | To handle threads, modify the code to create and handle the thread. 274 | 275 | * In Python, use the **Threading** or **AsyncIO** modules (each programming language hsa different implementations) 276 | 277 | These modules let us specify which parts of the code we want to run in separate threads or as separate asynchronous events, and how we want the results of each to be combined in the end 278 | 279 | If your script is mostly just waiting on input or output, also known as I/O bound, it might matter if it's executed on one processor or eight. 280 | 281 | --- 282 | 283 | ## Credit 284 | 285 | * [Coursera - Troubleshooting Debugging Techniques Week 2](https://www.coursera.org/learn/troubleshooting-debugging-techniques/home/week/2) 286 | -------------------------------------------------------------------------------- /troubleshooting-and-debugging-technique/week-two/thumbnail_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from concurrent import futures 4 | 5 | import argparse 6 | import logging 7 | import os 8 | import sys 9 | 10 | import PIL 11 | import PIL.Image 12 | 13 | from tqdm import tqdm 14 | 15 | def process_options(): 16 | kwargs = { 17 | 'format': '[%(levelname)s] %(message)s' 18 | } 19 | 20 | parser = argparse.ArgumentParser( 21 | description="Thumbnail generator", 22 | fromfile_prefix_chars='@' 23 | ) 24 | parser.add_argument('--debug', action='store_true') 25 | parser.add_argument('-v', '--verbose', action='store_true') 26 | 27 | def progress_bar(files): 28 | return tqdm(files, desc='Processing', total=len(files), dynamic_ncols=True) 29 | 30 | def main(): 31 | 32 | process_options() 33 | 34 | # Create the thumbnail directory 35 | if not os.path.exists('thumbnails'): 36 | os.mkdir('thumbnails') 37 | 38 | excutor = futures.ThreadPoolExecutor() 39 | for root, _, files in os.walk('images'): 40 | for basename in progress_bar(files): 41 | if not basename.endswith('.jpg'): 42 | continue 43 | excutor.submit(process_file, root, basename) 44 | print('Waiting for all threads to finish.') 45 | excutor.shutdown() 46 | return 0 47 | 48 | if __name__ == "__main__": 49 | sys.exit(main()) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-five/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 5 2 | 3 | ## Simple Tests 4 | 5 | ### What is testing? 6 | 7 | Software testing is a process of evaluating computer code to determine whether or not it does what you expect it to do. 8 | 9 | Writing tests can help wth: 10 | 11 | * Eliminate bugs 12 | * Improve the reliability and the quality of automation 13 | 14 | ### Manual Testing and Automated Testing 15 | 16 | The **manual testing** is the most basic way of testing a script by running the test with different parameters and see if it returns the expected values. 17 | 18 | The automatic testing is codifying tests into its own software and code that can be run to verify that the programs do what we expect them to do. The goal of automatic testing is to automate the process of checking __if the returned value matches the expectations__. 19 | 20 | --- 21 | 22 | ## Unit Tests 23 | 24 | ### Unit Tests 25 | **Unit tests** are used to verify that __small isolated parts of a program are correct__. 26 | 27 | An important characteristic of a unit test is **isolation**. 28 | 29 | Unit test should only test the unit of code they target, the function or method that's being tested. This ensures that any success or failure of the test is caused by the behavior of the unit in question and doesn't result from some external factor like the network being down or a database server being unresponsive. 30 | 31 | ### Writing Unit Tests in Python 32 | 33 | Use the **unittest module** which includes classes and methods for creating unit tests to write unit tests in Python. 34 | 35 | ### Edge Cases 36 | 37 | Edge cases are inputs that produce unexpected results, and are found at the extreme ends of the ranges of input that programs could work with. 38 | 39 | Some of the edge case examples for a function that expects a number include: 40 | * Passing zero to 41 | * Negative numbers 42 | * Extremely large numbers 43 | 44 | --- 45 | 46 | ## Other Test Comcepts 47 | 48 | ### Black Box vs. White Box 49 | 50 | * **White-box** testing, clear-box or transparent testing relies on the test creators knowledge of the software being tested to construct the test cases. 51 | * White-box tests are helpful because a test writer can use their knowledge of the source code to create tests that cover most of the ways that the program behaves. 52 | 53 | 54 | * **Black-box** tests or opaque testing are written with an awareness of what the program is supposed to do, its requirements or specifications, but not how it does it. 55 | * Black-box tests are useful because they don't rely on the knowledge of how the system works. This means their test cases are less likely to be biased by the code. They usually cover situations not anticipated by the programmer who originally wrote the script. 56 | 57 | ### Other Test Types 58 | 59 | For different tests, separate **test environment** for test may be needed to runs a test version of our software that we're trying to verify. 60 | 61 | * **Integration tests** take the individual modules of code that unit test verify then combine them into a group to test. It verify that the interactions between the different pieces of code in integrated environments are working as expected. 62 | * **Regression test** is a variant of unit tests. It is usually written as part of a debugging and troubleshooting process to verify that an issue or error has been fixed once it's been identified. Regression tests are useful part of a test suite because they ensure that the same mistake doesn't happen twice. 63 | * **Smoke test** sometimes called build verification test, get their name from a concept that comes from testing hardware equipment. 64 | * Smoke test for a web service would be to check if there's a service running on the corresponding port 65 | * Smoke test for an automation script would be to run it manually with some basic input and check that the script finishes successfully 66 | * **Load tests** verify that the system behaves well when it's under significant load. 67 | 68 | Taking together a group of tests of one or many kinds is commonly referred to as a **test suite**. 69 | 70 | ### Test-Driven Development 71 | 72 | **Test-driven development** or TDD is a process that calls for creating the test before writing the code. 73 | 74 | **Continuous Integration** is a process that combines version control system and developmement processes. When engineers submit their code, it's integrated into the main repository and tests are automatically run against it to spot bugs and errors. 75 | 76 | --- 77 | 78 | ## Errors and Exceptions 79 | 80 | ### The Try-Except Construct 81 | **Try-except construct** is useful when trying to handle possible errors that could happen instead of multiple if/else statements. 82 | 83 | The code in the except block is only executed if one of the instructions in the try block raise an error of the matching type. To use a try-except block, be aware of the errors that functions that we're calling might raise. 84 | 85 | ### Raising Errors 86 | 87 | --- 88 | 89 | ## Credit 90 | 91 | * [Coursera - Python Operating System Week 5](https://www.coursera.org/learn/python-operating-system/home/week/5) 92 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-five/charfreq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | def char_frequency(filename): 4 | """ 5 | Counts the frequency of each character in the given file. 6 | """ 7 | # First try to open the file 8 | try: 9 | f = open(filename) 10 | # code in the except block is only executed if one of the instructions in the try block raise an error of the matching type 11 | except OSError: 12 | return None 13 | 14 | # Now process the file 15 | characters = {} 16 | for line in f: 17 | for char in line: 18 | characters[char] = characters.get(char, 0) + 1 19 | f.close() 20 | return characters -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-five/rearrange.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | 5 | def rearrange_name(name): 6 | result = re.search(r"^([\w .]*), ([\w .]*)$", name) 7 | # to account edge cases 8 | if result is None: 9 | return name 10 | return "{} {}".format(result[2], result[1]) 11 | 12 | # from rearrange import rearrange_name 13 | 14 | # from keyword imports a function from a script in the python3 interpreter 15 | # importing it in this way, the function can be called without having to write the module name each time we want to call it 16 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-five/rearrange_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Writing Unit Tests in Python 4 | # from keyword imports a function from a script in the python3 interpreter 5 | # the function can be called without having to write the module name each time we want to call it 6 | from rearrange import rearrange_name 7 | # unittest module includes classes and methods for creating unit tests 8 | import unittest 9 | 10 | # Include the class that we want to inherit from in the parentheses 11 | # TestRearrange class inherits from TestCase 12 | class TestRearrange(unittest.TestCase): 13 | def test_basic(self): 14 | testcase = "Lovelace, Ada" 15 | expected = "Ada Lovelace" 16 | self.assertEqual(rearrange_name(testcase), expected) 17 | 18 | # test for an edge case 19 | def test_empty(self): 20 | testcase = "" 21 | expected = "" 22 | self.assertEqual(rearrange_name(testcase), expected) 23 | 24 | def test_double_name(self): 25 | testcase = "Hopper, Grace M." 26 | expected = "Grace M. Hopper" 27 | self.assertEqual(rearrange_name(testcase), expected) 28 | 29 | def test_one_name(self): 30 | testcase = "Voltaire" 31 | expected = "Voltaire" 32 | self.assertEqual(rearrange_name(testcase), expected) 33 | 34 | # Runs the test 35 | unittest.main() -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-five/validations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | def validate_user(username, minlen): 4 | assert type(username) == str, "username must be a string" 5 | if minlen < 1: 6 | raise ValueError("minlen must be at least 1") 7 | if len(username) < minlen: 8 | return False 9 | if not username.isalnum(): 10 | return False 11 | return True -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 4 2 | 3 | ## Data Streams 4 | 5 | ### Reading Data Interactively 6 | 7 | Use **input function** to prpmpt the user for a certain value which can be used in a script. 8 | 9 | ### Standard Streams 10 | 11 | **I/O streams** are the basic mechanism for performing input and output operations in your programs. 12 | 13 | Most operating systems supply three different I/O streams by default each with a different purpose which aren't restricted to just Python. 14 | 15 | * STDIN - the standard input stream which is a channel between a program and a source of input 16 | * STDOUT - the standard output stream which is a pathway between a program and a target of output 17 | * SDTERR - the standard error stream which displays output like standard out, but is used specifically as a channel to show error messages and diagnostics from the program 18 | 19 | ### Environment Variables 20 | 21 | A **shell** is a command line interface used to interact with your operating system. Python programs get executed inside a shell command-line environment. 22 | 23 | The **variable set** in that environment are another source of information that can be used in scripts. 24 | 25 | **Echo** is a command that prints texts in Linux shell. In order to access the value of the variable in the shell, use a prefix and name of the variable with a dollar sign. 26 | 27 | ```Bash 28 | echo $PATH 29 | ``` 30 | 31 | In order to access environment variables, use the **Environ dictionary** provided by the OS module. 32 | 33 | ### Command-Line Arguments and Exit Status 34 | 35 | **Command line arguments** are parameters that are passed to a program when it started. 36 | 37 | * Allows a code of the script to be generic 38 | * Allows scripts to run automatically without requiring any interactive user input 39 | * Useful for system administration tasks as it allow us to specify the information that we want our program to use 40 | 41 | Parameter values can be accessed using the **argv** in the **sys** module. 42 | 43 | **Exit status** or **return code** is a value returned by a program to the shell. It provides another source of information between the shell and the programs that get executed inside of it. 44 | 45 | To see what the exit status of the last executed command was use the following commands: 46 | 47 | * Use ?$ to see the contents 48 | * Use wc to count the number of lines words and characters in a file 49 | 50 | 51 | ### Obtaining the Output of a System Command 52 | 53 | The **host** command converts a host name to an IP address and vice versa. 54 | 55 | The **decode function** applies an encoding to transform the bytes into a string 56 | * It uses a UTF-8 encoding by default 57 | 58 | ### Advanced Subprocess Management 59 | 60 | The usual strategy for modifying the environment of a child process is: 61 | 62 | 1. Copy the environment seen by the process 63 | 2. Do any necessary changes 64 | 3. Pass the changed environment as the environment that the child process will see 65 | 66 | The **copy method** of the OS environ creates a new dictionary that can be changed as needed without modifying the original environment 67 | The **path variable** indicates where the operating system will look for the executable programs 68 | 69 | Some of parameters that can be used with the run function includes: 70 | 71 | * The CWD parameter which changes the current working directory where the command will be executed. 72 | * Useful when working with a set of directories where you need to run a command on each of them. 73 | * The timeout parameter which cause the run function to kill the process if it takes longer than a given number of seconds to finish. 74 | * Useful if you're running a command that you know might get stuck. 75 | * The shell parameter which if set to true, Python will first execute an instance of the default system shell and then run the given command inside of it. 76 | * Useful if command line needs include variable expansions and other shell operations. 77 | 78 | --- 79 | 80 | ## Processing Log Files 81 | 82 | ### Filtering Log Files with Regular Expressions 83 | The usual technique to operate on files is to: 84 | 85 | 1. Call the open function which returns a file object 86 | 2. Iterate through each of its lines using a for-loop 87 | 88 | For performance reasons, when files are large, it's generally a good practice to read them line by line instead of loading the entire contents into memory. 89 | 90 | ## Credit 91 | 92 | * [Coursera - Python Operating System Week 4](https://www.coursera.org/learn/python-operating-system/home/week/4) 93 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/check_cron.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | 4 | logfile = sys.argv[1] 5 | with open(logfile) as f: 6 | for line in f: 7 | if "CRON" not in line: 8 | # continue keyword tells the loop to go to the next element 9 | continue 10 | pattern = r"USER \((\w+)\)$" 11 | result = re.search(pattern, line) 12 | print(result) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/course-two-week-four.py: -------------------------------------------------------------------------------- 1 | 2 | # Standard Streams 3 | # STDIN - the standard input stream which is a channel between a program and a source of input 4 | # STDOUT - the standard output stream which is a pathway between a program and a target of output 5 | # SDTERR - the standard error stream which displays output like standard out, but is used specifically as a channel to show error messages and diagnostics from the program 6 | data = input("This comes from STDIN: ") 7 | print("We are writing it to STDOUT: " + data) 8 | # print("We are generating an error to STDERR: " + data + 1) 9 | 10 | # ----------------------------------------------------------------------------------------- 11 | 12 | # Environment Variables 13 | # os module provides environ dictionary 14 | import os 15 | 16 | # environ variables allows us to access environment variables 17 | # get function specifies a default value when the key that we're looking for isn't in the dictionary 18 | print("HOME: " + os.environ.get("HOME", "")) 19 | print("SHELL: " + os.environ.get("SHELL", "")) 20 | 21 | # export FRUIT=Pineapple 22 | print("FRUIT: " + os.environ.get("FRUIT", "")) 23 | 24 | # ----------------------------------------------------------------------------------------- 25 | 26 | # Command-Line Arguments and Exit Status 27 | import sys 28 | print(sys.argv) 29 | 30 | import os 31 | 32 | # Receives a file name as a command line argument 33 | # filename = sys.argv[0] 34 | 35 | # Checks whether the file name exist or not 36 | # When the file doesn't exist, it creates it by writing a line to it 37 | # if not os.path.exists(filename): 38 | # with open(filename, "w") as f: 39 | # f.write("New file created\n") 40 | # When the file exist, our script print an error message and exits with an exit value of one 41 | # else: 42 | # print("Error, the file {} already exists!".format(filename)) 43 | # sys.exit(1) 44 | 45 | # ========================================================================================= 46 | 47 | # Python Subprocesses 48 | # Running System Commands in Python 49 | import subprocess 50 | print(subprocess.run(["date"])) 51 | print(subprocess.run(["sleep", "2"])) 52 | 53 | # ----------------------------------------------------------------------------------------- 54 | 55 | # Obtaining the Output of a System Command 56 | # The "host" command converts a host name to an IP address and vice versa 57 | # Stores the result in a variable by passing the capture_output=True so that the result can be accessed 58 | result = subprocess.run(["host", "8.8.8.8"], capture_output=True) 59 | print(result.returncode) 60 | # OUTPUT: b'8.8.8.8.in-addr.arpa domain name pointer dns.google.\n' 61 | # b' indicates that the output is an array of bytes 62 | print(result.stdout) 63 | # Decode function applies an encoding to transform the bytes into a string 64 | # It uses a UTF-8 encoding by default 65 | print(result.stdout.decode().split()) 66 | 67 | # Executes the rm command to remove file that doesn't exist 68 | result = subprocess.run(["rm", "does_not_exist"], capture_output=True) 69 | print(result.returncode) 70 | # stdout prints empty value since there is an error 71 | print(result.stdout) 72 | # stderr prints error value as the value can be accessed through the stderr attribute 73 | print(result.stderr) 74 | 75 | # ----------------------------------------------------------------------------------------- 76 | 77 | # Advanced Subprocess Management 78 | # The copy method creates a new dictionary that can be changed as needed without modifying the original environment 79 | my_env = os.environ.copy() 80 | # The path variable indicates where the operating system will look for the executable programs 81 | # Joins /opt/myapp and the old value of the path variable to the path separator 82 | my_env["PATH"] = os.pathsep.join(["/opt/myapp/", my_env["PATH"]]) 83 | # Calls the myapp command, setting the end parameter to the new environment 84 | result = subprocess.run(["myapp"], env=my_env) 85 | 86 | # ========================================================================================= 87 | 88 | # Filtering Log Files with Regular Expressions 89 | import re 90 | 91 | logfile = sys.argv[1] 92 | with open(logfile) as f: 93 | for line in f: 94 | if "CRON" not in line: 95 | # continue keyword tells the loop to go to the next element 96 | continue 97 | pattern = r"USER \((\w+)\)$" 98 | result = re.search(pattern, line) 99 | print(result) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/create_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | # Receives a file name as a command line argument 4 | filename=sys.argv[1] 5 | 6 | # Checks whether the file name exist or not 7 | # When the file doesn't exist, it creates it by writing a line to it 8 | if not os.path.exists(filename): 9 | with open(filename, "w") as f: 10 | f.write("New file created\n") 11 | # When the file exist, our script print an error message and exits with an exit value of one 12 | else: 13 | print("Error, the file {} already exists!".format(filename)) 14 | sys.exit(1) 15 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/hello.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Input function aloows for prompt the user for a cetain value which can be used in scripts 4 | name = input("Please enter your name: ") 5 | print("Hello, " + name) 6 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/myapp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | # The copy method creates a new dictionary that can be changed as needed without modifying the original environment 5 | my_env = os.environ.copy() 6 | 7 | # The path variable indicates where the operating system will look for the executable programs 8 | # Joins /opt/myapp and the old value of the path variable to the path separator 9 | my_env["PATH"] = os.pathsep.join(["/opt/myapp/", my_env["PATH"]]) 10 | 11 | # Calls the myapp command, setting the end parameter to the new environment 12 | result = subprocess.run(["myapp"], env=my_env) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/parameters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | print(sys.argv) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/seconds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/enb python3 2 | 3 | def to_seconds(hours, minutes, seconds): 4 | return hours*3600+minutes*60+seconds 5 | 6 | print("Welcome to this time converter") 7 | 8 | cont = "y" 9 | while(cont.lower() == "y"): 10 | hours = int(input("Enter the number of hours: ")) 11 | minutes = int(input("Enter the number of minutes: ")) 12 | seconds = int(input("Enter the number of seconds: ")) 13 | 14 | print("That's {} seconds.".format(to_seconds(hours, minutes, seconds))) 15 | print() 16 | cont = input("Do you want to do another conversion? [y to continue] ") 17 | 18 | print("Good bye!") -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-four/variables.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | 5 | print("HOME: " + os.environ.get("HOME", "")) 6 | print("SHELL: " + os.environ.get("SHELL", "")) 7 | # export FRUIT=Pineapple 8 | print("FRUIT: " + os.environ.get("FRUIT", "")) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-one/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 1 2 | 3 | ## Getting Ready for Python 4 | 5 | --- 6 | 7 | ## Runnig Python Locally 8 | 9 | --- 10 | 11 | ## Automating Tasks Through Programming 12 | 13 | 14 | --- 15 | 16 | ## Credit 17 | * [Coursera - Python Operating System Week 1](https://www.coursera.org/learn/python-operating-system/home/week/1) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-one/health_checks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from network import * 3 | import shutil 4 | import psutil 5 | def check_disk_usage(disk): 6 | """Verifies that there's enough free space on disk""" 7 | du = shutil.disk_usage(disk) 8 | free = du.free / du.total * 100 9 | return free > 20 10 | def check_cpu_usage(): 11 | """Verifies that there's enough unused CPU""" 12 | usage = psutil.cpu_percent(1) 13 | return usage < 75 14 | # If there's not enough disk, or not enough CPU, print an error 15 | if not check_disk_usage('/') or not check_cpu_usage(): 16 | print("ERROR!") 17 | elif check_localhost() and check_connectivity(): 18 | print("Everything ok") 19 | else: 20 | print("Network checks failed") -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-one/network.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | import socket 4 | def check_localhost(): 5 | localhost = socket.gethostbyname('localhost') 6 | if localhost == '127.0.0.1': 7 | return True 8 | return False 9 | def check_connectivity(): 10 | request = requests.get('http://www.google.com') 11 | response = request.status_code 12 | if response == 200: 13 | return True 14 | return False -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-seven/final.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/using-python-to-interact-with-the-operating-system/week-seven/final.sh -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 6 2 | 3 | ## Interacting with the Command Line Shell 4 | 5 | ### Basic Linux Commands 6 | A lot of commands come from Unix. The philosophy when designing how these programs should behave was that they should do one thing and do it very well. Which means there will be a lot of commands, each for doing specific thing. 7 | 8 | Managing files and directories 9 | 10 | * cd directory: changes the current working directory to the specified one 11 | * pwd: prints the current working directory 12 | * ls: lists the contents of the current directory 13 | * ls directory: lists the contents of the received directory 14 | * ls -l: lists the additional information for the contents of the directory 15 | * ls -a: lists all files, including those hidden 16 | * ls -la: applies both the -l and the -a flags 17 | * mkdir directory: creates the directory with the received name 18 | * rmdir directory: deletes the directory with the received name (if empty) 19 | * cp old_name new_name: copies old_name into new_name 20 | * mv old_name new_name: moves old_name into new_name 21 | * touch file_name: creates an empty file or updates the modified time if it exists 22 | * chmod modifiers files: changes the permissions for the files according to the provided modifiers; we've seen +x to make the file executable 23 | * chown user files: changes the owner of the files to the given user 24 | * chgrp group files: changes the group of the files to the given group 25 | 26 | Operating with the content of files 27 | 28 | * cat file: shows the content of the file through standard output 29 | * wc file: counts the amount of characters, words, and lines in the given file; can also count the same values of whatever it receives via stdin 30 | * file file: prints the type of the given file, as recognized by the operating system 31 | * head file: shows the first 10 lines of the given file 32 | * tail file: shows the last 10 lines of the given file 33 | * less file: scrolls through the contents of the given file (press "q" to quit) 34 | * sort file: sorts the lines of the file alphabetically 35 | * cut -dseparator -ffields file: for each line in the given file, splits the line according to the given separator and prints the given fields (starting from 1) 36 | 37 | Additional commands 38 | 39 | * echo "message": prints the message to standard output 40 | * date: prints the current date 41 | * who: prints the list of users currently logged into the computer 42 | * man command: shows the manual page of the given command; manual pages contain a lot of information explaining how to use each command (press "q" to quit) 43 | * uptime: shows how long the computer has been running 44 | * free: shows the amount of unused memory on the current system 45 | 46 | ### Redirecting Streams 47 | * Redirection - the process of sending a stream to a different destination 48 | * Provided by OS 49 | * Use '>', greater symbol for redirect an output 50 | 51 | ```Bash 52 | ./stdout_example.py 53 | ./stdout_example.py > new_file.txt 54 | ``` 55 | 56 | After the second line, stoud.example.py is outputted in new_file.txt instead of inside of the terminal via STDOUT. 57 | 58 | Be careful not to overwrite an existing file as redirection overwrites destination each time we perform a redirection of STDOUT. 59 | 60 | * Append - sends a stream to a different destination and adds onto the existiing destination instead of overwriting it 61 | * Use '>>', double greater symbol to append 62 | 63 | ```Bash 64 | ./stdout_example.py 65 | ./stdout_example.py >> new_file.txt 66 | ``` 67 | 68 | These are the redirectors that we can use to take control of the streams of our programs 69 | 70 | * command > file: redirects standard output, overwrites file 71 | * command >> file: redirects standard output, appends to file 72 | * command < file: redirects standard input from file 73 | * command 2> file: redirects standard error to file 74 | 75 | ### Pipes 76 | Pipe, |, connects multiple scripts, commands, or other programs together into a data processing pipeline. Pipes connect the output of one program to the input of another. 77 | * command1 | command2: connects the output of command1 to the input of command2 78 | 79 | Python scripts can be used with pipelines too. Python can read from standard input using the stdin file object provided by the sys module. 80 | 81 | ### Signalling Processes 82 | Signals are tokens delivered to running processes to indicate a desired action. 83 | 84 | These are some commands that are useful to know in Linux when interacting with processes. 85 | 86 | * ps: lists the processes executing in the current terminal for the current user 87 | * ps ax: lists all processes currently executing for all users 88 | * ps e: shows the environment for the processes listed 89 | * kill PID: sends the SIGINT signal to the process identified by PID 90 | * fg: causes a job that was stopped or in the background to return to the foreground 91 | * bg: causes a job that was stopped to go to the background 92 | * jobs: lists the jobs currently running or stopped 93 | * top: shows the processes currently using the most CPU time (press "q" to quit) 94 | 95 | --- 96 | 97 | ## Bash Scripting 98 | 99 | ### Creating Bash Scripts 100 | 101 | **Bash** is the most commonly used shell on Linux. Bash is not only the interpreter that runs our commands, it's also a __scripting language__. 102 | 103 | * ps command can list all the current running processes 104 | * free command can show you the amount of free memory 105 | * uptime command can tell you how long the computer has been on 106 | 107 | ### Using Variables and Globs 108 | 109 | **Environment variables** are set in the environment in which the command is executing. 110 | 111 | * = sign is used to set an environment variables 112 | * There can be no spaces between the name of the variable and the equal sign, or between the equal sign and the value 113 | * Any variable that you define in a script or in the command line is local to the environment where it was defined 114 | * $ prefix is used to access the value of a variable 115 | * export command to export commands from that environment 116 | 117 | **Globs** are characters that allow us to create list of files. The star and question mark are the most common globs. 118 | * using a \* in the command line will match all filenames that follow the format that we specify 119 | * using a ? sign in the command line will match exactly one character instead of any amount of characters 120 | * ? can be repeated as many times as we need 121 | 122 | ### Conditional Execution in Bash 123 | 124 | In bash scripting, the condition used is based on the exit status of commands, $? 125 | * In bash scripting an exit value of zero means success. 126 | 127 | **Test** is a command that evaluates the conditions received and exits with zero when they are true and with one when they're false. 128 | 129 | --- 130 | 131 | ## Advanced Bash Concepts 132 | 133 | ### While Loops in Bash Scripts 134 | The loops in Bash starts with the **do** keyword and finishes with a **done** keyword. 135 | 136 | To increment the value of the variable N, we're using a bash construct of double parentheses that lets us do arithmetic operations with our variables. 137 | 138 | Below is an example while loop that runs up to 5 times: 139 | 140 | ```Bash 141 | n=1 142 | while [ $n -le 5 ]; do 143 | echo "Iteration number $n" 144 | ((n+=1)) 145 | done 146 | ``` 147 | 148 | The value of a command line argument can be accessed by using the $1. In Python, we get the same information using sys.argv[1] 149 | 150 | When rerunning scripts due to some kind of failure, sleep command may needed to wait a bit before trying again. 151 | 152 | ### For Loops in Bash Scripts 153 | In Bash, we construct sequences of data by listing the elements with spaces in between. In Python, the sequences are data structures like a list or a tuple or a string. 154 | 155 | ```Bash 156 | for fruit in peach orange apple; do 157 | echo "I like $fruit!" 158 | done 159 | ``` 160 | To work with a list of file, we can use globs like star and question mark to create lists of files. These lists are separated by spaces and so we can use them in our loops to iterate over a list of files that match a criteria. 161 | 162 | * Basename command takes a filename and an extension and then returns the name without the extension. 163 | * Surround file variable with double-quotes to allow the command to work even if the file has spaces in its name. 164 | * This is a good practice in Bash scripts when dealing with file names or any variables that could include spaces. 165 | * It is also good idea to first run the script without actually modifying the file system to catch any possible bugs that the script might have. 166 | * Add an echo in front of the MV command 167 | 168 | ### Choosing Between Bash and Python 169 | Use Bash when we're operating with files and system commands, as long as what we're doing is simple enough that the script is self-explanatory. 170 | 171 | Avoid using Bash as it becomes hard to understand what the script is doing. It's better to write it in a more general scripting language like Python. 172 | 173 | Also Bash scripts aren't as flexible or robust as Python and some commands might not be present on certain operating system. 174 | 175 | --- 176 | 177 | ## Credit 178 | * [Coursera - Python Operating System Week 6](https://www.coursera.org/learn/python-operating-system/home/week/6) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/check-localhost.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if grep "127.0.0.1" /etc/hosts; then 4 | echo "Everything ok" 5 | else 6 | echo "ERROR! 127.0.0.1 is not in /etc/hosts" 7 | fi -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/fruits.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for fruit in peach orange apple; do 4 | echo "I like $fruit!" 5 | done -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/gather-information.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | line="-------------------------------------------------------------------" 4 | 5 | echo "Starting time at: $(date)"; echo $line 6 | 7 | echo "UPTIME"; uptime; echo $line 8 | 9 | echo "FREE"; free; echo $line 10 | 11 | echo "WHO"; who; echo $line 12 | 13 | echo "Finshing at: $(date)" -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/old_website/about.HTM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/using-python-to-interact-with-the-operating-system/week-six/old_website/about.HTM -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/old_website/contact.HTM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/using-python-to-interact-with-the-operating-system/week-six/old_website/contact.HTM -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/old_website/footer.HTM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/using-python-to-interact-with-the-operating-system/week-six/old_website/footer.HTM -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/old_website/index.HTM: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jeremymaya/google-it-automation-with-python/19aaee57055c600b23a65392ca7471cdfca88500/using-python-to-interact-with-the-operating-system/week-six/old_website/index.HTM -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/old_website/rename.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for file in *.HTM; do 4 | name=$(basename "$file" .HTM) 5 | echo mv "$file" "$name.html" 6 | done -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/random-exit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import random 5 | 6 | # Uses random randint to generate a value between zero and three 7 | value=random.randint(0, 3) 8 | # Prints the selected value and exits with it 9 | print("Returning: " + str(value)) 10 | sys.exit(value) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/retry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | n=0 4 | command=$1 5 | while ! $command && [ $n -le 5 ]; do 6 | sleep $n 7 | ((n=n+1)) 8 | echo "Retry #$n" 9 | done; -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/toploglines.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for logfile in /var/log/*log; do 4 | echo "Processing: $logfile" 5 | cut -d' ' -f5- $logfile | sort | uniq -c | sort -nr | head -5 6 | done -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-six/while.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | n=1 4 | while [ $n -le 5 ]; do 5 | echo "Iteration number $n" 6 | ((n+=1)) 7 | done -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-three/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 3 2 | 3 | ## Regular Expressions 4 | ### What is regualr expressions? 5 | A regular expression, also known as regex or regexp, is essentially a search query for text that's expressed by __string pattern__. It allows for searching a text for strings matching a specific pattern. 6 | 7 | When a particular piece of text searched using regex, anything that matches a regular expression pattern you specified, is returned as a result of the search. 8 | 9 | ### Why use regular expressions? 10 | Regex allows for flexibility in string searching and avoids using brittle solution such as index function. 11 | 12 | ### Basic Matching with grep 13 | Grep grep works by printing out any line that matches the passed query. 14 | 15 | grep __case sensitive__ example to search for "thon": 16 | ```bash 17 | grep thon /usr/share/dict/words 18 | ``` 19 | 20 | grep __case insensitive__ example to search for "thon": 21 | ```bash 22 | grep -i thon /usr/share/dict/words 23 | ``` 24 | 25 | Reserved characters to process an irregular expressions in regex: 26 | * ".", dot, matches any character which means that it can be used as a wildcard that can be replaced by any other character in the results. 27 | * "^", caret or circumflex, indicates the beginning of the line. 28 | * "$", dollar, indicates the end of the line. 29 | 30 | grep . example to search for a string mathching with "l.rts": 31 | ```bash 32 | grep l.rts /usr/share/dict/words 33 | ``` 34 | 35 | grep cafrat example to search for a string starting with "fruit": 36 | ```bash 37 | grep ^fruit /usr/share/dict/words 38 | ``` 39 | 40 | grep $ example to search for a string ending wiht "cat": 41 | ```bash 42 | grep cat$ /usr/share/dict/words 43 | ``` 44 | 45 | --- 46 | 47 | ## Basic Regular Expressions 48 | ### Simple Matching in Python 49 | * The "r" at the beginning of the pattern indicates that this is a rawstring. 50 | * Always use rawstrings for regular expressions in Python. 51 | * __None__ is a special value that Python uses that show that there's none actual value there. 52 | * The **match attribute** always has a value of the actual sub string that match the search pattern. 53 | * The **span attribute** indicates the range where the sub string can be found in the string. 54 | * Additional options to the search function can be added as a third parameter. 55 | * The re.IGNORECASE option returns a match that is case insensitive 56 | 57 | ### Wildcards and Character Classes 58 | A **wildcard** can match more than one character. 59 | * . in a regular expressions as a special character that can match any character. 60 | * Using a . is the broadest possible wildcard because it matches absolutely any character. 61 | 62 | Character classes are written inside square brackets, []: 63 | * It list the characters to match inside of the brackets 64 | * A range of characters can be defined using a dash 65 | * Use a ^, circumflex, inside the square brackets to match any characters that aren't in a group. 66 | * Use a |, pipe symbol to match either one expression or another 67 | 68 | The **search function** returns the __first matching string only__ when there are multiple matches. Use the **findall function** provided by the re module to get __all possible matches__. 69 | 70 | ### Repetition Qualifiers 71 | Repeated matches can be searched using the expressions below: 72 | * Use a . followed by * to matches any character repeated as many times as possible including zero - greedy behavior. 73 | * Use a +, plus character, to match one or more occurrences of the character that comes before it. 74 | * Use a ?, question mark symbol, for either zero or one occurrence of the character before it. 75 | * It is used to specified optional characters 76 | 77 | ### Escaping Characters 78 | A pattern that includes a \ could be escaping a special regex character or a special string character 79 | * Use a \, escape character, to match one of the special characters 80 | * Use a \w to match any alphanumeric character including letters, numbers, and underscores 81 | * Use a \d to match digits 82 | * Use a \s for matching whitespace characters like space, tab or new line 83 | * Use a \b for word boundaries 84 | 85 | --- 86 | 87 | ## Advanced Regular Expressions 88 | ### Capturing Groups 89 | Use parentheses to capture groups which are portions of the pattern that are enclosed in 90 | * The group method returns a tuple of two elements 91 | * Use indexing to access these groups 92 | * The first element contains the text matched by the entire regular expression 93 | * Each successive element contains the data that was matched by every subsequent match group 94 | 95 | ### More on Repetition Qualifiers 96 | * Use {}, curly brackets and one or two numbers to specify a range with numeric repetition qualifiers. 97 | * Use \b, which matches word limits at the beginning and end of the pattern, to match full words. 98 | 99 | ### Splitting and Replacing 100 | Split function from the re module works by taking any regular expression as a separator 101 | * Use capturing parentheses to split list to include the elements that is used to split the values 102 | 103 | Sub function from the re module is used for creating new strings by substituting all or part of them for a different string 104 | * It uses regular expressions for both the matching and the replacing 105 | 106 | --- 107 | 108 | ## Credit 109 | * [Coursera - Python Operating System Week 3](https://www.coursera.org/learn/python-operating-system/home/week/3) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-three/course-two-week-three.py: -------------------------------------------------------------------------------- 1 | # Regular Expressions 2 | log = "July 31 7:51:48 mycomputer bad_process[12345]: ERROR Perfroming package upgrade" 3 | index = log.index('[') 4 | 5 | # Brittle way to extracing numbers by using index function 6 | print(log[index+1:index+6]) 7 | 8 | # re module allows for search function to find regular expressions inside strings 9 | import re 10 | log = "July 31 7:51:48 mycomputer bad_process[12345]: ERROR Perfroming package upgrade" 11 | regex = r"\[(\d+)\]" 12 | result = re.search(regex, log) 13 | print(result[1]) 14 | 15 | # ========================================================================================= 16 | 17 | # Basic Regular Expressions 18 | # Simple Matching in Python 19 | # The "r" at the beginning of the pattern indicates that this is a rawstring 20 | # Always use rawstrings for regular expressions in Python 21 | result = re.search(r'aza', 'plaza') 22 | print(result) 23 | 24 | result = re.search(r'aza', 'bazaar') 25 | print(result) 26 | 27 | # None is a special value that Python uses that show that there's none actual value there 28 | result = re.search(r'aza', 'maze') 29 | print(result) 30 | 31 | 32 | # The match attribute always has a value of the actual sub string that match the search pattern 33 | # The span attribute indicates the range where the sub string can be found in the string 34 | print(re.search(r"^x", "xenon")) 35 | print(re.search(r"p.ng", "penguin")) 36 | print(re.search(r"p.ng", "sponge")) 37 | 38 | # Additional options to the search function can be added as a third parameter 39 | # The re.IGNORECASE option returns a match that is case insensitive 40 | print(re.search(r"p.ng", "Pangaea", re.IGNORECASE)) 41 | 42 | # ----------------------------------------------------------------------------------------- 43 | 44 | # Wildcards and Character Classes 45 | # Character classes are written inside square brackets 46 | # It list the characters to match inside of the brackets 47 | # A range of characters can be defined using a dash 48 | print(re.search(r"[a-z]way", "The end of the highway")) 49 | print(re.search(r"[a-z]way", "What a way to go")) 50 | print(re.search(r"cloud[a-zA-Z0-9]", "cloudy")) 51 | print(re.search(r"cloud[a-zA-Z0-9]", "cloud9")) 52 | 53 | # Use a ^, circumflex, inside the square brackets to match any characters that aren't in a group 54 | print(re.search(r"[^a-zA-Z]", "This is a sentence with spaces.")) 55 | print(re.search(r"[^a-zA-Z ]", "This is a sentence with spaces.")) 56 | 57 | # Use a |, pipe symbol to match either one expression or another 58 | # The search function returns the first matching string only when there are multiple matches 59 | print(re.search(r"cat|dog", "I like cats.")) 60 | print(re.search(r"cat|dog", "I like dogs.")) 61 | 62 | # Use the findall function provided by the re module to get all possible matches 63 | print(re.findall(r"cat|dog", "I like both cats and dogs.")) 64 | 65 | # ----------------------------------------------------------------------------------------- 66 | 67 | # Repetition Qualifiers 68 | # Repeated matches is a common expressions that include a . followed by a * 69 | # It matches any character repeated as many times as possible including zero - greedy behavior 70 | print(re.search(r"Py.*n", "Pygmalion")) 71 | print(re.search(r"Py.*n", "Python Programming")) 72 | print(re.search(r"Py[a-z]*n", "Python Programming")) 73 | print(re.search(r"Py.*n", "Pyn")) 74 | 75 | # Use a +, plus character, to match one or more occurrences of the character that comes before it 76 | print(re.search(r"o+l+", "goldfish")) 77 | print(re.search(r"o+l+", "woolly")) 78 | print(re.search(r"o+l+", "boil")) 79 | 80 | # Use a ?, question mark symbol, for either zero or one occurrence of the character before it 81 | # It is used to specified optional characters 82 | print(re.search(r"p?each", "To each their own")) 83 | print(re.search(r"p?each", "I like peaches")) 84 | 85 | # ----------------------------------------------------------------------------------------- 86 | 87 | # Escaping Characters 88 | # A pattern that includes a \ could be escaping a special regex character or a special string character 89 | # Use a \, escape character, to match one of the special characters 90 | print(re.search(r".com", "welcome")) 91 | print(re.search(r"\.com", "welcome")) 92 | print(re.search(r"\.com", "mydomain.com")) 93 | 94 | # Use \w to match any alphanumeric character including letters, numbers, and underscores 95 | # Use \d to match digits 96 | # Use \s for matching whitespace characters like space, tab or new line 97 | # Use \b for word boundaries 98 | print(re.search(r"\w*", "This is an example")) 99 | print(re.search(r"\w*", "And_this_is_another")) 100 | 101 | # ----------------------------------------------------------------------------------------- 102 | 103 | # Regular Expressions in Action 104 | # "Azerbaijan" returns "Azerbaija" because we did not specify the end 105 | print(re.search(r"A.*a", "Argentina")) 106 | print(re.search(r"A.*a", "Azerbaijan")) 107 | 108 | # "Azerbaijan" returns None 109 | print(re.search(r"^A.*a$", "Azerbaijan")) 110 | print(re.search(r"^A.*a$", "Australia")) 111 | 112 | pattern = r"^[a-zA-Z0-9_]*$" 113 | print(re.search(pattern, "this_is_a_valid_variable_name")) 114 | print(re.search(pattern, "this isn't a valid variable name")) 115 | print(re.search(pattern, "my_variable1")) 116 | print(re.search(pattern, "2my_variable1")) 117 | 118 | # ========================================================================================= 119 | 120 | # Advanced Regular Expressions 121 | # Capturing Groups 122 | # Use parentheses to capture groups which are portions of the pattern that are enclosed in 123 | # Below line defines two separate groups 124 | result = re.search(r"^(\w*), (\w*)$", "Lovelace, Ada") 125 | print(result) 126 | 127 | # The group method returns a tuple of two elements 128 | print(result.groups()) 129 | 130 | # Use indexing to access these groups 131 | # The first element contains the text matched by the entire regular expression 132 | # Each successive element contains the data that was matched by every subsequent match group 133 | print(result[0]) 134 | print(result[1]) 135 | print(result[2]) 136 | print("{} {}".format(result[2], result[1])) 137 | 138 | def rearrange_name(name): 139 | result = re.search(r"^(\w*), (\w*)$", name) 140 | if result is None: 141 | return print(name) 142 | return print("{} {}".format(result[2], result[1])) 143 | 144 | rearrange_name("Lovelace, Ada") 145 | rearrange_name("Ritchie, Dennis") 146 | rearrange_name("Hopper, Grace M.") 147 | 148 | def rearrange_name_updated(name): 149 | result = re.search(r"^([\w \.-]*), ([\w \.-]*)$", name) 150 | if result is None: 151 | return print(name) 152 | return print("{} {}".format(result[2], result[1])) 153 | 154 | rearrange_name_updated("Hopper, Grace M.") 155 | 156 | # ----------------------------------------------------------------------------------------- 157 | 158 | # More on Repetition Qualifiers 159 | # Use {}, curly brackets and one or two numbers to specify a range with numeric repetition qualifiers 160 | print(re.search(r"[a-zA-Z]{5}", "a ghost")) 161 | print(re.search(r"[a-zA-Z]{5}", "a scary ghost appeared")) 162 | print(re.findall(r"[a-zA-Z]{5}", "a scary ghost appeared")) 163 | 164 | # Use \b, which matches word limits at the beginning and end of the pattern, to match full words 165 | print(re.findall(r"\b[a-zA-Z]{5}\b", "A scary ghost appeared")) 166 | print(re.findall(r"\w{5,10}", "I really like strawberries")) 167 | print(re.search(r"s\w{,20}", "I really like strawberries")) 168 | 169 | # ----------------------------------------------------------------------------------------- 170 | 171 | # Extracting a PID Using regexes in Python 172 | log = "July 31 07:51:48 mycomputer bad_process[12345]: ERROR Performing package upgrade" 173 | regex = r"\[(\d+)]" 174 | result = re.search(regex, log) 175 | print(result[1]) 176 | 177 | result = re.search(regex, "A completely different string that also has numbers [34567]") 178 | print(result[1]) 179 | 180 | # Trying to access the index 1. Therefore returs None 181 | # result = re.search(regex, "99 elephants in a [cage]") 182 | # print(result[1]) 183 | 184 | def extract_pid(log_line): 185 | regex = r"\[(\d+)]" 186 | result = re.search(regex, log_line) 187 | if result is None: 188 | return "" 189 | return result[1] 190 | 191 | print(extract_pid(log)) 192 | print(extract_pid("99 elephants in a [cage]")) 193 | 194 | # ----------------------------------------------------------------------------------------- 195 | 196 | # Splitting and Replacing 197 | # Split function from the re module works by taking any regular expression as a separator 198 | # Use capturing parentheses to split list to include the elements that is used to split tje values 199 | print(re.split(r"[.?!]", "One sentence. Another one? And the last one!")) 200 | print(re.split(r"([.?!])", "One sentence. Another one? And the last one!")) 201 | 202 | # Sub function from the re module is used for creating new strings by substituting all or part of them for a different string 203 | # It uses regular expressions for both the matching and the replacing 204 | print(re.sub(r"[\w.%+-]+@[\w.-]+", "[REDACTED]", "Received an email for go_nuts95@my.example.com")) -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/README.md: -------------------------------------------------------------------------------- 1 | # Using Python to Interact with the Operating System - Week 2 2 | 3 | ## Reading and Writing Files 4 | ### Programming with Files 5 | File systems are usually organized in a __tree structure__ with directories and files nested under their parents. 6 | * A relative paths use only a portion of a path to show where the resource is located in relation to the current working directory. 7 | * An absolute path is a full path to the resource in the file system. 8 | 9 | ### Reading Files 10 | When processing large chunks of data, it's a good idea to read that data from files. 11 | * **readline method** reads a single line of a file and updates the current position in the file. 12 | * **read method** reads from the current position until the end of the file instead of just one line. 13 | 14 | The *open-use-close pattern* is a typical way of working with files in most programming languages. It's a good idea to close files after you've opened them for a few reasons. 15 | 1. When a file is opening a script, the file system usually lock it down and so no other programs or scripts can use it until it is closed. 16 | 2. There's a limited number of file descriptors that can be creates before file system runs out of them. 17 | 3. Leaving open files hanging around can lead to race conditions which occur when multiple processes try to modify and read from one resource at the same time and can cause unexpected behavior. 18 | 19 | In Python, **file descriptor** is stored as an __attribute__ of the files object. This is a token generated by the OS that allows programs to do more operations with the file. 20 | 21 | the "with" keyword creates a block of code with the code to be excuted inside of it. When we use a "with" block, __Python will automatically close the file__ 22 | 23 | The open-use-close approach vs. the "with" approach 24 | * Using a "with" block is a good way to open and work on a single file then have the file automatically closed at the end of the block. 25 | * Using open outside of a block means we can use a file object in other places in our code. So we're not restricted to just one single block. But when taking this approach, we need to remember to close it when we're finished. 26 | 27 | ### Iterating through Files 28 | There are 2 ways to iterate a file object: 29 | 1. Iterate the file object like list or strings 30 | 2. Iterates the file object by reading the file lines into a list 31 | * When Python reads the file line by line, the line variable will always have a new line character at the end 32 | 33 | The file has a new line character at the end of each line. 34 | * Empty lines can be avoided by using strip method. 35 | 36 | Python uses backlashes to display some of special symbols or operations: 37 | * A newline character is displayed using "\n" symbol when printing a list of strings. 38 | * 'tab' is displayed using \t. 39 | * A single or double quote can be displayed using backlashes as well. 40 | 41 | Be careful when reading the entire contents of a file into a variable of our programs: 42 | If the file is large, it can take a lot of computer's memory to hold it, which can lead to poor performance. If a file is just a few kilobytes, it's more efficient to process it line by line. 43 | 44 | ### Writing Files 45 | File objects can be opened in several different modes. A mode is similar to a file permission. It governs what can be done with the file. 46 | 47 | | Character | Meaning | 48 | |:-:|:-:| 49 | | r | open for reading (default) | 50 | | w | open for writing, truncating the file first | 51 | | x | open for exclusive creation, failing if the file already exists | 52 | | a | open for writing, appending to the end of the file if it exists | 53 | | b | binary mode | 54 | | t | text mode (default) | 55 | | + | open for updating (reading and writing) | 56 | 57 | --- 58 | 59 | ## Managing Files and Directories 60 | ### Working with Files 61 | **os module** provides a layer of abstraction between Python and the operating system which allows us to interact with the underlying system without us knowing whether we're working on a Windows, Mac, Linux, or any other operating system supported by Python. 62 | 63 | Some of the functions available from the os module are: 64 | * remove function deletes a file 65 | * Trying to remove the file that doesn't exist, the function will raise an error. 66 | * exists function checks whether a file exist 67 | * rename function changes name of a file 68 | * The first parameter to rename function is the old name of the file and the second is new name. 69 | 70 | ### More File Informations 71 | * getsize checks a file size and returns the file size in bytes. 72 | * getmitime checks when the file was last modified and returns Unix timestamp. 73 | * abspath function takes a filename and turns it into an absolute path. 74 | 75 | **datetime module** provides a function to convert the Unix timestamp. 76 | 77 | --- 78 | 79 | ## Reading and Writing CSV Files 80 | 81 | 82 | --- 83 | 84 | ## Credit 85 | * [Coursera - Python Operating System Week 2](https://www.coursera.org/learn/python-operating-system/home/week/2) 86 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/course-two-week-two.py: -------------------------------------------------------------------------------- 1 | # Reading and Writing Files 2 | # Creates a new file object and assigning it to a variable called file 3 | # Current line assumes that the spider.txt file is in the same directory 4 | file = open("using-python-to-interact-with-the-operating-system/week-two/spider.txt") 5 | 6 | # readline method reads a single line of a file 7 | print(file.readline()) 8 | # readline method reads the second line of a file - each time the readline method is called, the file object updates the current position in the file 9 | print(file.readline()) 10 | # read method, which reads from the current position until the end of the file instead of just one line 11 | print(file.read()) 12 | 13 | # Opened file needs to be closed to conform the open-close approach 14 | file.close() 15 | 16 | # "with" keyword creates a block of code with the work that needs to be done with the file inside of it 17 | # When "with" block is used, Python will automatically close the file 18 | with open("using-python-to-interact-with-the-operating-system/week-two/spider.txt") as file: 19 | print(file.readline()) 20 | 21 | # Iterates the file object like list or strings 22 | # The file has a new line character at the end of each line 23 | # When Python reads the file line by line, the line variable will always have a new line character at the end 24 | with open("using-python-to-interact-with-the-operating-system/week-two/spider.txt") as file: 25 | for line in file: 26 | print(line.upper()) 27 | 28 | # Empty lines can be avoided by using strip method 29 | with open("using-python-to-interact-with-the-operating-system/week-two/spider.txt") as file: 30 | for line in file: 31 | print(line.strip().upper()) 32 | 33 | # Iterates the file object by reading the file lines into a list 34 | file = open("using-python-to-interact-with-the-operating-system/week-two/spider.txt") 35 | lines = file.readlines() 36 | file.close 37 | lines.sort() 38 | print(lines) 39 | 40 | # "with" block pattern takes the second argument which specifies in which mode the file should be opened as 41 | with open("using-python-to-interact-with-the-operating-system/week-two/novel.txt", "w") as file: 42 | file.write("It was a dark and stormy night") 43 | 44 | # Managing Files and Directories 45 | # os module provides a layer of abstraction between Python and the operating system 46 | import os 47 | 48 | # remove function deletes a file 49 | os.remove("using-python-to-interact-with-the-operating-system/week-two/novel.txt") 50 | 51 | # rename function renames a file 52 | # The first parameter to rename function is the old name of the file and the second is new name 53 | os.rename("using-python-to-interact-with-the-operating-system/week-two/spider.txt", "using-python-to-interact-with-the-operating-system/week-two/spider_rename.txt") 54 | os.rename("using-python-to-interact-with-the-operating-system/week-two/spider_rename.txt", "using-python-to-interact-with-the-operating-system/week-two/spider.txt") 55 | 56 | # the OS path sub-module's exists function checks whether a file exist 57 | print(os.path.exists("using-python-to-interact-with-the-operating-system/week-two/spider_rename.txt")) 58 | 59 | # getsize checks a file size and returns the file size in bytes 60 | print(os.path.getsize("using-python-to-interact-with-the-operating-system/week-two/spider.txt")) 61 | 62 | # getmitime checks when the file was last modified and returns Unix timestamp 63 | print(os.path.getmtime("using-python-to-interact-with-the-operating-system/week-two/spider.txt")) 64 | 65 | # datetime module provides a function to convert the Unix timestamp 66 | import datetime 67 | timestap = os.path.getmtime("using-python-to-interact-with-the-operating-system/week-two/spider.txt") 68 | print(datetime.datetime.fromtimestamp(timestap)) 69 | 70 | # abspath function takes a filename and turns it into an absolute path 71 | print(os.path.abspath("spider.txt")) 72 | 73 | # Directories 74 | # getcwd function checks which current directory the Python program is executing 75 | print(os.getcwd()) 76 | 77 | # mkdir function creates a directory 78 | os.mkdir("using-python-to-interact-with-the-operating-system/week-two/new_dir") 79 | 80 | # chdir function changes a directory 81 | os.chdir("using-python-to-interact-with-the-operating-system/week-two/new_dir") 82 | print(os.getcwd()) 83 | os.chdir("..") 84 | print(os.getcwd()) 85 | 86 | # rmdir function removes a directory 87 | os.rmdir(os.path.abspath("new_dir")) 88 | 89 | os.chdir("..") 90 | print(os.getcwd()) 91 | # os.listdir function returns a list of all the files and sub-directories in a given directory 92 | print(os.listdir("week-two")) 93 | dir = "week-two" 94 | os.mkdir("week-two/another_new_dir") 95 | for name in os.listdir(dir): 96 | # os.path.join creates the full path and it can be used to to check if the files are directory or not 97 | fullname = os.path.join(dir, name) 98 | if os.path.isdir(fullname): 99 | print("{} is a directory".format(fullname)) 100 | else: 101 | print("{} is a file".format(fullname)) 102 | 103 | os.rmdir("week-two/another_new_dir") 104 | 105 | # Reading CSV Files 106 | import csv 107 | f = open("week-two/csv_file.txt") 108 | 109 | # parses the file using the CSV module 110 | csv_f = csv.reader(f) 111 | for row in csv_f: 112 | # unpakcs the value from the CSV file 113 | name, phone, role = row 114 | print("Name: {}, Phone: {}, Role: {}".format(name, phone, role)) 115 | f.close() 116 | 117 | # Reading and Writing CSV Files with Dictionaries 118 | # DictReader turns each row of the data in a CSV file into a dictionary 119 | with open ("week-two/software.csv") as software: 120 | reader = csv.DictReader(software) 121 | for row in reader: 122 | print("{} has {} users".format(row["name"], row["users"])) 123 | 124 | # DictWriter generates a CSV file from the contents of a list of dictionaries 125 | users = [{"name": "Sol Mansi", "username": "solm", "department": "IT infrastructure"}, 126 | {"name": "Lio Nelson", "username": "lion", "department": "User Experience Research"}, 127 | {"name": "Charlie Grey", "username": "greyc", "department": "Development" }] 128 | 129 | keys = ["name", "username", "department"] 130 | 131 | with open("week-two/by_department.csv", "w") as by_department: 132 | writer = csv.DictWriter(by_department, fieldnames=keys) 133 | writer.writeheader() 134 | writer.writerows(users) 135 | 136 | os.remove("week-two/by_department.csv") -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/csv_file.txt: -------------------------------------------------------------------------------- 1 | Sabrina Green,802-867-5309,System Admin 2 | Eli Jones,684-348-1127,IT Specialist -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/generate_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import csv 3 | def read_employees(csv_file_location): 4 | csv.register_dialect('empDialect', skipinitialspace=True, strict=True) 5 | employee_file = csv.DictReader(open(csv_file_location), dialect = 'empDialect') 6 | employee_list = [] 7 | for data in employee_file: 8 | employee_list.append(data) 9 | return employee_list 10 | 11 | employee_list = read_employees('/home/student-01-7ae3abd56cd7/data/employees.csv') 12 | 13 | def process_data(employee_list): 14 | department_list = [] 15 | for employee_data in employee_list: 16 | department_list.append(employee_data['Department']) 17 | 18 | department_data = {} 19 | for department_name in set(department_list): 20 | department_data[department_name] = department_list.count(department_name) 21 | return department_data 22 | 23 | dictionary = process_data(employee_list) 24 | 25 | def write_report(dictionary, report_file): 26 | with open(report_file, "w+") as f: 27 | for k in sorted(dictionary): 28 | f.write(str(k)+':'+str(dictionary[k])+'\n') 29 | f.close() 30 | 31 | write_report(dictionary, '/home/student-01-7ae3abd56cd7/test_report.txt') 32 | -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/software.csv: -------------------------------------------------------------------------------- 1 | name,version,status,users 2 | MailTree,5.34,production,324 3 | CalDoor,1.24,beta,22 -------------------------------------------------------------------------------- /using-python-to-interact-with-the-operating-system/week-two/spider.txt: -------------------------------------------------------------------------------- 1 | Incy Wincy spider went up the water spout 2 | Down came the rain and washed the spider out 3 | Out came the sun and dried up all the rain 4 | So, Incy Wincy spider went up the water spout again 5 | 6 | Incy Wincy spider went up the water spout 7 | Down came the rain and washed the spider out 8 | Out came the sun and dried up all the rain 9 | So, Incy Wincy spider went up the water spout again 10 | 11 | Incy Wincy spider went up the water spout 12 | Down came the rain and washed the spider out 13 | Out came the sun and dried up all the rain 14 | So, Incy Wincy spider went up the water spout again --------------------------------------------------------------------------------