├── README.md ├── example.py ├── module2 ├── README.md └── final_project_week7.py ├── module7 ├── week1 │ ├── README.md │ └── converter.py ├── week2 │ ├── README.md │ └── run.py ├── week3 │ ├── README.md │ ├── cars.py │ ├── emails.py │ └── reports.py └── week4 │ ├── changeImage.py │ ├── cron_file │ ├── emails.py │ ├── health_check.py │ ├── report_email.py │ ├── reports.py │ ├── run.py │ └── supplier_image_upload.py └── test2.py /README.md: -------------------------------------------------------------------------------- 1 | # Google-IT-automation-with-Python 2 | Google's coursera course on Python automation 3 | 4 | https://www.coursera.org/professional-certificates/google-it-automation 5 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | def git_operation(): 4 | print("I am adding example.py file to the remote repository.") 5 | 6 | git_operation() 7 | -------------------------------------------------------------------------------- /module2/README.md: -------------------------------------------------------------------------------- 1 | # Log Analysis Using Regular Expressions 2 | 3 | ## Introduction 4 | 5 | Imagine your company uses a server that runs a service called ticky, an internal ticketing system. 6 | The service logs events to syslog, both when it runs successfully and when it encounters errors. 7 | 8 | The service's developers need your help getting some information from those logs so that they can 9 | better understand how their software is used and how to improve it. So, for this lab, you'll write 10 | some automation scripts that will process the system log and generate reports based on the information 11 | extracted from the log files. 12 | 13 | ## What you'll do 14 | 15 | Use regex to parse a log file 16 | Append and modify values in a dictionary 17 | Write to a file in CSV format 18 | Move files to the appropriate directory for use with the CSV->HTML converter 19 | You'll have 90 minutes to complete this lab. 20 | -------------------------------------------------------------------------------- /module2/final_project_week7.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | import csv 5 | import operator 6 | 7 | error_msg = {} 8 | user_count = {} 9 | 10 | pattern = r"ticky: ([\w]{4,5}) ([\w\s]+)[\[\]#\d\s]*\((\w*)" 11 | 12 | logfile = "syslog.log" 13 | 14 | err_file = "error_message.csv" 15 | usr_stat = "user_statistics.csv" 16 | 17 | 18 | with open(logfile) as f: 19 | for log in f: 20 | if re.search(pattern, log): 21 | result = re.search(pattern, log) 22 | type = result.group(1) #log type 23 | emsg = result.group(2).strip() #error message 24 | user = result.group(3) #user name 25 | 26 | if type == "ERROR": 27 | if emsg not in error_msg.keys(): 28 | error_msg[emsg] = 0 29 | error_msg[emsg] += 1 30 | 31 | if user not in user_count.keys(): 32 | user_count[user] = {} 33 | user_count[user]["INFO"] = 0 34 | user_count[user]["ERROR"] = 0 35 | 36 | if type == "ERROR": 37 | user_count[user]["ERROR"] += 1 38 | elif type == "INFO": 39 | user_count[user]["INFO"] += 1 40 | 41 | error_msg = sorted(error_msg.items(), key = operator.itemgetter(1), reverse=True) 42 | user_count = sorted(user_count.items()) 43 | 44 | f.close() 45 | 46 | with open(err_file, 'w') as f: 47 | writer = csv.writer(f) 48 | writer.writerow(["ERROR", "COUNT"]) 49 | writer.writerows(error_msg) 50 | f.close() 51 | 52 | with open(usr_stat, 'w') as f: 53 | writer = csv.writer(f) 54 | writer.writerow(["USERNAME", "INFO", "ERROR"]) 55 | for line in user_count: 56 | writer.writerow([line[0], line[1]["INFO"], line[1]["ERROR"]]) 57 | f.close() 58 | 59 | 60 | -------------------------------------------------------------------------------- /module7/week1/README.md: -------------------------------------------------------------------------------- 1 | # Qwiklabs Assessment: Scale and convert images using PIL 2 | ## Introduction 3 | Your company is in the process of updating its website, and they’ve hired a design contractor to create some new icon graphics for the site. But the contractor has delivered the final designs in the wrong format -- rotated 90° and too large. Oof! You’re not able to get in contact with the designers and your own deadline is approaching fast. You’ll need to use Python to get these images ready for launch. 4 | 5 | ## What you'll do 6 | Use the Python Imaging Library to do the following to a batch of images: 7 | * Open an image 8 | * Rotate an image 9 | * Resize an image 10 | * Save an image in a specific format in a separate directory 11 | 12 | -------------------------------------------------------------------------------- /module7/week1/converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os, sys 4 | from PIL import Image 5 | 6 | for root, dirs, files in os.walk("."): 7 | for file in files: 8 | f, e = os.path.splitext(file) 9 | outfile = "/opt/icons/" + f 10 | try: 11 | #print(outfile) 12 | Image.open(file).rotate(270).resize((128,128)).convert("RGB").save(outfile, "JPEG") 13 | 14 | except IOError: 15 | print("cannot convert", file) 16 | 17 | 18 | -------------------------------------------------------------------------------- /module7/week2/README.md: -------------------------------------------------------------------------------- 1 | # Qwiklabs Assessment: Process Text Files with Python Dictionaries and Upload to Running Web Service 2 | ## Introduction 3 | You’re working at a company that sells second-hand cars. Your company constantly collects feedback in the form of customer reviews. Your manager asks you to take those reviews (saved as .txt files) and display them on your company’s website. To do this, you’ll need to write a script to convert those .txt files and process them into Python dictionaries, then upload the data onto your company’s website (currently using Django). 4 | 5 | ## What you’ll do 6 | * Use the Python OS module to process a directory of text files 7 | * Manage information stored in Python dictionaries 8 | * Use the Python requests module to upload content to a running Web service 9 | * Understand basic operations for Python requests like GET and POST methods 10 | -------------------------------------------------------------------------------- /module7/week2/run.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import os 4 | import requests 5 | 6 | path = "/data/feedback/" 7 | reviews = os.listdir(path) 8 | 9 | dict = {} 10 | 11 | for review in reviews: 12 | with open(path + review) as f: 13 | dict["title"] = f.readline().strip() 14 | dict["name"] = f.readline().strip() 15 | dict["date"] = f.readline().strip() 16 | dict["feedback"] = f.readline().strip() 17 | 18 | response = requests.post("http://34.123.16.109/feedback/", json=dict) 19 | response.raise_for_status() 20 | print(response.status_code) 21 | 22 | 23 | -------------------------------------------------------------------------------- /module7/week3/README.md: -------------------------------------------------------------------------------- 1 | # Qwiklabs Assessment: Automatically Generate a PDF and send it by Email 2 | ## Introduction 3 | You work for a company that sells second hand cars. Management wants to get a summary of the amounts of vehicles that have been sold at the end of every month. The company already has a web service which serves sales data at the end of every month but management wants an email to be sent out with an attached PDF so that data is more easily readable. 4 | ## What you'll do 5 | * Write a script that summarizes and processes sales data into difference categories 6 | * Generate a PDF using Python 7 | * Automatically send a PDF by email 8 | -------------------------------------------------------------------------------- /module7/week3/cars.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import locale 5 | import sys 6 | import emails 7 | import os 8 | import reports 9 | from reportlab.graphics.shapes import Drawing 10 | from reportlab.graphics.charts.piecharts import Pie 11 | 12 | 13 | 14 | def load_data(filename): 15 | """Loads the contents of filename as a JSON file.""" 16 | with open(filename) as json_file: 17 | data = json.load(json_file) 18 | return data 19 | 20 | 21 | def format_car(car): 22 | """Given a car dictionary, returns a nicely formatted name.""" 23 | return "{} {} ({})".format( 24 | car["car_make"], car["car_model"], car["car_year"]) 25 | 26 | 27 | def process_data(data): 28 | """Analyzes the data, looking for maximum 29 | Returns a list of lines that summarize the information. 30 | """ 31 | locale.setlocale(locale.LC_ALL, 'en_US.UTF8') 32 | max_revenue = {"revenue": 0} 33 | max_sale = {"total_sales": 0} 34 | car_year_count = {} 35 | for item in data: 36 | # Calculate the revenue generated by this model (price * total_sales) 37 | # We need to convert the price from "$1234.56" to 1234.56 38 | item_price = locale.atof(item["price"].strip("$")) 39 | item_revenue = item["total_sales"] * item_price 40 | if item_revenue > max_revenue["revenue"]: 41 | item["revenue"] = item_revenue 42 | max_revenue = item 43 | # TODO: also handle max sales 44 | if item["total_sales"] > max_sale["total_sales"]: 45 | max_sale = item 46 | # TODO: also handle most popular car_year 47 | if item["car"]["car_year"] not in car_year_count: 48 | car_year_count[item["car"]["car_year"]] = 0 49 | car_year_count[item["car"]["car_year"]] += item["total_sales"] 50 | else: 51 | car_year_count[item["car"]["car_year"]] += item["total_sales"] 52 | 53 | best_year, best_year_sales = max(car_year_count.items(), key=lambda x:x[1]) 54 | 55 | summary = [ 56 | "The {} generated the most revenue: ${}".format( 57 | format_car(max_revenue["car"]), max_revenue["revenue"]) 58 | #"The {} generated the most sales: {}".format( 59 | #format_car(max_sale["car"]), max_sale["total_sales"]), 60 | #"The most popular year was {} with {} sales.".format( 61 | #best_year, best_year_sales) 62 | ] 63 | max_sale_format = "The {} generated the most sales: {}".format(format_car(max_sale["car"]), max_sale["total_sales"]) 64 | best_year_format = "The most popular year was {} with {} sales.".format(best_year, best_year_sales) 65 | 66 | summary.append(max_sale_format) 67 | summary.append(best_year_format) 68 | return summary 69 | 70 | 71 | def cars_dict_to_table(car_data): 72 | """Turns the data in car_data into a list of lists.""" 73 | table_data = [["ID", "Car", "Price", "Total Sales"]] 74 | for item in car_data: 75 | table_data.append([item["id"], format_car(item["car"]), item["price"], item["total_sales"]]) 76 | return table_data 77 | 78 | def pie_chart(car_data): 79 | report_pie = Pie() 80 | report_pie.width = 380 81 | report_pie.height = 380 82 | report_pie.x = 42 83 | report_pie.y = -230 84 | report_pie.data = [] 85 | report_pie.labels = [] 86 | car_make = {} #storing different car_make as key and their sum of sales as value 87 | for item in car_data: 88 | car_info = format_car(item["car"]) 89 | report_pie.data.append(item["total_sales"]) 90 | report_pie.labels.append(car_info) 91 | report_chart = Drawing() 92 | report_chart.add(report_pie) 93 | return(report_chart) 94 | 95 | 96 | def main(argv): 97 | """Process the JSON data and generate a full report out of it.""" 98 | data = load_data("car_sales.json") 99 | summary = process_data(data) 100 | table = cars_dict_to_table(data) 101 | #remove the first line in the table so I can sort the rest. 102 | table_raw = table[1:] 103 | table_sorted = sorted(table_raw, key=lambda x:x[3], reverse=True) 104 | #adding the header back to the table 105 | table_sorted.insert(0, ["ID", "Car", "Price", "Total Sales"]) 106 | #calling pie chart 107 | pie = pie_chart(data) 108 | 109 | print(summary) 110 | report_summary = summary[0] + "
" + summary[1] + "
" + summary[2] + "
" 111 | # TODO: turn this into a PDF report 112 | reports.generate("/tmp/cars.pdf", "Title", report_summary , table_sorted, pie) #Sorting the table by Sales 113 | # TODO: send the PDF report as an email attachment 114 | sender = "automation@example.com" 115 | receiver = "{}@example.com".format(os.environ.get('USER')) 116 | subject = "Sales summary for last month" 117 | body = summary[0] + "\n" + summary[1] + "\n" + summary[2] + "\n" 118 | package = "/tmp/cars.pdf" 119 | 120 | message = emails.generate(sender, receiver, subject, body, package) 121 | emails.send(message) 122 | 123 | if __name__ == "__main__": 124 | main(sys.argv) 125 | -------------------------------------------------------------------------------- /module7/week3/emails.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import email.message 4 | import mimetypes 5 | import os.path 6 | import smtplib 7 | 8 | def generate(sender, recipient, subject, body, attachment_path): 9 | """Creates an email with an attachement.""" 10 | # Basic Email formatting 11 | message = email.message.EmailMessage() 12 | message["From"] = sender 13 | message["To"] = recipient 14 | message["Subject"] = subject 15 | message.set_content(body) 16 | 17 | # Process the attachment and add it to the email 18 | attachment_filename = os.path.basename(attachment_path) 19 | mime_type, _ = mimetypes.guess_type(attachment_path) 20 | mime_type, mime_subtype = mime_type.split('/', 1) 21 | 22 | with open(attachment_path, 'rb') as ap: 23 | message.add_attachment(ap.read(), 24 | maintype=mime_type, 25 | subtype=mime_subtype, 26 | filename=attachment_filename) 27 | 28 | return message 29 | 30 | def send(message): 31 | """Sends the message to the configured SMTP server.""" 32 | mail_server = smtplib.SMTP('localhost') 33 | mail_server.send_message(message) 34 | mail_server.quit() 35 | -------------------------------------------------------------------------------- /module7/week3/reports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from reportlab.platypus import SimpleDocTemplate 4 | from reportlab.platypus import Paragraph, Spacer, Table, Image 5 | from reportlab.lib.styles import getSampleStyleSheet 6 | from reportlab.lib import colors 7 | 8 | def generate(filename, title, additional_info, table_data, pie_data): 9 | #added pie_data parameter for the optional challenge 10 | styles = getSampleStyleSheet() 11 | report = SimpleDocTemplate(filename) 12 | report_title = Paragraph(title, styles["h1"]) 13 | report_info = Paragraph(additional_info, styles["BodyText"]) 14 | table_style = [('GRID', (0,0), (-1,-1), 1, colors.black), 15 | ('FONTNAME', (0,0), (-1,0), 'Helvetica-Bold'), 16 | ('ALIGN', (0,0), (-1,-1), 'CENTER')] 17 | report_table = Table(data=table_data, style=table_style, hAlign="LEFT") 18 | empty_line = Spacer(1,20) 19 | #added pie_data to build 20 | report.build([report_title, empty_line, report_info, empty_line, report_table, pie_data]) 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /module7/week4/changeImage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from PIL import Image 3 | import os, sys 4 | 5 | path = "supplier-data/images/" 6 | pictures = os.listdir(path) 7 | 8 | for pic in pictures: 9 | if 'tiff' in pic: 10 | #grab the picture name without the .tiff extension 11 | file_name = os.path.splitext(pic)[0] 12 | outfile = "supplier-data/images/" + file_name + ".jpeg" 13 | try: 14 | Image.open(path + pic).convert("RGB").resize((600,400)).save(outfile,"JPEG") 15 | except IOError: 16 | print("cannot convert", pic) 17 | 18 | -------------------------------------------------------------------------------- /module7/week4/cron_file: -------------------------------------------------------------------------------- 1 | In order for cron_file to work, you need to modifiy the health_check.py. 2 | line 34 3 | receiver = "{}@example.com".format(os.environ["USER"]) 4 | to 5 | receiver = "{user-name}@example.com" 6 | 7 | 1 * * * * . $HOME/.profile; python3 /home/{user-name}/health_check.py 8 | 9 | 10 | 11 | {user-name} is the username provided by your qwiklabs session 12 | -------------------------------------------------------------------------------- /module7/week4/emails.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import email.message 4 | import mimetypes 5 | import os.path 6 | import smtplib 7 | 8 | def generate_email(sender, recipient, subject, body, attachment_path = None): 9 | """Generate email, default is with no attachment""" 10 | # Basic Email formatting 11 | message = email.message.EmailMessage() 12 | message['Subject'] = subject 13 | message['From'] = sender 14 | message['To'] = recipient 15 | message.set_content(body) 16 | 17 | if attachment_path != None: 18 | attachment_name = os.path.basename(attachment_path) 19 | mime_type, _ = mimetypes.guess_type(attachment_path) 20 | mime_type, mime_subtype = mime_type.split("/", 1) 21 | with open(attachment_path, 'rb') as fp: 22 | message.add_attachment(fp.read(), 23 | maintype=mime_type, 24 | subtype=mime_subtype, 25 | filename=attachment_name) 26 | return message 27 | 28 | def send_email(package): 29 | """Sends the email package to the configured SMTP server.""" 30 | mail_server = smtplib.SMTP('localhost') 31 | mail_server.send_message(package) 32 | mail_server.quit() 33 | 34 | 35 | -------------------------------------------------------------------------------- /module7/week4/health_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import psutil, shutil 4 | import socket 5 | import emails 6 | import os, sys 7 | 8 | #Warning if CPU usage is over 80% 9 | def cpu_check(): 10 | cpu_usage = psutil.cpu_percent(1) 11 | return not cpu_usage > 80 12 | 13 | #Report an error if available disk space is lower than 20% 14 | def disc_space_check(): 15 | disk_usage = shutil.disk_usage("/") 16 | disk_total = disk_usage.total 17 | disk_free = disk_usage.used 18 | threshold = disk_free / disk_total * 100 19 | return threshold > 20 20 | 21 | #Report an error if available memory is less than 500MB 22 | def available_memory_check(): 23 | available = psutil.virtual_memory().available 24 | available_in_MB = available / 1024 ** 2 #convert to MB 25 | return available_in_MB > 500 26 | 27 | #Report an error if the hostname "localhost" cannot be resolved to "127.0.0.1" 28 | def hostname_check(): 29 | local_host_ip = socket.gethostbyname('localhost') 30 | return local_host_ip == "127.0.0.1" 31 | 32 | def email_warning(error): 33 | sender = "automation@example.com" 34 | receiver = "{}@example.com".format(os.environ["USER"]) 35 | subject = error 36 | body = "Please check your system and resolve the issue as soon as possible." 37 | message = emails.generate_email(sender, receiver, subject, body) 38 | emails.send_email(message) 39 | 40 | if not cpu_check(): 41 | subject = "Error - CPU usage is over 80%" 42 | email_warning(subject) 43 | 44 | if not disc_space_check(): 45 | subject = "Error - Available disk space is less than 20%" 46 | email_warning(subject) 47 | 48 | if not available_memory_check(): 49 | subject = "Error - Available memory is less than 500MB" 50 | email_warning(subject) 51 | 52 | if not hostname_check(): 53 | subject = "Error - localhost cannot be resolved to 127.0.0.1" 54 | email_warning(subject) 55 | -------------------------------------------------------------------------------- /module7/week4/report_email.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os, datetime 4 | import reports 5 | import emails 6 | 7 | #get the current time in GMT 8 | current_date = datetime.datetime.now().strftime('%Y-%m-%d') 9 | 10 | def generate_pdf(path): 11 | pdf = "" 12 | files = os.listdir(path) 13 | for file in files: 14 | if file.endswith(".txt"): 15 | with open(path + file, 'r') as f: 16 | inline = f.readlines() 17 | name = inline[0].strip() 18 | weight = inline[1].strip() 19 | pdf += "name: " + name + "
" + "weight: " + weight + "

" 20 | return pdf 21 | 22 | if __name__ == "__main__": 23 | path = "supplier-data/descriptions/" 24 | title = "Process Updated on " + current_date 25 | #generate the package for pdf body 26 | package = generate_pdf(path) 27 | reports.generate_report("/tmp/processed.pdf", title, package) 28 | 29 | #generate email information 30 | sender = "automation@example.com" 31 | receiver = "{}@example.com".format(os.environ["USER"]) 32 | subject = "Upload Completed - Online Fruit Store" 33 | body = "All fruits are uploaded to our website successfully. A detailed list is attached to this email." 34 | attachment = "/tmp/processed.pdf" 35 | 36 | #generate email for the online fruit store report and pdf attachment 37 | message = emails.generate_email(sender, receiver, subject, body, attachment) 38 | emails.send_email(message) 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /module7/week4/reports.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from reportlab.platypus import SimpleDocTemplate 3 | from reportlab.platypus import Paragraph, Spacer 4 | from reportlab.lib.styles import getSampleStyleSheet 5 | 6 | def generate_report(filename, title, info): 7 | styles = getSampleStyleSheet() 8 | report = SimpleDocTemplate(filename) 9 | report_title = Paragraph(title, styles["h1"]) 10 | report_info = Paragraph(info, styles["BodyText"]) 11 | empty_line = Spacer(1,20) 12 | 13 | report.build([report_title, empty_line, report_info, empty_line]) -------------------------------------------------------------------------------- /module7/week4/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os, sys 3 | import json 4 | import requests 5 | 6 | path = "supplier-data/descriptions/" 7 | url = "http://localhost/fruits/" 8 | 9 | files = os.listdir(path) 10 | 11 | for file in files: 12 | if file.endswith("txt"): 13 | with open(path + file, 'r') as f: 14 | #grab the file name, ex. 001, 002 to use for image file 15 | fruit_name = os.path.splitext(file)[0] 16 | data = f.read() 17 | data = data.split("\n") 18 | fruit_dic = {"name": data[0], "weight": int(data[1].strip(" lbs")), "description": data[2], "image_name": fruit_name + ".jpeg"} 19 | response = requests.post(url, json=fruit_dic) 20 | response.raise_for_status() 21 | print(response.request.url) 22 | print(response.status_code) -------------------------------------------------------------------------------- /module7/week4/supplier_image_upload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os, sys 3 | import requests 4 | 5 | url = "http://localhost/upload/" 6 | path = "supplier-data/images/" 7 | 8 | images = os.listdir(path) 9 | 10 | for image in images: 11 | if image.endswith(".jpeg"): 12 | with open(path + image, 'rb') as opened: 13 | r = requests.post(url, files={'file': opened}) 14 | -------------------------------------------------------------------------------- /test2.py: -------------------------------------------------------------------------------- 1 | 2 | --------------------------------------------------------------------------------