├── Web-Scraping
    ├── README.md
    ├── get_all_links.py
    ├── simple_request.py
    ├── headless_selenium.py
    ├── simple_selenium.py
    ├── get_headings.py
    ├── get_thumbnail.py
    ├── data.json
    ├── rss_reader.py
    ├── robots_parser.py
    └── Web Scraping With Python and Requests-HTML.ipynb
├── Sitemaps
    └── Sitemap From List of URLs
    │   ├── .gitignore
    │   ├── sitemap-0.xml.gz
    │   ├── sitemap-1.xml.gz
    │   ├── sitemap_2.xml.gz
    │   ├── sitemap_3.xml.gz
    │   ├── sitemap_4.xml.gz
    │   ├── sitemap_5.xml.gz
    │   ├── sitemap_6.xml.gz
    │   ├── sitemap_7.xml.gz
    │   ├── sitemap_8.xml.gz
    │   ├── list-of-urls.csv
    │   ├── list_of_urls.txt
    │   ├── create-simple-sitemaps.py
    │   └── sitemap_0.xml
├── Reddit
    ├── .gitignore
    ├── simple_reddit_bot.py
    ├── simple_request.py
    ├── README.md
    ├── textsummarizer.py
    ├── refreshtoken.py
    ├── prawbot.py
    └── Use Reddit API With Python.ipynb
├── .vscode
    └── settings.json
├── robots_txt
    ├── output
    │   └── www_jcchouinard_com
    │   │   ├── 2020-11-20:10:0657-robots.txt
    │   │   └── 2020-11-20:10:0713-robots.txt
    └── robotstxt_smoketext.py
├── Google-Search-Console
    ├── requirements.txt
    ├── createMysqlDb.py
    ├── README.md
    ├── Connect search console using OAuth keys.ipynb
    ├── gscbackups.py
    ├── Simple Google Search Console API Call With Filters.ipynb
    ├── GSC_to_CSV_All_keywords.py
    └── GSC_to_CSV_by_month.py
├── Causal-impact
    ├── README.md
    ├── run.py
    ├── simple_extract.py
    └── causal.py
├── GSC_Tutorial
    ├── branded-non-branded.py
    ├── gsc_by_url.py
    ├── gsc_with_filters.py
    ├── oauth.py
    ├── file_manip.py
    ├── date_manip.py
    ├── gsc_to_csv_by_month.py
    └── run.ipynb
├── Logs Analysis
    └── plotly-log.py
├── GoogleMyBusiness
    └── gmb_cat_finder_@zchtodd.py
├── README.md
├── robotstxt_to_df.py
├── knowledge graph api.py
├── .gitignore
├── .gsource
├── Pagespeed
    └── pagespeed-api-by-daniel.py
├── Google trends
    └── gsc_to_trends.py
└── Rainbow_lorikeet.jpg


/Web-Scraping/README.md:
--------------------------------------------------------------------------------
1 | # Web-Scraping


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/.gitignore:
--------------------------------------------------------------------------------
1 | .csv
2 | .txt


--------------------------------------------------------------------------------
/Reddit/.gitignore:
--------------------------------------------------------------------------------
1 | *.json*
2 | *reddit_api_json_documentation*


--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 |     "python.pythonPath": "/Users/jean.christophe/opt/anaconda3/bin/python"
3 | }


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap-0.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap-0.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap-1.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap-1.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_2.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_2.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_3.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_3.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_4.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_4.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_5.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_5.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_6.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_6.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_7.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_7.xml.gz


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_8.xml.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jcchouinard/SEO-Projects/HEAD/Sitemaps/Sitemap From List of URLs/sitemap_8.xml.gz


--------------------------------------------------------------------------------
/Web-Scraping/get_all_links.py:
--------------------------------------------------------------------------------
 1 | from lxml import html
 2 | import requests
 3 | 
 4 | url = 'https://example.com'
 5 | 
 6 | page = requests.get(url)
 7 | webpage = html.fromstring(page.content)
 8 | 
 9 | links = webpage.xpath('//a/@href')
10 | links


--------------------------------------------------------------------------------
/robots_txt/output/www_jcchouinard_com/2020-11-20:10:0657-robots.txt:
--------------------------------------------------------------------------------
 1 | User-agent: *
 2 | Sitemap: https://www.jcchouinard.com/sitemap_index.xml
 3 | 
 4 | # Admin
 5 | Disallow: /wp-*
 6 | Allow: /wp-admin/admin-ajax.php
 7 | 
 8 | # JS/CSS
 9 | Allow: *.js
10 | Allow: *.css
11 | 
12 | # Images
13 | Allow: *.gif
14 | Allow: *.jpeg
15 | Allow: *.jpg
16 | 
17 | #feed
18 | Disallow: */rss*
19 | Disallow: */feed*


--------------------------------------------------------------------------------
/robots_txt/output/www_jcchouinard_com/2020-11-20:10:0713-robots.txt:
--------------------------------------------------------------------------------
 1 | User-agent: *
 2 | Sitemap: https://www.jcchouinard.com/sitemap_index.xml
 3 | 
 4 | # Admin
 5 | Disallow: /wp-*
 6 | Allow: /wp-admin/admin-ajax.php
 7 | 
 8 | # JS/CSS
 9 | Allow: *.js
10 | Allow: *.css
11 | 
12 | # Images
13 | Allow: *.gif
14 | Allow: *.jpeg
15 | Allow: *.jpg
16 | Allow: *.png
17 | 
18 | #feed
19 | Disallow: */rss*
20 | Disallow: */feed*


--------------------------------------------------------------------------------
/Google-Search-Console/requirements.txt:
--------------------------------------------------------------------------------
 1 | certifi==2020.4.5.1
 2 | google-api-python-client==1.8.3
 3 | google-auth==1.15.0
 4 | google-auth-httplib2==0.0.3
 5 | google-auth-oauthlib==0.4.1
 6 | httplib2==0.18.1
 7 | idna==2.6
 8 | oauth2client==4.1.2
 9 | oauthlib==3.1.0
10 | pyasn1==0.4.2
11 | pyasn1-modules==0.2.1
12 | requests==2.23.0
13 | requests-oauthlib==0.8.0
14 | rsa==3.4.2
15 | six==1.11.0
16 | uritemplate==3.0.0
17 | urllib3==1.25.9
18 | 


--------------------------------------------------------------------------------
/Causal-impact/README.md:
--------------------------------------------------------------------------------
 1 | # Run pyCausalImpact Report with GSC
 2 | 
 3 | Author, Jean-Christophe Chouinard
 4 | Role: Senior SEO Specialist and DataScience Enthusiast
 5 | 
 6 | - Twitter: [@ChouinardJC](https://twitter.com/ChouinardJC)
 7 | - [Linkedin](https://www.linkedin.com/in/jeanchristophechouinard)
 8 | - Blog: [jcchouinard.com](https://www.jcchouinard.com)
 9 | - Based on this blog post: [SEO Experiment with pyCausalImpact and GSC](https://www.jcchouinard.com/pycausalimpact/)
10 | 
11 | 1. Open terminal to install the library: `$ pip install searchconsole`
12 | 2. [Download GSC credentials](https://www.jcchouinard.com/google-api/)
13 | 3. Run `run.py`


--------------------------------------------------------------------------------
/Web-Scraping/simple_request.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | '''
 3 | Simple HTTP Get Request
 4 | @author:    Jean-Christophe Chouinard. 
 5 | @role:      Sr. SEO Specialist at SEEK.com.au
 6 | @website:   jcchouinard.com
 7 | @LinkedIn:  linkedin.com/in/jeanchristophechouinard/ 
 8 | @Twitter:   twitter.com/@ChouinardJC
 9 | $ pip install requests
10 | '''
11 | import requests
12 | 
13 | url = 'https://www.jcchouinard.com/python-for-seo/' 
14 | 
15 | headers = {
16 |     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
17 | }
18 | 
19 | try:
20 |     r = requests.get(url, headers=headers)
21 | except requests.exceptions.RequestException as e: 
22 |     raise SystemExit(e)
23 | 
24 | print(r.text)
25 | print(r.url)
26 | print(r.status_code)


--------------------------------------------------------------------------------
/Web-Scraping/headless_selenium.py:
--------------------------------------------------------------------------------
 1 | 
 2 | #!/usr/bin/env python
 3 | '''
 4 | Run Selenium in headless Chrome
 5 | @author:    Jean-Christophe Chouinard. 
 6 | @role:      Sr. SEO Specialist at SEEK.com.au
 7 | @website:   jcchouinard.com
 8 | @LinkedIn:  linkedin.com/in/jeanchristophechouinard/ 
 9 | @Twitter:   twitter.com/@ChouinardJC
10 | '''
11 | import time
12 | 
13 | from selenium import webdriver
14 | from selenium.webdriver.chrome.options import Options
15 | 
16 | url = 'https://www.jcchouinard.com/python-for-seo/' 
17 | 
18 | options = Options()
19 | options.headless = True
20 | # options.add_argument("--window-size=1920,1200")
21 | driver = webdriver.Chrome(options=options,executable_path='/usr/local/bin/chromedriver')
22 | 
23 | driver.get(url)
24 | time.sleep(3)
25 | t = driver.title
26 | print(f'Title: {t}')
27 | 
28 | driver.quit()


--------------------------------------------------------------------------------
/Web-Scraping/simple_selenium.py:
--------------------------------------------------------------------------------
 1 | 
 2 | #!/usr/bin/env python
 3 | '''
 4 | Simple Selenium browser automation.
 5 | @author:    Jean-Christophe Chouinard. 
 6 | @role:      Sr. SEO Specialist at SEEK.com.au
 7 | @website:   jcchouinard.com
 8 | @LinkedIn:  linkedin.com/in/jeanchristophechouinard/ 
 9 | @Twitter:   twitter.com/@ChouinardJC
10 | What it does:
11 | Opens Chrome
12 | Visit URL
13 | Print the HTML of the page
14 | Close browser.
15 | '''
16 | import time
17 | 
18 | from selenium import webdriver
19 | 
20 | url = 'https://www.jcchouinard.com/learn-selenium-python-seo-automation/' # Define page to run 
21 | driver = webdriver.Chrome('/usr/local/bin/chromedriver') # Open Chrome
22 | driver.get(url)             # Visit URL
23 | 
24 | print(driver.page_source)   # Print HTML
25 | time.sleep(3)               # Wait 3 Seconds
26 | driver.quit()               # Close browser


--------------------------------------------------------------------------------
/Web-Scraping/get_headings.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | '''
 3 | Print headings in a page
 4 | @author:    Jean-Christophe Chouinard. 
 5 | @role:      Sr. SEO Specialist at SEEK.com.au
 6 | @website:   jcchouinard.com
 7 | @LinkedIn:  linkedin.com/in/jeanchristophechouinard/ 
 8 | @Twitter:   twitter.com/@ChouinardJC
 9 | $ pip install requests
10 | $ pip install bs4
11 | '''
12 | from bs4 import BeautifulSoup
13 | import requests
14 | 
15 | url = 'https://www.jcchouinard.com/python-for-seo/' 
16 | 
17 | headers = {
18 |     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
19 | }
20 | 
21 | try:
22 |     r = requests.get(url, headers=headers)
23 | except requests.exceptions.RequestException as e: 
24 |     raise SystemExit(e)
25 | 
26 | page = BeautifulSoup(r.text, "lxml")
27 | for h in page.find_all(['h1','h2','h3']):
28 |     print(f'{h.name}: {h.text.strip()}')
29 | 
30 | 
31 | 


--------------------------------------------------------------------------------
/Reddit/simple_reddit_bot.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import praw
 3 | import requests
 4 | 
 5 | subr = 'pythonsandlot'
 6 | credentials = 'client_secrets.json'
 7 | 
 8 | with open(credentials) as f:
 9 |     creds = json.load(f)
10 | 
11 | reddit = praw.Reddit(client_id=creds['client_id'],
12 |                      client_secret=creds['client_secret'],
13 |                      user_agent=creds['user_agent'],
14 |                      redirect_uri=creds['redirect_uri'],
15 |                      refresh_token=creds['refresh_token'])
16 | 
17 | print(reddit.user.me())
18 | 
19 | subreddit = reddit.subreddit(subr)
20 | 
21 | title = 'Just Made My first Post on Reddit Using Python.'
22 | selftext = '''
23 | I am learning how to use the Reddit API with Python using the PRAW wrapper.
24 | By following the tutorial on https://www.jcchouinard.com/post-on-reddit-with-python/
25 | This post was uploaded from my Python Script
26 | '''
27 | 
28 | subreddit.submit(title,selftext=selftext)
29 | 
30 | 


--------------------------------------------------------------------------------
/Web-Scraping/get_thumbnail.py:
--------------------------------------------------------------------------------
 1 | from PIL import Image
 2 | os.getcwd()
 3 | def thumbnail(file, thumbnail, width, height):
 4 |     path = os.getcwd()
 5 |     thumb = width, height
 6 |     img = Image.open(str(path)+ '/' + str(file))
 7 |     width, height = img.size
 8 | 
 9 |     if width > height:
10 |         delta = width - height
11 |         left = int(delta / 2)
12 |         upper = 0
13 |         right = height + left
14 |         lower = height
15 |     else:
16 |         delta = height - width
17 |         left = 0
18 |         upper = int(delta / 2)
19 |         right = width
20 |         lower = width + upper
21 | 
22 |     img = img.crop((left, upper, right, lower))
23 |     img.thumbnail(thumb, Image.ANTIALIAS)
24 |     img.save(thumbnail)
25 |     img.close()
26 | 
27 | thumbnail('Rainbow_lorikeet.jpg', 'lorikeet_thumb.jpg', 150, 150)
28 | 
29 | os.listdir()
30 | 
31 | Image.open(r'/Users/jean.christophe/Documents/Github/SEO-Projects/Web-Scraping/Rainbow_lorikeet.jpg')


--------------------------------------------------------------------------------
/GSC_Tutorial/branded-non-branded.py:
--------------------------------------------------------------------------------
 1 | import matplotlib.pyplot as plt
 2 | import pandas as pd
 3 | 
 4 | import file_manip as fm
 5 | 
 6 | site = 'https://www.jcchouinard.com'
 7 | filename = 'gsc_data.csv'
 8 | 
 9 | df = fm.return_df(site,filename)
10 | r = r'.*chouinard.*'
11 | df['query_type'] = ''
12 | df['query_type'][df['query'].str.contains(r,regex=True)] = 'Brand'
13 | df['query_type'][~df['query'].str.contains(r,regex=True)] = 'Non-Brand'
14 | 
15 | df = fm.return_df(v['url'],filename)
16 | df['query_type'] = ''
17 | df['query_type'][~df['query'].str.contains(v['brand'],regex=True)] = 'Non-Brand'
18 | df['query_type'][df['query'].str.contains(v['brand'],regex=True)] = 'Brand'
19 | t = df.groupby(['date','query_type'])['clicks'].sum().reset_index()
20 | t = t.set_index(['date','query_type'])['clicks'].unstack()
21 | t = t.reset_index().rename_axis(None, axis=1)
22 | t = fm.date_to_index(t,'date')
23 | t.plot(subplots=True,
24 |         sharex=True,
25 |         figsize=(6,6),
26 |         sharey=False)
27 | plt.show()


--------------------------------------------------------------------------------
/Logs Analysis/plotly-log.py:
--------------------------------------------------------------------------------
 1 | # Read Log File in a Plotly Graph
 2 | 
 3 | import pandas as pd
 4 | import re
 5 | import requests
 6 | import plotly.express as px
 7 | import datetime
 8 | 
 9 | #Get Log File Template from here 'https://github.com/jcchouinard/SEO-Projects/blob/master/access_log_20200602-101559.log'
10 | 
11 | logs = 'access_log_20200602-101559.log'
12 | log_data = []
13 | columns = ['ip','date','http_request','status_code','count','request_url','user_agent']
14 | regex = '([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)"'
15 | 
16 |         
17 | with open(logs, 'r') as f:
18 | # Get components from each line of the log file into a structured dict
19 |     for line in f:
20 |         line = re.match(regex, line).groups()
21 |         log_data.append(line)
22 |         
23 | log_data = pd.DataFrame(log_data, columns=columns)
24 | log_data['date']= pd.to_datetime(log_data['date'], format='%d/%b/%Y:%H:%M:%S %z')
25 | log_data['date'] =  pd.to_datetime(log_data['date'].dt.strftime('%Y-%m-%d'))
26 | 
27 | fig = px.line(log_data, x='date', y='count', title='Total Count', color='status_code')
28 | fig.show()


--------------------------------------------------------------------------------
/Web-Scraping/data.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "url": "https://ca.indeed.com/Python-jobs",
 3 |   "h1": [
 4 |     "\n            Python jobs"
 5 |   ],
 6 |   "h2": [
 7 |     "\n\nBackend Integrations Engineer (Python)\n",
 8 |     "\n\nData\u00a0Analyst\n",
 9 |     "\n\ngeographic information systems (GIS) technologist\n",
10 |     "\n\nMacro Investment Research Co-op/Internship (Fall 2020)\n",
11 |     "\n\nEntry level Technical Informatics/Analytics Analyst\n",
12 |     "\n\nCash Operations Co-op/Internship (Fall 2020)\n",
13 |     "\n\nGIS Analyst\n",
14 |     "\n\nLinked Data (LINCS) Developer\n",
15 |     "\n\nIntern, Business Intelligence & Analytics\n",
16 |     "\n\nGIS Analyst\n"
17 |   ],
18 |   "description": [
19 |     "Search 3,862 Python jobs now available on Indeed.com, the world's largest job site."
20 |   ],
21 |   "title": [
22 |     "Python Jobs (with Salaries) | Indeed.com"
23 |   ],
24 |   "canonical": [
25 |     "/Python-jobs"
26 |   ],
27 |   "hreflang": [
28 |     "http://ca.indeed.com/rss?q=Python",
29 |     "/m/jobs?q=Python",
30 |     "/m/jobs?q=Python",
31 |     "android-app://com.indeed.android.jobsearch/https/ca.indeed.com/m/jobs?q=Python"
32 |   ],
33 |   "robot_tag": []
34 | }


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/list-of-urls.csv:
--------------------------------------------------------------------------------
 1 | https://www.jcchouinard.com/category/analytics/
 2 | https://www.jcchouinard.com/category/analytics/google-analytics/
 3 | https://www.jcchouinard.com/category/automation/
 4 | https://www.jcchouinard.com/category/coding/automation/
 5 | https://www.jcchouinard.com/category/coding/github/
 6 | https://www.jcchouinard.com/category/coding/python/
 7 | https://www.jcchouinard.com/category/coding/sql/
 8 | https://www.jcchouinard.com/category/github/
 9 | https://www.jcchouinard.com/category/python/
10 | https://www.jcchouinard.com/category/seo/
11 | https://www.jcchouinard.com/category/seo/a-b-split-testing/
12 | https://www.jcchouinard.com/category/social-medias/facebook/
13 | https://www.jcchouinard.com/category/web-dev/
14 | https://www.jcchouinard.com/category/analytics/google-search-console/
15 | https://www.jcchouinard.com/page/1/
16 | https://www.jcchouinard.com/page/2/
17 | https://www.jcchouinard.com/page/3/
18 | https://www.jcchouinard.com/page/4/
19 | https://www.jcchouinard.com/page/5/
20 | https://www.jcchouinard.com/page/6/
21 | https://www.jcchouinard.com/30-days-of-pandas/
22 | https://www.jcchouinard.com/a-tribute-to-hamlet-batista/
23 | https://www.jcchouinard.com/a-year-in-review/
24 | https://www.jcchouinard.com/ab-split-testing-resources/
25 | https://www.jcchouinard.com/web-stories/getting-started-with-django/
26 | https://www.jcchouinard.com/best-seo-posts/


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/list_of_urls.txt:
--------------------------------------------------------------------------------
 1 | https://www.jcchouinard.com/category/analytics/
 2 | https://www.jcchouinard.com/category/analytics/google-analytics/
 3 | https://www.jcchouinard.com/category/automation/
 4 | https://www.jcchouinard.com/category/coding/automation/
 5 | https://www.jcchouinard.com/category/coding/github/
 6 | https://www.jcchouinard.com/category/coding/python/
 7 | https://www.jcchouinard.com/category/coding/sql/
 8 | https://www.jcchouinard.com/category/github/
 9 | https://www.jcchouinard.com/category/python/
10 | https://www.jcchouinard.com/category/seo/
11 | https://www.jcchouinard.com/category/seo/a-b-split-testing/
12 | https://www.jcchouinard.com/category/social-medias/facebook/
13 | https://www.jcchouinard.com/category/web-dev/
14 | https://www.jcchouinard.com/category/analytics/google-search-console/
15 | https://www.jcchouinard.com/page/1/
16 | https://www.jcchouinard.com/page/2/
17 | https://www.jcchouinard.com/page/3/
18 | https://www.jcchouinard.com/page/4/
19 | https://www.jcchouinard.com/page/5/
20 | https://www.jcchouinard.com/page/6/
21 | https://www.jcchouinard.com/30-days-of-pandas/
22 | https://www.jcchouinard.com/a-tribute-to-hamlet-batista/
23 | https://www.jcchouinard.com/a-year-in-review/
24 | https://www.jcchouinard.com/ab-split-testing-resources/
25 | https://www.jcchouinard.com/web-stories/getting-started-with-django/
26 | https://www.jcchouinard.com/best-seo-posts/


--------------------------------------------------------------------------------
/Causal-impact/run.py:
--------------------------------------------------------------------------------
 1 | # Based on this blog post: https://www.jcchouinard.com/pycausalimpact/
 2 | import os
 3 | import searchconsole
 4 | 
 5 | from causal import Causal
 6 | 
 7 | # define metric that you want to test
 8 | # impressions, clicks, ctr
 9 | metric = 'clicks' 
10 | 
11 | # define intervention data
12 | intervention = '2021-08-01'
13 | 
14 | # give the path of your credential file
15 | client_secrets = 'client_secrets.json'
16 | 
17 | # define sites on which you ran the experiment (required)
18 | test_sites = [
19 |     'https://ca.example.com/', 
20 |     'https://us.example.com/', 
21 |     'https://au.example.com/'
22 |     ]
23 | 
24 | # define control sites that were not shown the experiment (optional)
25 | # set list as empty [] to run simple pre-post experiment
26 | control_sites = [
27 |     'https://www.example.fr/',
28 |     'https://uk.example.com/'
29 |     ]
30 | 
31 | 
32 | def authenticate(config='client_secrets.json', token='credentials.json'):
33 |     """Authenticate GSC"""
34 |     if os.path.isfile(token):
35 |         account = searchconsole.authenticate(client_config=config,
36 |                                             credentials=token)
37 |     else:
38 |         account = searchconsole.authenticate(client_config=config,
39 |                                         serialize=token)
40 |     return account
41 | 
42 | 
43 | if __name__ == '__main__':
44 |     account = authenticate(config=client_secrets)
45 |     c = Causal(
46 |         account,
47 |         intervention,
48 |         test_sites,
49 |         control_sites=control_sites, 
50 |         metric='clicks')
51 |     c.run_causal()


--------------------------------------------------------------------------------
/Reddit/simple_request.py:
--------------------------------------------------------------------------------
 1 | import json 
 2 | import pandas as pd
 3 | import requests
 4 |  
 5 | subreddit = 'python'
 6 | limit = 1
 7 | timeframe = 'month' #hour, day, week, month, year, all
 8 | listing = 'top' # controversial, best, hot, new, random, rising, top
 9 |  
10 | def get_reddit(subreddit,listing,limit,timeframe):
11 |     try:
12 |         base_url = f'https://www.reddit.com/r/{subreddit}/{listing}.json?limit={limit}&t={timeframe}'
13 |         request = requests.get(base_url, headers = {'User-agent': 'yourbot'})
14 |     except:
15 |         print('An Error Occured')
16 |     return request.json()
17 | 
18 | def show_tags(r):
19 |     '''
20 |     show what tags you can use
21 |     '''
22 |     for post in r['data']['children']:
23 |         for k in post['data'].keys():
24 |             print(k)
25 | 
26 | def get_post_titles(r):
27 |     '''
28 |     Get a List of post titles
29 |     '''
30 |     posts = []
31 |     for post in r['data']['children']:
32 |         x = post['data']['title']
33 |         posts.append(x)
34 |     return posts
35 | 
36 | def get_results(r):
37 |     '''
38 |     Create a DataFrame Showing Title, URL, Score and Number of Comments.
39 |     '''
40 |     myDict = {}
41 |     for post in r['data']['children']:
42 |         myDict[post['data']['title']] = {'url':post['data']['url'],'score':post['data']['score'],'comments':post['data']['num_comments']}
43 |     df = pd.DataFrame.from_dict(myDict, orient='index')
44 |     return df
45 | 
46 | if __name__ == '__main__':
47 |     r = get_reddit(subreddit,listing,limit,timeframe)
48 |     posts = get_post_titles(r)
49 |     df = get_results(r)
50 | 
51 | 
52 | 


--------------------------------------------------------------------------------
/Reddit/README.md:
--------------------------------------------------------------------------------
 1 | # Reddit
 2 | 
 3 | ## Other Interesting Guides from The Author
 4 | 
 5 | [Python for SEO: Complete Guide (in 5 Chapters)](https://www.jcchouinard.com/python-for-seo/)
 6 | 
 7 | - [Install Python With Anaconda (On Windows)](https://www.jcchouinard.com/install-python-with-anaconda-on-windows/)
 8 | - [Python Libraries for Beginners](https://www.jcchouinard.com/python-libraries-for-seo/)
 9 | 
10 | [Learn Git and Github (Complete Guide)](https://www.jcchouinard.com/learn-git-and-github/)
11 | 
12 | - [Clone Github Repository on Windows](https://www.jcchouinard.com/clone-github-repository-on-windows/)
13 | - [Add a File to Github Using Git Bash](https://www.jcchouinard.com/add-a-file-to-github-with-git-bash/)
14 | 
15 | Automation
16 | 
17 | - [Python Script Automation Using Task Scheduler (Windows)](https://www.jcchouinard.com/python-automation-using-task-scheduler/)
18 | - [Python Script Automation Using CRON (Mac)](https://www.jcchouinard.com/python-automation-with-cron-on-mac/)
19 | 
20 | WebScraping
21 | 
22 | - [Web Scraping With Python and Requests-HTML](https://www.jcchouinard.com/web-scraping-with-python-and-requests-html/)
23 | - [How to use Selenium](https://www.jcchouinard.com/learn-selenium-python-seo-automation/)
24 | 
25 | APIs
26 | 
27 | - [Google Search Console API: Complete Guide (in 4 Chapters)](https://www.jcchouinard.com/google-search-console-api/)
28 | - [Intro to GSC API with Python (Video)](https://www.jcchouinard.com/intro-to-gsc-api-with-python/)
29 | - [Google Analytics Reporting API v4 using Python [2020 update]](https://www.jcchouinard.com/google-analytics-api-using-python/)
30 | - [How to use Reddit API With Python](https://www.jcchouinard.com/how-to-use-reddit-api-with-python/)
31 | 
32 | Others
33 | 
34 | - [Chrome DevTools for SEO](https://www.jcchouinard.com/chrome-devtools-commands-for-seo/)


--------------------------------------------------------------------------------
/GSC_Tutorial/gsc_by_url.py:
--------------------------------------------------------------------------------
 1 | # Standard library imports
 2 | import pandas as pd
 3 | 
 4 | # Third party modules imports
 5 | from collections import defaultdict
 6 | import datetime
 7 | from dateutil import relativedelta
 8 | 
 9 | # Local modules imports
10 | from date_manip import date_to_str
11 | from oauth import authorize_creds, execute_request
12 | 
13 | today = datetime.date.today()
14 | days = relativedelta.relativedelta(days=3)
15 | default_end = today - days 
16 | 
17 | def gsc_by_url(list_of_urls,creds,start_date,end_date=default_end,storage='authorizedcreds.dat'):
18 |     webmasters_service = authorize_creds(creds)     # Get credentials to log in the api
19 |     scDict = defaultdict(list)
20 |     for url in list_of_urls:
21 |         request = {
22 |                     'startDate': date_to_str(start_date),
23 |                     'endDate': date_to_str(end_date),
24 |                     'dimensions': 'page',  #country, device, page, query, searchAppearance
25 |                     'dimensionFilterGroups': [{
26 |                     'filters': [{
27 |                         'dimension': 'page',              
28 |                         'operator': 'equals',           #contains, equals, notEquals, notContains
29 |                         'expression': url
30 |                     }]
31 |                     }]
32 |             }
33 | 
34 |         response = execute_request(webmasters_service, site, request)
35 |     
36 |         scDict['page'].append(url)
37 | 
38 |         try:
39 |             for row in response['rows']:
40 |                 scDict['clicks'].append(row['clicks'] or 0)
41 |                 scDict['impressions'].append(row['impressions'] or 0)
42 |         except Exception as e:
43 |             print(f'An error occurred while extracting {url}: {e}')
44 |     # Add response to dataframe 
45 |     df = pd.DataFrame(data = scDict)
46 |     return df


--------------------------------------------------------------------------------
/Web-Scraping/rss_reader.py:
--------------------------------------------------------------------------------
 1 | # Read a RSS Feed with Python and BeautifulSoup
 2 | # https://www.jcchouinard.com/read-rss-feed-with-python/
 3 | 
 4 | # Author: Jean-Christophe Chouinard
 5 | # Role: Sr. SEO Specialist at Seek.com.au
 6 | # twitter: twitter.com/ChouinardJC
 7 | # linkedin: linkedin.com/in/jeanchristophechouinard
 8 | 
 9 | from bs4 import BeautifulSoup
10 | import requests
11 | 
12 | headers = {
13 |             'User-Agent': 'your-user-agent-here'
14 |         }
15 | 
16 | class ReadRss:
17 | 
18 |     def __init__(self, rss_url, headers):
19 | 
20 |         self.url = rss_url
21 |         self.headers = headers
22 |         try:
23 |             self.r = requests.get(rss_url, headers=self.headers)
24 |             self.status_code = self.r.status_code
25 |         except Exception as e:
26 |             print('Error fetching the URL: ', rss_url)
27 |             print(e)
28 |         try:    
29 |             self.soup = BeautifulSoup(self.r.text, 'lxml')
30 |         except Exception as e:
31 |             print('Could not parse the xml: ', self.url)
32 |             print(e)
33 |         self.articles = self.soup.findAll('item')
34 |         self.articles_dicts = [{'title':a.find('title').text,'link':a.link.next_sibling.replace('\n','').replace('\t',''),'description':a.find('description').text,'pubdate':a.find('pubdate').text} for a in self.articles]
35 |         self.urls = [d['link'] for d in self.articles_dicts if 'link' in d]
36 |         self.titles = [d['title'] for d in self.articles_dicts if 'title' in d]
37 |         self.descriptions = [d['description'] for d in self.articles_dicts if 'description' in d]
38 |         self.pub_dates = [d['pubdate'] for d in self.articles_dicts if 'pubdate' in d]
39 | 
40 | if __name__ == '__main__':
41 | 
42 |     feed = ReadRss('https://www.jcchouinard.com/author/jean-christophe-chouinard/feed/', headers)
43 |     print(feed.urls)


--------------------------------------------------------------------------------
/GSC_Tutorial/gsc_with_filters.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | 
 3 | from collections import defaultdict
 4 | import datetime
 5 | from dateutil import relativedelta
 6 | 
 7 | from oauth import authorize_creds, execute_request
 8 | 
 9 | today = datetime.date.today()
10 | days = relativedelta.relativedelta(days=3)
11 | default_end = today - days 
12 | 
13 | # Run the extraction
14 | def gsc_with_filters(site,creds,start_date,end_date=default_end,storage='authorizedcreds.dat'):
15 | 
16 |     scDict = defaultdict(list) # Create a dict to populate with extraction
17 |     webmasters_service = authorize_creds(creds,storage) # Authorize the API
18 |     request = {
19 |         'startDate': start_date,
20 |         'endDate': end_date,
21 |         'dimensions': ['date','page','query'],  #country, device, page, query, searchAppearance
22 |         'dimensionFilterGroups': [{
23 |                         'filters': [{
24 |                             'dimension': 'query',              
25 |                             'operator': 'contains',           #contains, equals, notEquals, notContains
26 |                             'expression': 'chouinard'
27 |                         }]
28 |                         }]
29 |     }
30 | 
31 |     response = execute_request(webmasters_service, site, request)
32 |     
33 |     try:
34 |         for row in response['rows']:
35 |             scDict['date'].append(row['keys'][0] or 0)    
36 |             scDict['page'].append(row['keys'][1] or 0)
37 |             scDict['query'].append(row['keys'][2] or 0)
38 |             scDict['clicks'].append(row['clicks'] or 0)
39 |             #scDict['ctr'].append(row['ctr'] or 0)
40 |             #scDict['impressions'].append(row['impressions'] or 0)
41 |             #scDict['position'].append(row['position'] or 0)
42 |     except Exception as e:
43 |         print(f'An error occurred: {e}')
44 | 
45 |     # Add response to dataframe 
46 |     df = pd.DataFrame(data = scDict)
47 |     return df
48 | 


--------------------------------------------------------------------------------
/Causal-impact/simple_extract.py:
--------------------------------------------------------------------------------
 1 | # Based on this blog post: [SEO Experiment with pyCausalImpact and GSC](https://www.jcchouinard.com/pycausalimpact/)
 2 | from causalimpact import CausalImpact
 3 | import pandas as pd 
 4 | 
 5 | # load control and test data
 6 | X = pd.read_csv('control_gsc_data.csv')
 7 | y = pd.read_csv('test_gsc_data.csv')
 8 | 
 9 | # define metric that you want to test
10 | # impressions, clicks, ctr
11 | metric = 'clicks' 
12 | 
13 | # define intervention data
14 | intervention = '2021-08-01'
15 | 
16 | 
17 | def get_pre_post(data):
18 |     """Get pre-post periods based on the intervention date
19 | 
20 |     Args:
21 |         data (dataframe): df comming from create_master_df()
22 | 
23 |     Returns:
24 |         tuple: tuple of lists showing index edges of period before and after intervention
25 |     """        
26 |     pre_start = min(data.index)
27 |     pre_end = int(data[data['date'] == intervention].index.values)
28 |     post_start = pre_end + 1
29 |     post_end = max(data.index)
30 | 
31 |     pre_period = [pre_start, pre_end] 
32 |     post_period = [post_start, post_end]
33 |     return pre_period, post_period
34 | 
35 | 
36 | def make_report(data, pre_period, post_period):
37 |     """Creates the built-in CausalImpact report
38 | 
39 |     Args:
40 |         data (dataframe): df comming from create_master_df()
41 |         pre_period (list): list coming from get_pre_post()
42 |         post_period (list): list coming from get_pre_post()
43 |     """        
44 |     ci = CausalImpact(data.drop(['date'], axis=1), pre_period, post_period)
45 |     print(ci.summary())
46 |     print(ci.summary(output='report'))
47 |     ci.plot()
48 | 
49 | 
50 | if __name__ == '__main__':
51 |     y = y[['date', metric]].rename(columns={metric:'y'})
52 |     X = X[['date', metric]].rename(columns={metric:'X'})
53 |     data = y.merge(X, on='date', how='left')
54 |     data.sort_values(by='date').reset_index(drop=True)
55 |     pre_period, post_period = get_pre_post(data)
56 |     make_report(data, pre_period, post_period)


--------------------------------------------------------------------------------
/GSC_Tutorial/oauth.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import httplib2
 3 | import requests
 4 | 
 5 | from collections import defaultdict
 6 | from dateutil import relativedelta
 7 | from googleapiclient.discovery import build
 8 | #from oauth2client.client import OAuth2WebServerFlow
 9 | from oauth2client import client
10 | from oauth2client import file
11 | from oauth2client import tools
12 | 
13 | def authorize_creds(creds, storage='authorizedcreds.dat'):
14 |     print('Authorizing creds')
15 |     # Variable parameter that controls the set of resources that the access token permits.
16 |     SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly'] 
17 | 
18 |     # Path to client_secrets.json file
19 |     CLIENT_SECRETS_PATH = creds
20 | 
21 |     # Create a parser to be able to open browser for Authorization
22 |     parser = argparse.ArgumentParser(
23 |         formatter_class=argparse.RawDescriptionHelpFormatter,
24 |         parents=[tools.argparser])
25 |     flags = parser.parse_args([])
26 | 
27 |     flow = client.flow_from_clientsecrets(
28 |         CLIENT_SECRETS_PATH, scope = SCOPES,
29 |         message = tools.message_if_missing(CLIENT_SECRETS_PATH))
30 | 
31 |     # Prepare credentials and authorize HTTP
32 |     # If they exist, get them from the storage object
33 |     # credentials will get written back to a file.
34 |     storage = file.Storage(storage)
35 |     credentials = storage.get()
36 | 
37 |     # If authenticated credentials don't exist, open Browser to authenticate
38 |     if credentials is None or credentials.invalid:
39 |         credentials = tools.run_flow(flow, 'authorizedcreds.dat', flags)
40 |     http = credentials.authorize(http=httplib2.Http())
41 |     webmasters_service = build('webmasters', 'v3', http=http)
42 |     print('Successful authorization')
43 |     return webmasters_service
44 | 
45 | # Create Function to execute your API Request
46 | def execute_request(service, property_uri, request):
47 |     return service.searchanalytics().query(siteUrl=property_uri, body=request).execute()
48 | 
49 | 


--------------------------------------------------------------------------------
/Google-Search-Console/createMysqlDb.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | """
 3 | This Script will help create a mySQL database using Python.
 4 | 
 5 | @author: Jean-Christophe Chouinard: Technical SEO / Data Scientist. 
 6 | Twitter > https://www.twitter.com/ChouinardJC, 
 7 | LinkedIn > https://www.linkedin.com/in/jeanchristophechouinard/, 
 8 | Blog > https://www.jcchouinard.com/
 9 | 
10 | 
11 | Install packages: conda install -c anaconda pymysql
12 | """
13 | 
14 | import pymysql.cursors
15 | 
16 | # Create a mySQL Database
17 | # Establish connection
18 | connection = pymysql.connect(host='localhost',
19 |                              user='root',
20 |                              port='',
21 |                              password='')
22 | 
23 | # Simulate the CREATE DATABASE function of mySQL
24 | try:
25 |     with connection.cursor() as cursor:
26 |         cursor.execute('CREATE DATABASE gsc_db')
27 | 
28 | finally:
29 |     connection.close()
30 |     
31 |     
32 | # Create a table
33 | connection = pymysql.connect(host='localhost',
34 |                              user='root',
35 |                              port='',
36 |                              password='',
37 |                              db='gsc_db',
38 |                              cursorclass=pymysql.cursors.DictCursor)
39 | 
40 | try:
41 |     with connection.cursor() as cursor:
42 |         sqlQuery = '''CREATE TABLE IF NOT EXISTS backup_gsc(Date DATE, 
43 |                                                             Page LONGTEXT, 
44 |                                                             Query LONGTEXT, 
45 |                                                             Clicks INT, 
46 |                                                             Impressions INT, 
47 |                                                             Ctr DECIMAL(10,2), 
48 |                                                             Position DECIMAL(3,2))''' #Set-up the Schema datatype. DECIMAL(3,2) = 3 digits before the decimal, 2 after the decimal 
49 |         cursor.execute(sqlQuery)
50 | finally:
51 |     connection.close()
52 |     
53 |     
54 | 


--------------------------------------------------------------------------------
/Google-Search-Console/README.md:
--------------------------------------------------------------------------------
 1 | # Google-Search-Console
 2 | This repo is related to making Google Search Console API calls.
 3 | 
 4 | @author: Jean-Christophe Chouinard: Technical SEO / Data Scientist. 
 5 | Twitter > https://www.twitter.com/ChouinardJC, 
 6 | LinkedIn > https://www.linkedin.com/in/jeanchristophechouinard/, 
 7 | Blog > https://www.jcchouinard.com/
 8 | 
 9 | 
10 | ## Other Interesting Guides from The Author
11 | 
12 | [Python for SEO: Complete Guide (in 5 Chapters)](https://www.jcchouinard.com/python-for-seo/)
13 | 
14 | - [Install Python With Anaconda (On Windows)](https://www.jcchouinard.com/install-python-with-anaconda-on-windows/)
15 | - [Python Libraries for Beginners](https://www.jcchouinard.com/python-libraries-for-seo/)
16 | 
17 | [Learn Git and Github (Complete Guide)](https://www.jcchouinard.com/learn-git-and-github/)
18 | 
19 | - [Clone Github Repository on Windows](https://www.jcchouinard.com/clone-github-repository-on-windows/)
20 | - [Add a File to Github Using Git Bash](https://www.jcchouinard.com/add-a-file-to-github-with-git-bash/)
21 | 
22 | Automation
23 | 
24 | - [Python Script Automation Using Task Scheduler (Windows)](https://www.jcchouinard.com/python-automation-using-task-scheduler/)
25 | - [Python Script Automation Using CRON (Mac)](https://www.jcchouinard.com/python-automation-with-cron-on-mac/)
26 | 
27 | WebScraping
28 | 
29 | - [Web Scraping With Python and Requests-HTML](https://www.jcchouinard.com/web-scraping-with-python-and-requests-html/)
30 | - [How to use Selenium](https://www.jcchouinard.com/learn-selenium-python-seo-automation/)
31 | 
32 | APIs
33 | 
34 | - [Google Search Console API: Complete Guide (in 4 Chapters)](https://www.jcchouinard.com/google-search-console-api/)
35 | - [Intro to GSC API with Python (Video)](https://www.jcchouinard.com/intro-to-gsc-api-with-python/)
36 | - [Google Analytics Reporting API v4 using Python [2020 update]](https://www.jcchouinard.com/google-analytics-api-using-python/)
37 | - [How to use Reddit API With Python](https://www.jcchouinard.com/how-to-use-reddit-api-with-python/)
38 | 
39 | Others
40 | 
41 | - [Chrome DevTools for SEO](https://www.jcchouinard.com/chrome-devtools-commands-for-seo/)


--------------------------------------------------------------------------------
/GoogleMyBusiness/gmb_cat_finder_@zchtodd.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | '''
 3 | What GMB Categories are competition using with Python and Google Places API
 4 | (Billing Enablement Required)
 5 | 
 6 | Original Post: https://www.jcchouinard.com/find-competition-gmb-categories-with-python/
 7 | @author:    Zach Todd 
 8 | @Twitter:   twitter.com/zchtodd
 9 | '''
10 | 
11 | import requests
12 | import json
13 | import re
14 | import csv
15 | import sys
16 | import os
17 | 
18 | def get_places(keyword, lat, lng):
19 |     place_results = requests.get(
20 |         "https://maps.googleapis.com/maps/api/place/nearbysearch/json",
21 |         params={
22 |             "key": os.environ["GOOGLE_MAPS_API_KEY"],
23 |             "keyword": keyword,
24 |             "location": "{},{}".format(lat, lng),
25 |             "radius": 5000,
26 |         },
27 |     ).json()
28 | 
29 |     return place_results["results"]
30 | 
31 | 
32 | def get_place_details(place_id):
33 |     place_details = requests.get(
34 |         "https://maps.googleapis.com/maps/api/place/details/json",
35 |         params={
36 |             "key": os.environ["GOOGLE_MAPS_API_KEY"],
37 |             "place_id": place_id,
38 |             "fields": "url",
39 |         },
40 |     ).json()
41 | 
42 |     return place_details["result"]
43 | 
44 | 
45 | def get_location_categories(cid):
46 |     response = requests.get(
47 |         "https://www.google.com/maps?cid={}&hl=en".format(cid),
48 |         proxies={"http": os.environ["PROXY_URL"], "https": os.environ["PROXY_URL"]},
49 |         timeout=10,
50 |     )
51 | 
52 |     start = response.text.find("window.APP_INITIALIZATION_STATE=")
53 |     end = response.text.find("window.APP_FLAGS", start)
54 | 
55 |     if start > 0 and end > 0:
56 |         content = json.loads(
57 |             response.text[start + len("window.APP_INITIALIZATION_STATE=") : end - 1]
58 |         )
59 |         content = json.loads(content[3][6][5:])
60 |         return content[6][13]
61 | 
62 |     return []
63 | 
64 | 
65 | def find_categories(keyword, lat, lng):
66 |     writer = csv.writer(open("categories.csv", "w"), dialect="excel")
67 |     writer.writerow(["URL", "Name", "Categories"])
68 | 
69 |     for place in get_places(keyword, lat, lng):
70 |         details = get_place_details(place["place_id"])
71 | 
72 |         try:
73 |             categories = get_location_categories(details["url"].split("=")[1])
74 |         except:
75 |             continue
76 | 
77 |         writer.writerow([details["url"], place["name"], ", ".join(categories)])
78 | 
79 | 
80 | if __name__ == "__main__":
81 |     find_categories(sys.argv[1], sys.argv[2], sys.argv[3])


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/create-simple-sitemaps.py:
--------------------------------------------------------------------------------
 1 | # This Script Will Help You Create a XML Sitemap From a List of URLS
 2 | # @author: Jean-Christophe Chouinard: Technical SEO / Data Scientist. 
 3 | # Twitter > https://www.twitter.com/ChouinardJC, 
 4 | # LinkedIn > https://www.linkedin.com/in/jeanchristophechouinard/, 
 5 | # Blog > https://www.jcchouinard.com/create-xml-sitemap-with-python/
 6 | 
 7 | import pandas as pd
 8 | import os
 9 | import datetime 
10 | from jinja2 import Template
11 | import gzip
12 | 
13 | # Import List of URLs
14 | # list_of_urls = pd.read_csv('list-of-urls.csv',header=None)
15 | 
16 | #or
17 | with open('list_of_urls.txt','r') as f:
18 |     list_of_urls = f.read()
19 |     list_of_urls = list_of_urls.split('\n')
20 | 
21 | 
22 | # Set-Up Maximum Number of URLs (recommended max 50,000)
23 | # Added 20 here because I am using a small template of URLs and I want to show an example
24 | n = 200
25 | 
26 | # Create New Empty Row to Store the Splitted File Number
27 | list_of_urls.loc[:,'name'] = ''
28 | 
29 | # Split the file with the maximum number of rows specified
30 | new_df = [list_of_urls[i:i+n] for i in range(0,list_of_urls.shape[0],n)]
31 | 
32 | # For Each File Created, add a file number to a new column of the dataframe
33 | for i,v in enumerate(new_df):
34 |     v.loc[:,'name'] = str(v.iloc[0,1])+'_'+str(i)
35 |     print(v)
36 |             
37 | # Create a Sitemap Template to Populate
38 | 
39 | sitemap_template='''<?xml version="1.0" encoding="UTF-8"?>
40 | <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
41 |     {% for page in pages %}
42 |     <url>
43 |         <loc>{{page[1]|safe}}</loc>
44 |         <lastmod>{{page[3]}}</lastmod>
45 |         <changefreq>{{page[4]}}</changefreq>
46 |         <priority>{{page[5]}}</priority>        
47 |     </url>
48 |     {% endfor %}
49 | </urlset>'''
50 | 
51 | template = Template(sitemap_template)
52 | 
53 | # Get Today's Date to add as Lastmod
54 | lastmod_date = datetime.datetime.now().strftime('%Y-%m-%d')
55 | 
56 | # Fill the Sitemap Template and Write File
57 | for i in new_df:                           # For each URL in the list of URLs ...                                                          
58 |     i.loc[:,'lastmod'] = lastmod_date      # ... add Lastmod date
59 |     i.loc[:,'changefreq'] = 'never'        # ... add changefreq
60 |     i.loc[:,'priority'] = '1.0'            # ... add priority 
61 | 
62 |     # Render each row / column in the sitemap
63 |     sitemap_output = template.render(pages = i.itertuples()) 
64 |     
65 |     # Create a filename for each sitemap like: sitemap_0.xml.gz, sitemap_1.xml.gz, etc.
66 |     filename = 'sitemap' + str(i.iloc[0,1]) + '.xml' 
67 | 
68 |     # Write the File to Your Working Folder
69 |     with open(filename, 'wt') as f:   
70 |         f.write(sitemap_output)
71 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SEO-Projects
 2 | Repository of all public SEO projects I am working on.
 3 | 
 4 | 
 5 | @author: Jean-Christophe Chouinard: Technical SEO / Data Scientist. 
 6 | Twitter > https://www.twitter.com/ChouinardJC, 
 7 | LinkedIn > https://www.linkedin.com/in/jeanchristophechouinard/, 
 8 | Blog > https://www.jcchouinard.com/
 9 | 
10 | Python scripts details are available on jcchouinard.com.
11 | 
12 | [Get Google Search Console API Keys](https://www.jcchouinard.com/how-to-get-google-search-console-api-keys/)
13 | robotstxt_to_df.py: [Parse Robots.txt to a DataFrame with Python](https://www.jcchouinard.com/robots-txt-parsing-with-python/)
14 | Connect search console using OAuth keys.ipynb: [How to Use Google Search Console API With Python](https://www.jcchouinard.com/how-to-connect-to-google-search-console-api-using-python/)
15 | GSC_to_CSV_All_keywords.py: [Get All Your Search traffic With Google Search Console API (With Code Sample)](https://www.jcchouinard.com/get-all-keywords-with-search-console-api/)
16 | gscbackups.py: [Backup Google Search Console Data Into MySQL With Python](https://www.jcchouinard.com/backup-google-search-console-data-into-mysql-with-python/)
17 | 
18 | ## Other Interesting Guides from The Author
19 | 
20 | [Python for SEO: Complete Guide (in 5 Chapters)](https://www.jcchouinard.com/python-for-seo/)
21 | 
22 | - [Install Python With Anaconda (On Windows)](https://www.jcchouinard.com/install-python-with-anaconda-on-windows/)
23 | - [Python Libraries for Beginners](https://www.jcchouinard.com/python-libraries-for-seo/)
24 | 
25 | [Learn Git and Github (Complete Guide)](https://www.jcchouinard.com/learn-git-and-github/)
26 | 
27 | - [Clone Github Repository on Windows](https://www.jcchouinard.com/clone-github-repository-on-windows/)
28 | - [Add a File to Github Using Git Bash](https://www.jcchouinard.com/add-a-file-to-github-with-git-bash/)
29 | 
30 | Automation
31 | 
32 | - [Python Script Automation Using Task Scheduler (Windows)](https://www.jcchouinard.com/python-automation-using-task-scheduler/)
33 | - [Python Script Automation Using CRON (Mac)](https://www.jcchouinard.com/python-automation-with-cron-on-mac/)
34 | 
35 | WebScraping
36 | 
37 | - [Web Scraping With Python and Requests-HTML](https://www.jcchouinard.com/web-scraping-with-python-and-requests-html/)
38 | - [How to use Selenium](https://www.jcchouinard.com/learn-selenium-python-seo-automation/)
39 | 
40 | APIs
41 | 
42 | - [Google Search Console API: Complete Guide (in 4 Chapters)](https://www.jcchouinard.com/google-search-console-api/)
43 | - [Intro to GSC API with Python (Video)](https://www.jcchouinard.com/intro-to-gsc-api-with-python/)
44 | - [Google Analytics Reporting API v4 using Python [2020 update]](https://www.jcchouinard.com/google-analytics-api-using-python/)
45 | - [How to use Reddit API With Python](https://www.jcchouinard.com/how-to-use-reddit-api-with-python/)
46 | 
47 | Others
48 | 
49 | - [Chrome DevTools for SEO](https://www.jcchouinard.com/chrome-devtools-commands-for-seo/)


--------------------------------------------------------------------------------
/Reddit/textsummarizer.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # coding: utf-8
 3 | 
 4 | from nltk.corpus import stopwords
 5 | from nltk.cluster.util import cosine_distance
 6 | import numpy as np
 7 | import networkx as nx
 8 |  
 9 | def read_article(file_name):
10 |     file = open(file_name, "r")
11 |     filedata = file.readlines()
12 |     article = filedata[0].split(". ")
13 |     sentences = []
14 | 
15 |     for sentence in article:
16 |         print(sentence)
17 |         sentences.append(sentence.replace("[^a-zA-Z]", " ").split(" "))
18 |     sentences.pop() 
19 |     
20 |     return sentences
21 | 
22 | def sentence_similarity(sent1, sent2, stopwords=None):
23 |     if stopwords is None:
24 |         stopwords = []
25 |  
26 |     sent1 = [w.lower() for w in sent1]
27 |     sent2 = [w.lower() for w in sent2]
28 |  
29 |     all_words = list(set(sent1 + sent2))
30 |  
31 |     vector1 = [0] * len(all_words)
32 |     vector2 = [0] * len(all_words)
33 |  
34 |     # build the vector for the first sentence
35 |     for w in sent1:
36 |         if w in stopwords:
37 |             continue
38 |         vector1[all_words.index(w)] += 1
39 |  
40 |     # build the vector for the second sentence
41 |     for w in sent2:
42 |         if w in stopwords:
43 |             continue
44 |         vector2[all_words.index(w)] += 1
45 |  
46 |     return 1 - cosine_distance(vector1, vector2)
47 |  
48 | def build_similarity_matrix(sentences, stop_words):
49 |     # Create an empty similarity matrix
50 |     similarity_matrix = np.zeros((len(sentences), len(sentences)))
51 |  
52 |     for idx1 in range(len(sentences)):
53 |         for idx2 in range(len(sentences)):
54 |             if idx1 == idx2: #ignore if both are same sentences
55 |                 continue 
56 |             similarity_matrix[idx1][idx2] = sentence_similarity(sentences[idx1], sentences[idx2], stop_words)
57 | 
58 |     return similarity_matrix
59 | 
60 | 
61 | def generate_summary(file_name, top_n=5):
62 |     stop_words = stopwords.words('english')
63 |     summarize_text = []
64 | 
65 |     # Step 1 - Read text anc split it
66 |     sentences =  read_article(file_name)
67 | 
68 |     # Step 2 - Generate Similary Martix across sentences
69 |     sentence_similarity_martix = build_similarity_matrix(sentences, stop_words)
70 | 
71 |     # Step 3 - Rank sentences in similarity martix
72 |     sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)
73 |     scores = nx.pagerank(sentence_similarity_graph)
74 | 
75 |     # Step 4 - Sort the rank and pick top sentences
76 |     ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)    
77 |     print("Indexes of top ranked_sentence order are ", ranked_sentence)    
78 | 
79 |     for i in range(top_n):
80 |       summarize_text.append(" ".join(ranked_sentence[i][1]))
81 | 
82 |     # Step 5 - Offcourse, output the summarize texr
83 |     print("Summarize Text: \n", ". ".join(summarize_text))
84 | 
85 | # let's begin
86 | generate_summary( "msft.txt", 2)


--------------------------------------------------------------------------------
/robotstxt_to_df.py:
--------------------------------------------------------------------------------
 1 | '''
 2 | Parse Robots.txt to a DataFrame with Python
 3 | 
 4 | This script will read the robots.txt file.
 5 | Parse the robots.txt and add it to a Dataframe.
 6 | 
 7 | @author: Jean-Christophe Chouinard. Technical SEO and Python for SEO. 
 8 | @website: jcchouinard.com
 9 | @original post: https://www.jcchouinard.com/robots-txt-parsing-with-python/‎
10 | @other work: https://www.jcchouinard.com/python-for-seo, https://www.jcchouinard.com/google-search-console-api
11 | @LinkedIn: https://www.linkedin.com/in/jeanchristophechouinard/ 
12 | @Twitter: https://www.twitter.com/@ChouinardJC
13 | '''
14 | 
15 | import pandas as pd
16 | import os
17 | from urllib.parse import urlparse
18 | 
19 | ua = 'User-agent'
20 | url = 'https://www.example.com' 
21 | 
22 | def get_robots_url(url):
23 |     domain_url = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(url))
24 |     robots_url = domain_url + '/robots.txt'
25 |     return robots_url
26 | 
27 | def read_robots_txt(url):
28 |     robot_url = get_robots_url(url)
29 |     robot_file = os.popen(f'curl {robot_url}').read()
30 |     return robot_file
31 | 
32 | 
33 | def initialize_dict(url):
34 |     robot_file = read_robots_txt(url)
35 |     result_data_set = {ua:{}}
36 |     for line in robot_file.split("\n"):
37 |         if line.startswith(ua):
38 |             result_data_set[ua].update({line.split(':')[1].strip():{}})
39 |     keys = []
40 |     for key in result_data_set[ua].keys():
41 |         keys.append(key)
42 |     return result_data_set, keys, robot_file
43 | 
44 | def parse_robot(url):
45 |     idict = initialize_dict(url)
46 |     result_data_set = idict[0]
47 |     keys = idict[1]
48 |     robot_file = idict[2]
49 |     print_flag = False
50 |     for i in range(len(keys)):
51 |         if i <= len(keys)-2:
52 |             end_str = keys[i+1]
53 |         else:
54 |             end_str = 'We are done'
55 | 
56 |         result_data_set[ua][keys[i]]['Disallow'] = []
57 |         result_data_set[ua][keys[i]]['Allow'] = []
58 |         for line in robot_file.split("\n"):
59 |             if end_str in line:
60 |                 print_flag = False
61 |             elif keys[i] in line:
62 |                 print_flag = True
63 |             elif print_flag:
64 |                 if line.startswith('Disallow') or line.startswith('Allow'):
65 |                     status = line.split(':')[0].strip()
66 |                     val = line.split(':')[1].strip()
67 |                     result_data_set[ua][keys[i]][status].append(val)
68 |     return result_data_set
69 | 
70 | def robots_to_df(url):
71 |     result_data_set = parse_robot(url)
72 |     ls = {ua:[],'Status':[],'Pattern':[]}
73 |     for k,v in result_data_set.items():
74 |         for v in result_data_set[k]:
75 |             for key,value in result_data_set[k][v].items():
76 |                 for value in result_data_set[k][v][key]:
77 |                     ls[ua].append(v)
78 |                     ls['Status'].append(key)
79 |                     ls['Pattern'].append(value)
80 |     robots_df = pd.DataFrame.from_dict(ls)
81 |     return robots_df
82 | 
83 | 
84 | robots_to_df(url)


--------------------------------------------------------------------------------
/Web-Scraping/robots_parser.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | from bs4 import BeautifulSoup
 3 | from urllib.parse import urlparse
 4 | from reppy import Robots
 5 | from fake_useragent import UserAgent
 6 | import json
 7 | 
 8 | ua = UserAgent()
 9 | 
10 | url = 'https://ca.indeed.com/Python-jobs'
11 | bad_url = 'https://ca.indeed.com/jobs?q=Python&from=brws&nc=brws'
12 | 
13 | def get_bot_loc(url):
14 |     domain_url = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(url))
15 |     bot_loc = domain_url + '/robots.txt'
16 |     return bot_loc
17 | 
18 | def robot_parser(url):
19 |     bot_loc = get_bot_loc(url)
20 |     parser = Robots.fetch(bot_loc)
21 |     validation = parser.allowed(url, '*')
22 |     return validation
23 | 
24 | 
25 | def fetch(url):
26 |     ua = UserAgent()
27 |     header = {'User-Agent':str(ua.random)}
28 |     validation = robot_parser(url)
29 |     if validation:
30 |         try:
31 |             response = requests.get(url, headers=header)
32 |         except requests.exceptions.ConnectionError:
33 |             print('Error: "%s" is not available!' % url)
34 |         content = BeautifulSoup(response.text, 'html.parser') 
35 |     else:
36 |         content = '%s is blocked by robots.txt' % url
37 |         response = 'Blocked'
38 |     return content, response
39 | 
40 | def write_to_file(data):
41 |     with open('data.json', 'w') as json_file:
42 |         json.dump(data,json_file,indent=2)
43 | 
44 | def parse(url):
45 |     r = fetch(url)
46 |     response = r[0]
47 |     status_code = r[1].status_code
48 | 
49 |     if status_code == 200:
50 |         if isinstance(response, BeautifulSoup): # If fetch returns HTML 
51 |             h1, h2, description, title, canonical, hreflang, robot_tag = ([] for i in range(7))
52 | 
53 |             for h in response.find_all('h1'):
54 |                 h1.append(h.text)
55 |             for h in response.find_all('h2'):
56 |                 h2.append(h.text)
57 |             for tag in response.find_all('meta', {'name': 'description'}):
58 |                 description.append(tag['content'])
59 |             for tag in response.find_all('title'):
60 |                 title.append(tag.text)
61 |             for tag in response.find_all('link', {'rel': 'canonical'}):
62 |                 canonical.append(tag['href'])
63 |             for tag in response.find_all('link', {'rel': 'alternate'}):
64 |                 hreflang.append(tag['href'])
65 |             for tag in response.find_all('meta', {'name': 'robots'}):
66 |                 robot_tag.append(tag['content'])
67 |             
68 |             data = {
69 |                 'url': url,
70 |                 'h1': h1, 
71 |                 'h2': h2,
72 |                 'description': description,
73 |                 'title': title,
74 |                 'canonical': canonical,
75 |                 'hreflang': hreflang,
76 |                 'robot_tag': robot_tag 
77 |             }
78 |             print(data)
79 | 
80 |             write_to_file(data)
81 |         else:
82 |             print('Could not fetch %s' % url)
83 |     else:
84 |         print('Could not fetch %s, Status code:' % url + str(response.status_code))
85 | 
86 | parse(url)
87 | 
88 | 
89 | 
90 | 
91 | 


--------------------------------------------------------------------------------
/knowledge graph api.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import urllib.parse
 3 | import requests
 4 | 
 5 | print('Knowledge Graph API')
 6 | api_key = open('api-key.txt').read()
 7 | while True:
 8 |     query = input('Enter query: ')
 9 |     querynum = input('How many results do you want? (Number): ')
10 |     queryDesc = input('Return Knowledge Graph description? (Y/N): ')
11 |     typequestion = input('Add entity filter? (Y/N): ')
12 |     if typequestion == 'Y':
13 |         print('Check Schema.org for list of entity "types": https://schema.org/docs/schemas.html')
14 |         kptype = input('What type of entity do you want? ')
15 | 
16 |     service_url = 'https://kgsearch.googleapis.com/v1/entities:search'
17 |     params = {
18 |         'query': str(query),
19 |         'limit': int(querynum),
20 |         'indent': True,
21 |         'key': api_key,
22 | }
23 | 
24 |     if typequestion == 'Y':
25 |         params['types'] = kptype
26 | 
27 |     url = f'{service_url}?{urllib.parse.urlencode(params)}'
28 |     print(url)
29 |     response = requests.get(url, verify=False)
30 |     json_response = json.loads(response.text)
31 | 
32 |     print(f'Query: {query}' + '\n')
33 | 
34 |     try:
35 |         for element in json_response['itemListElement']:
36 |             if queryDesc == 'Y':
37 |                 print(element['result']['name'] + ' (ID: ' + element['result']['@id'] + ')' + ' \
38 |                 (Score: ' + str(element['resultScore']) + ')' + 'Description: ' + element['result']['description'])
39 |             else:
40 |                 print(element['result']['name'] + ' (ID: ' + element['result']['@id'] + ')' + ' \
41 |                 (Score: ' + str(element['resultScore']) + ')')
42 |     except KeyError:
43 |         print('<Error: Name not found>')
44 | 
45 |     print('')
46 |     savefiles = input('Save files locally?(Y/N) ')
47 |     if savefiles == 'Y':
48 |         download_dir = 'knowledge-graph-results.csv'
49 |         file = open(download_dir, 'w')
50 |         columnTitleRow = "Query, Name, MID, Score, Description\n"
51 |         file.write(columnTitleRow)
52 |             
53 |         try:
54 |             for element in json_response['itemListElement']:
55 |                 if queryDesc == 'Y':
56 |                     q = query
57 |                     name = element['result']['name']
58 |                     ID = element['result']['@id']
59 |                     score = element['resultScore']
60 |                     description = element['result']['description']
61 |                     descrip2 = f'"{description}"'
62 | 
63 |                     row = f'{q},{name},{ID},{score},{descrip2}\n'
64 |                     file.write(row)
65 |                 else:
66 |                     q = query
67 |                     name = element['result']['name']
68 |                     ID = element['result']['@id']
69 |                     score = element['resultScore']
70 |                     description = element['result']['description']
71 |                     row2 = f'{q},{name},{ID},{score}\n'
72 |                     file.write(row2)
73 |         except KeyError:
74 |             print('')
75 |     if savefiles == 'Y':
76 |         print(f'Success. Results saved locally at {download_dir}.')
77 |         file.close()
78 |     choice = input('Continue Querying?(Y/N): ')
79 |     if choice == 'N':
80 |         break
81 | 
82 | 


--------------------------------------------------------------------------------
/Reddit/refreshtoken.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | """This example demonstrates the flow for retrieving a refresh token.
 4 | 
 5 | In order for this example to work your application's redirect URI must be set to
 6 | http://localhost:8080.
 7 | 
 8 | This tool can be used to conveniently create refresh tokens for later use with your web
 9 | application OAuth2 credentials.
10 | 
11 | """
12 | import random
13 | import socket
14 | import sys
15 | 
16 | import praw
17 | 
18 | 
19 | def receive_connection():
20 |     """Wait for and then return a connected socket..
21 | 
22 |     Opens a TCP connection on port 8080, and waits for a single client.
23 | 
24 |     """
25 |     server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
26 |     server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
27 |     server.bind(("localhost", 8080))
28 |     server.listen(1)
29 |     client = server.accept()[0]
30 |     server.close()
31 |     return client
32 | 
33 | 
34 | def send_message(client, message):
35 |     """Send message to client and close the connection."""
36 |     print(message)
37 |     client.send(f"HTTP/1.1 200 OK\r\n\r\n{message}".encode("utf-8"))
38 |     client.close()
39 | 
40 | 
41 | def main():
42 |     """Provide the program's entry point when directly executed."""
43 |     print(
44 |         "Go here while logged into the account you want to create a token for: "
45 |         "https://www.reddit.com/prefs/apps/"
46 |     )
47 |     print(
48 |         "Click the create an app button. Put something in the name field and select the"
49 |         " script radio button."
50 |     )
51 |     print("Put http://localhost:8080 in the redirect uri field and click create app")
52 |     client_id = input(
53 |         "Enter the client ID, it's the line just under Personal use script at the top: "
54 |     )
55 |     client_secret = input("Enter the client secret, it's the line next to secret: ")
56 |     commaScopes = input(
57 |         "Now enter a comma separated list of scopes, or all for all tokens: "
58 |     )
59 | 
60 |     if commaScopes.lower() == "all":
61 |         scopes = ["*"]
62 |     else:
63 |         scopes = commaScopes.strip().split(",")
64 | 
65 |     reddit = praw.Reddit(
66 |         client_id=client_id.strip(),
67 |         client_secret=client_secret.strip(),
68 |         redirect_uri="http://localhost:8080",
69 |         user_agent="praw_refresh_token_example",
70 |     )
71 |     state = str(random.randint(0, 65000))
72 |     url = reddit.auth.url(scopes, state, "permanent")
73 |     print(f"Now open this url in your browser: {url}")
74 |     sys.stdout.flush()
75 | 
76 |     client = receive_connection()
77 |     data = client.recv(1024).decode("utf-8")
78 |     param_tokens = data.split(" ", 2)[1].split("?", 1)[1].split("&")
79 |     params = {
80 |         key: value for (key, value) in [token.split("=") for token in param_tokens]
81 |     }
82 | 
83 |     if state != params["state"]:
84 |         send_message(
85 |             client,
86 |             f"State mismatch. Expected: {state} Received: {params['state']}",
87 |         )
88 |         return 1
89 |     elif "error" in params:
90 |         send_message(client, params["error"])
91 |         return 1
92 | 
93 |     refresh_token = reddit.auth.authorize(params["code"])
94 |     send_message(client, f"Refresh token: {refresh_token}")
95 |     return 0
96 | 
97 | 
98 | if __name__ == "__main__":
99 |     sys.exit(main())


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # VSCode
  2 | .vscode/*
  3 | !.vscode/settings.json
  4 | !.vscode/tasks.json
  5 | !.vscode/launch.json
  6 | !.vscode/extensions.json
  7 | *.code-workspace
  8 | # Local History for Visual Studio Code
  9 | .history/
 10 | 
 11 | # Common credential files
 12 | **/credentials.json
 13 | **/client_secrets.json
 14 | **/client_secret.json
 15 | *creds*
 16 | *.dat
 17 | *password*
 18 | *.httr-oauth*
 19 | 
 20 | # Private Node Modules
 21 | node_modules/
 22 | creds.js
 23 | 
 24 | # Private Files
 25 | *.json
 26 | *.csv
 27 | *.csv.gz
 28 | *.tsv
 29 | *.tsv.gz
 30 | *.xlsx
 31 | 
 32 | 
 33 | # Mac/OSX
 34 | .DS_Store
 35 | 
 36 | 
 37 | # Byte-compiled / optimized / DLL files
 38 | __pycache__/
 39 | *.py[cod]
 40 | *$py.class
 41 | 
 42 | # C extensions
 43 | *.so
 44 | 
 45 | # Distribution / packaging
 46 | .Python
 47 | build/
 48 | develop-eggs/
 49 | dist/
 50 | downloads/
 51 | eggs/
 52 | .eggs/
 53 | lib/
 54 | lib64/
 55 | parts/
 56 | sdist/
 57 | var/
 58 | wheels/
 59 | share/python-wheels/
 60 | *.egg-info/
 61 | .installed.cfg
 62 | *.egg
 63 | MANIFEST
 64 | 
 65 | # PyInstaller
 66 | #  Usually these files are written by a python script from a template
 67 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 68 | *.manifest
 69 | *.spec
 70 | 
 71 | # Installer logs
 72 | pip-log.txt
 73 | pip-delete-this-directory.txt
 74 | 
 75 | # Unit test / coverage reports
 76 | htmlcov/
 77 | .tox/
 78 | .nox/
 79 | .coverage
 80 | .coverage.*
 81 | .cache
 82 | nosetests.xml
 83 | coverage.xml
 84 | *.cover
 85 | *.py,cover
 86 | .hypothesis/
 87 | .pytest_cache/
 88 | cover/
 89 | 
 90 | # Translations
 91 | *.mo
 92 | *.pot
 93 | 
 94 | # Django stuff:
 95 | *.log
 96 | local_settings.py
 97 | db.sqlite3
 98 | db.sqlite3-journal
 99 | 
100 | # Flask stuff:
101 | instance/
102 | .webassets-cache
103 | 
104 | # Scrapy stuff:
105 | .scrapy
106 | 
107 | # Sphinx documentation
108 | docs/_build/
109 | 
110 | # PyBuilder
111 | .pybuilder/
112 | target/
113 | 
114 | # Jupyter Notebook
115 | .ipynb_checkpoints
116 | 
117 | # IPython
118 | profile_default/
119 | ipython_config.py
120 | 
121 | # pyenv
122 | #   For a library or package, you might want to ignore these files since the code is
123 | #   intended to run in multiple environments; otherwise, check them in:
124 | # .python-version
125 | 
126 | # pipenv
127 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
128 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
129 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
130 | #   install all needed dependencies.
131 | #Pipfile.lock
132 | 
133 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
134 | __pypackages__/
135 | 
136 | # Celery stuff
137 | celerybeat-schedule
138 | celerybeat.pid
139 | 
140 | # SageMath parsed files
141 | *.sage.py
142 | 
143 | # Environments
144 | .env
145 | .venv
146 | env/
147 | venv/
148 | ENV/
149 | env.bak/
150 | venv.bak/
151 | 
152 | # Spyder project settings
153 | .spyderproject
154 | .spyproject
155 | 
156 | # Rope project settings
157 | .ropeproject
158 | 
159 | # mkdocs documentation
160 | /site
161 | 
162 | # mypy
163 | .mypy_cache/
164 | .dmypy.json
165 | dmypy.json
166 | 
167 | # Pyre type checker
168 | .pyre/
169 | 
170 | # pytype static type analyzer
171 | .pytype/
172 | 
173 | # Cython debug symbols
174 | cython_debug/
175 | 


--------------------------------------------------------------------------------
/.gsource:
--------------------------------------------------------------------------------
  1 | # VSCode
  2 | .vscode/*
  3 | !.vscode/settings.json
  4 | !.vscode/tasks.json
  5 | !.vscode/launch.json
  6 | !.vscode/extensions.json
  7 | *.code-workspace
  8 | # Local History for Visual Studio Code
  9 | .history/
 10 | 
 11 | # Common credential files
 12 | **/credentials.json
 13 | **/client_secrets.json
 14 | **/client_secret.json
 15 | *creds*
 16 | *.dat
 17 | *password*
 18 | *.httr-oauth*
 19 | 
 20 | # Private Node Modules
 21 | node_modules/
 22 | creds.js
 23 | 
 24 | # Private Files
 25 | *.json
 26 | *.csv
 27 | *.csv.gz
 28 | *.tsv
 29 | *.tsv.gz
 30 | *.xlsx
 31 | 
 32 | 
 33 | # Mac/OSX
 34 | .DS_Store
 35 | 
 36 | 
 37 | # Byte-compiled / optimized / DLL files
 38 | __pycache__/
 39 | *.py[cod]
 40 | *$py.class
 41 | 
 42 | # C extensions
 43 | *.so
 44 | 
 45 | # Distribution / packaging
 46 | .Python
 47 | build/
 48 | develop-eggs/
 49 | dist/
 50 | downloads/
 51 | eggs/
 52 | .eggs/
 53 | lib/
 54 | lib64/
 55 | parts/
 56 | sdist/
 57 | var/
 58 | wheels/
 59 | share/python-wheels/
 60 | *.egg-info/
 61 | .installed.cfg
 62 | *.egg
 63 | MANIFEST
 64 | 
 65 | # PyInstaller
 66 | #  Usually these files are written by a python script from a template
 67 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 68 | *.manifest
 69 | *.spec
 70 | 
 71 | # Installer logs
 72 | pip-log.txt
 73 | pip-delete-this-directory.txt
 74 | 
 75 | # Unit test / coverage reports
 76 | htmlcov/
 77 | .tox/
 78 | .nox/
 79 | .coverage
 80 | .coverage.*
 81 | .cache
 82 | nosetests.xml
 83 | coverage.xml
 84 | *.cover
 85 | *.py,cover
 86 | .hypothesis/
 87 | .pytest_cache/
 88 | cover/
 89 | 
 90 | # Translations
 91 | *.mo
 92 | *.pot
 93 | 
 94 | # Django stuff:
 95 | *.log
 96 | local_settings.py
 97 | db.sqlite3
 98 | db.sqlite3-journal
 99 | 
100 | # Flask stuff:
101 | instance/
102 | .webassets-cache
103 | 
104 | # Scrapy stuff:
105 | .scrapy
106 | 
107 | # Sphinx documentation
108 | docs/_build/
109 | 
110 | # PyBuilder
111 | .pybuilder/
112 | target/
113 | 
114 | # Jupyter Notebook
115 | .ipynb_checkpoints
116 | 
117 | # IPython
118 | profile_default/
119 | ipython_config.py
120 | 
121 | # pyenv
122 | #   For a library or package, you might want to ignore these files since the code is
123 | #   intended to run in multiple environments; otherwise, check them in:
124 | # .python-version
125 | 
126 | # pipenv
127 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
128 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
129 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
130 | #   install all needed dependencies.
131 | #Pipfile.lock
132 | 
133 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
134 | __pypackages__/
135 | 
136 | # Celery stuff
137 | celerybeat-schedule
138 | celerybeat.pid
139 | 
140 | # SageMath parsed files
141 | *.sage.py
142 | 
143 | # Environments
144 | .env
145 | .venv
146 | env/
147 | venv/
148 | ENV/
149 | env.bak/
150 | venv.bak/
151 | 
152 | # Spyder project settings
153 | .spyderproject
154 | .spyproject
155 | 
156 | # Rope project settings
157 | .ropeproject
158 | 
159 | # mkdocs documentation
160 | /site
161 | 
162 | # mypy
163 | .mypy_cache/
164 | .dmypy.json
165 | dmypy.json
166 | 
167 | # Pyre type checker
168 | .pyre/
169 | 
170 | # pytype static type analyzer
171 | .pytype/
172 | 
173 | # Cython debug symbols
174 | cython_debug/
175 | 


--------------------------------------------------------------------------------
/Pagespeed/pagespeed-api-by-daniel.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | '''
 3 | @author:    Daniel Heredia Mejias 
 4 | @website:   danielherediamejias.com
 5 | 
 6 | Published on:
 7 | https://www.jcchouinard.com/pagespeed-api-and-lighthouse-forecasting/
 8 | '''
 9 | import time
10 | 
11 | from selenium import webdriver
12 | import urllib.request, json
13 | 
14 | device = "mobile"
15 | page = "https://www.danielherediamejias.com/"
16 | 
17 | url = "https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=" + page + "&strategy=" + device + "&locale=en"
18 | 
19 | try:
20 |     response = urllib.request.urlopen(url)
21 | except Exception as e:
22 |     print(e)
23 | 
24 | data = json.loads(response.read()) 
25 | 
26 | 
27 | fcp_time = data["lighthouseResult"]["audits"]["first-contentful-paint"]["displayValue"]
28 | speed_index = data["lighthouseResult"]["audits"]["speed-index"]["displayValue"]
29 | lcp = data["lighthouseResult"]["audits"]["largest-contentful-paint"]["displayValue"]
30 | time_interactive = data["lighthouseResult"]["audits"]["interactive"]["displayValue"]
31 | blocking_time_duration = data["lighthouseResult"]["audits"]["total-blocking-time"]["displayValue"]
32 | cls = data["lighthouseResult"]["audits"]["cumulative-layout-shift"]["displayValue"]
33 | overall_score = int(data["lighthouseResult"]["categories"]["performance"]["score"] * 100)
34 | 
35 | 
36 | fcp_time = ["First Contentful Time" ,int(float(fcp_time.replace("\xa0s", ""))*1000)]
37 | speed_index = ["Speed Index", int(float(speed_index.replace("\xa0s", ""))*1000)]
38 | time_interactive = ["Time to interactive", int(float(time_interactive.replace("\xa0s", ""))*1000)]
39 | blocking_time_duration = ["Total Blocking Time", int(blocking_time_duration.replace("\xa0ms", ""))]
40 | lcp = ["Large Contentful Paint", int(float(lcp.replace("\xa0s", ""))*1000)]
41 | cls = ["Cumulative Layout Shift", round(float(cls), 2)]
42 | 
43 | list = [fcp_time, speed_index, time_interactive, blocking_time_duration, lcp, cls]
44 | 
45 | driver=webdriver.Chrome() 
46 | driver.get('https://googlechrome.github.io/lighthouse/scorecalc/#FCP=' + str(list[0][1]) + '&SI=' + str(list[1][1]) + '&LCP=' + str(list[4][1]) + '&TTI=' + str(list[2][1]) + '&TBT=' + str(list[3][1]) + '&CLS=' + str(list[5][1]) + '&device=” +device + “&version=6')
47 | time.sleep(3)
48 | element = driver.find_element_by_class_name("lh-gauge__percentage")
49 | finalscore = int(element.text)
50 | 
51 | driver.quit()
52 | 
53 | differenceslist = []
54 | numLoops = len(list)
55 | 
56 | print('Adding a 50% increase to each metrics to check score improvement\n')
57 | print(f'This loop will fetch the Lighthouse {numLoops} times\n')
58 | for x in range(numLoops):
59 |     print(f'Testing improvement on {list[x][0]}')
60 |     oldnumber = list[x][1]
61 |     list[x][1] = list[x][1] - list[x][1] * 0.5 
62 |     driver = webdriver.Chrome() 
63 |     driver.get('https://googlechrome.github.io/lighthouse/scorecalc/#FCP='  + str(list[0][1]) + '&SI=' + str(list[1][1]) + '&LCP=' + str(list[4][1]) + '&TTI=' + str(list[2][1]) + '&TBT=' + str(list[3][1]) + '&CLS=' + str(list[5][1]) + '&device=' + device + '&version=6')
64 |     time.sleep(20)
65 |     element = driver.find_element_by_class_name("lh-gauge__percentage")
66 |     finalscore = int(element.text)
67 |     driver.quit()   
68 |     
69 |     differenceslist.append(finalscore - overall_score)
70 |     list[x][1] = oldnumber
71 | 
72 | print("\nWoking on " + str(list[differenceslist.index(max(differenceslist))][0]) + " would improve performance " + str(max(differenceslist)) + " points." )
73 | for x in range (len(differenceslist)):
74 |     print("If you improve 50% of " + str(list[x][0]) + " you will improve " + str(differenceslist[x]) + " points your overall score")
75 | 


--------------------------------------------------------------------------------
/Google-Search-Console/Connect search console using OAuth keys.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Connect to Google Search Console API using Python\n",
  8 |     "This notebook show you how to **Connect to Google Search Console API using Python** .\n",
  9 |     "\n",
 10 |     "@author: Jean-Christophe Chouinard: Technical SEO / Data Scientist.",
 11 |     "\n",
 12 |     "[LinkedIn](https://www.linkedin.com/in/jeanchristophechouinard/), [Twitter.com/@ChouinardJC](https://twitter.com/ChouinardJC), Blog > [jcchouinard.com](https://www.jcchouinard.com/)",
 13 |     "\n",
 14 |     "From the post: [How to Use Google Search Console API With Python](https://www.jcchouinard.com/how-to-connect-to-google-search-console-api-using-python/)",
 15 |     "\n",
 16 |     "Before you can run this code, make sure that you know how to get your [Google Search Console API Keys](https://www.jcchouinard.com/how-to-get-google-search-console-api-keys/)"
 17 |    ]
 18 |   },
 19 |   {
 20 |    "cell_type": "code",
 21 |    "execution_count": 1,
 22 |    "metadata": {},
 23 |    "outputs": [],
 24 |    "source": [
 25 |     "import argparse\n",
 26 |     "from apiclient.discovery import build\n",
 27 |     "import httplib2\n",
 28 |     "from oauth2client import client\n",
 29 |     "from oauth2client import file\n",
 30 |     "from oauth2client import tools\n",
 31 |     "\n",
 32 |     "SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly']\n",
 33 |     "DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/customsearch/v1/rest')"
 34 |    ]
 35 |   },
 36 |   {
 37 |    "cell_type": "code",
 38 |    "execution_count": 2,
 39 |    "metadata": {},
 40 |    "outputs": [],
 41 |    "source": [
 42 |     "parser = argparse.ArgumentParser(\n",
 43 |     "    formatter_class=argparse.RawDescriptionHelpFormatter,\n",
 44 |     "    parents=[tools.argparser])\n",
 45 |     "flags = parser.parse_args([])"
 46 |    ]
 47 |   },
 48 |   {
 49 |    "cell_type": "code",
 50 |    "execution_count": 3,
 51 |    "metadata": {},
 52 |    "outputs": [],
 53 |    "source": [
 54 |     "flow = client.OAuth2WebServerFlow(client_id='XXXXXXXXXXXXXXX.apps.googleusercontent.com',\n",
 55 |     "                           client_secret='XXXXXXXXXXXXXXXXXXXXX',\n",
 56 |     "                           scope=SCOPES)"
 57 |    ]
 58 |   },
 59 |   {
 60 |    "cell_type": "code",
 61 |    "execution_count": 6,
 62 |    "metadata": {},
 63 |    "outputs": [],
 64 |    "source": [
 65 |     "storage = file.Storage(r'C:\\Users\\YOUR-PATH\\searchconsole.dat')\n",
 66 |     "credentials = storage.get()\n",
 67 |     "if credentials is None or credentials.invalid:\n",
 68 |     "  credentials = tools.run_flow(flow, storage, flags)\n",
 69 |     "http = credentials.authorize(http=httplib2.Http())"
 70 |    ]
 71 |   },
 72 |   {
 73 |    "cell_type": "code",
 74 |    "execution_count": 7,
 75 |    "metadata": {},
 76 |    "outputs": [],
 77 |    "source": [
 78 |     "webmasters_service = build('webmasters', 'v3', http=http)"
 79 |    ]
 80 |   },
 81 |   {
 82 |    "cell_type": "code",
 83 |    "execution_count": 8,
 84 |    "metadata": {},
 85 |     "source": [
 86 |     "site_list = webmasters_service.sites().list().execute()\n",
 87 |     "\n",
 88 |     "verified_sites_urls = [s['siteUrl'] for s in site_list['siteEntry']\n",
 89 |     "                       if s['permissionLevel'] != 'siteUnverifiedUser'\n",
 90 |     "                          and s['siteUrl'][:4] == 'http']\n",
 91 |     "\n",
 92 |     "for site_url in verified_sites_urls:\n",
 93 |     "  print( site_url)"
 94 |    ]
 95 |   },
 96 |   {
 97 |    "cell_type": "code",
 98 |    "execution_count": null,
 99 |    "metadata": {},
100 |    "outputs": [],
101 |    "source": []
102 |   }
103 |  ],
104 |  "metadata": {
105 |   "kernelspec": {
106 |    "display_name": "Python 3",
107 |    "language": "python",
108 |    "name": "python3"
109 |   },
110 |   "language_info": {
111 |    "codemirror_mode": {
112 |     "name": "ipython",
113 |     "version": 3
114 |    },
115 |    "file_extension": ".py",
116 |    "mimetype": "text/x-python",
117 |    "name": "python",
118 |    "nbconvert_exporter": "python",
119 |    "pygments_lexer": "ipython3",
120 |    "version": "3.7.3"
121 |   }
122 |  },
123 |  "nbformat": 4,
124 |  "nbformat_minor": 2
125 | }
126 | 


--------------------------------------------------------------------------------
/robots_txt/robotstxt_smoketext.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from datetime import datetime
  3 | import json
  4 | import os
  5 | import requests
  6 | 
  7 | from urllib.parse import urlparse
  8 | 
  9 | site = 'https://www.jcchouinard.com/'
 10 | 
 11 | def main(credentials,site,filename='robots.txt'):
 12 |     '''
 13 |     Combine all functions
 14 |     '''
 15 |     url = site + filename
 16 |     domain = get_domain_name(site)
 17 |     output = path + '/output/' + domain + '/'
 18 |     create_project(output)
 19 |     r = fetch_page(url)
 20 |     output_files = get_robots_files(output)
 21 |     compare_robots(credentials,output,output_files,r)
 22 | 
 23 | def get_credentials(credentials):
 24 |     '''
 25 |     Read credentials from JSON file.
 26 |     '''
 27 |     with open(credentials, 'r') as f:
 28 |         creds = json.load(f)
 29 |     return creds['slack_webhook']
 30 | 
 31 | def post_to_slack(message,credentials):
 32 |     '''
 33 |     Create header with the message and post to slack.
 34 |     '''
 35 |     data = {'text':message}
 36 |     url = get_credentials(credentials)
 37 |     requests.post(url,json=data, verify=False)
 38 | 
 39 | def create_project(directory):
 40 |     '''
 41 |     Create a project if it does not exist
 42 |     '''
 43 |     if not os.path.exists(directory):
 44 |         print('Create project: '+ directory)
 45 |         os.makedirs(directory)
 46 |     else:
 47 |         print(f'{directory} project exists')
 48 | 
 49 | def get_domain_name(start_url):
 50 |     '''
 51 |     Get Domain Name in the www_domain_com format
 52 |     1. Parse URL
 53 |     2. Get Domain
 54 |     3. Replace dots to make a usable folder path
 55 |     '''
 56 |     url = urlparse(start_url)               # Parse URL into components
 57 |     domain_name = url.netloc                # Get Domain (or network location)
 58 |     domain_name = domain_name.replace('.','_')# Replace . by _ to create usable folder
 59 |     domain_name = domain_name.split(':')[0]
 60 |     return domain_name
 61 | 
 62 | def get_date():
 63 |     '''
 64 |     Get today's date as YYYY-MM-DD
 65 |     '''
 66 |     return datetime.today().strftime('%Y-%m-%d:%H:%M%S')
 67 | 
 68 | def fetch_page(url):
 69 |     '''
 70 |     Use requests to fetch URL
 71 |     '''
 72 |     try:
 73 |         r = requests.get(url)
 74 |     except Exception as e:
 75 |         print(e)
 76 |     return r
 77 | 
 78 | def get_robots_files(directory):
 79 |     '''
 80 |     List robots.txt file in output folder
 81 |     '''
 82 |     output_files = os.listdir(directory)
 83 |     output_files.sort()
 84 |     return output_files
 85 | 
 86 | def get_robotstxt_name(output_files):
 87 |     '''
 88 |     Get last saved robots.txt file.
 89 |     '''
 90 |     robots_txt = 'robots.txt'
 91 |     files = []
 92 |     for filename in output_files:
 93 |         if robots_txt in filename:
 94 |             files.append(filename)
 95 |     return files[-1]
 96 | 
 97 | def read_robotstxt(filename):
 98 |     '''
 99 |     Read robots.txt
100 |     '''
101 |     if os.path.isfile(filename):
102 |         with open(filename,'r') as f:
103 |             txt = f.read()
104 |     return txt
105 | 
106 | def write_robotstxt(file, output, filename='robots.txt'):
107 |     '''
108 |     Write robots.txt to file using date as identifier.
109 |     '''
110 |     filename = get_date() + '-' + filename
111 |     filename = output + filename
112 |     print(f'filename: {filename}')
113 |     with open(filename,'w') as f:
114 |         f.write(file)
115 | 
116 | def compare_robots(credentials,output,output_files,r):
117 |     '''
118 |     Compare previous robots to actual robots.
119 |     If different. Save File.
120 |     '''
121 |     new_robotstxt = r.text.replace('\r', '')
122 |     if output_files:
123 |         robots_filename = get_robotstxt_name(output_files) 
124 |         filename = output + robots_filename
125 |         last_robotstxt = read_robotstxt(filename)
126 |         last_robotstxt = last_robotstxt.replace('\r', '')
127 |         if new_robotstxt != last_robotstxt:
128 |             print('Robots.txt was modified')
129 |             write_robotstxt(new_robotstxt, output)
130 |             message = f'The Robots.txt was changed for {filename}'
131 |             post_to_slack(message,credentials)
132 |         else:
133 |             print('No Change to Robots.txt')
134 |     else:
135 |         print('No Existing Robots.txt. Saving one.')
136 |         write_robotstxt(r.text, output)
137 | 
138 | if __name__ == '__main__':
139 |     path = os.getcwd()
140 |     credentials = path + '/creds.json'
141 |     main(credentials,site)
142 | 


--------------------------------------------------------------------------------
/GSC_Tutorial/file_manip.py:
--------------------------------------------------------------------------------
  1 | import json
  2 | import glob
  3 | import os
  4 | import pandas as pd
  5 | 
  6 | from urllib.parse import urlparse
  7 | 
  8 | from date_manip import get_dates
  9 | 
 10 | # Create a project if it does not exist
 11 | def create_project(directory):
 12 |     if not os.path.exists(directory):
 13 |         print('Create project: '+ directory)
 14 |         os.makedirs(directory)
 15 |     else:
 16 |         print(f'{directory} project exists')
 17 | 
 18 | # Get Domain Name to Create a Project
 19 | def get_domain_name(start_url):
 20 |     domain_name = '{uri.netloc}'.format(uri=urlparse(start_url))  # Get Domain Name To Name Project
 21 |     domain_name = domain_name.replace('.','_')
 22 |     return domain_name
 23 | 
 24 | # Write to a JSON file
 25 | def write_to_json(data,filename):
 26 |     with open(filename, 'w') as f:
 27 |         json.dump(data, f, indent=4)
 28 | 
 29 | # Write to a CSV file
 30 | def write_to_csv(data,filename):
 31 |     if not os.path.isfile(filename):
 32 |         data.to_csv(filename, index=False)
 33 |     else: # else it exists so append without writing the header
 34 |         data.to_csv(filename, mode='a', header=False, index=False)
 35 | 
 36 | # Convert date column to a datetime Index
 37 | def date_to_index(df,datecol):
 38 |     if df.index.name == datecol:
 39 |         if isinstance(df.index, pd.DatetimeIndex):
 40 |             print(f'{datecol} is already a datetime index')
 41 |         else:
 42 |             df[datecol] = pd.to_datetime(df[datecol])
 43 |     else:
 44 |         df[datecol] = pd.to_datetime(df[datecol])
 45 |         df = df.set_index(datecol)
 46 |     return df
 47 | 
 48 | # Overwrite CSV without duplicates
 49 | def drop_duplicates(filename):
 50 |     f = pd.read_csv(filename).drop_duplicates(keep=False)
 51 |     f.to_csv(filename)
 52 | 
 53 | # Define where to export CSVs
 54 | def get_full_path(site,filename,date):
 55 |     YM_date = get_dates(date)[0]    # Get date as YYYY-MM
 56 |     output_path = YM_date + '_' + filename  # Add file location
 57 |     output = os.path.join(output_path)      # Merge to create YYYY-MM_filename.csv
 58 |     domain_name = get_domain_name(site)  # Get domain from site URL
 59 |     data_path = domain_name + '/' 
 60 |     full_path = data_path + output          # Output will be at /your_site_com/YYYY-MM_filename.csv
 61 |     return output, domain_name, full_path, data_path
 62 | 
 63 | # Check all files that ends with your filename to gather dates from
 64 | def loop_csv(full_path,filename):
 65 |     file_list = []
 66 |     for file in os.listdir(full_path):
 67 |         if file.endswith(os.path.join('_'+ filename)):
 68 |             file_list.append(file)
 69 |             file_list.sort()
 70 |     return file_list
 71 | 
 72 | # Read CSV if it exists to find dates that have already been processed.
 73 | def get_dates_from_csv(path):
 74 |     if os.path.isfile(path):
 75 |         data = pd.read_csv(path)
 76 |         data = pd.Series(data['date'].unique())
 77 |         return data
 78 |     else:
 79 |         pass
 80 | 
 81 | # Check All CSVs to check all dates that have been processed
 82 | def get_dates_csvs(full_path,site,filename):
 83 |     print(f'Checking existing dates in {full_path}')
 84 |     dset = set()
 85 |     csvs = loop_csv(full_path,filename)
 86 |     for csv in csvs:
 87 |         path = os.path.join(full_path + csv)
 88 |         dates = get_dates_from_csv(path)
 89 |         for date in dates:
 90 |             dset.add(date)
 91 |     return dset
 92 | 
 93 | # Read all dataframes in path
 94 | def read_all_dfs(path,filename):
 95 |     dfs = []
 96 |     files =[]
 97 |     globs = glob.glob(f'{path}/*{filename}')
 98 |     for g in globs:
 99 |         files.append(g)
100 |     for f in files:
101 |         print(f)
102 |         df = pd.read_csv(f)
103 |         dfs.append(df)
104 |     full_df = pd.concat(dfs)
105 |     return full_df
106 | 
107 | # Convert dict to DF
108 | def scdict_to_df(scDict,clicks=True,impressions=True,ctr=False,position=False):
109 |     df = pd.DataFrame(data = scDict)
110 |     if clicks:
111 |         df['clicks'] = df['clicks'].astype('int')
112 |     if impressions:
113 |         df['impressions'] = df['impressions'].astype('int')
114 |     if ctr:
115 |         df['ctr'] = df['ctr']*100
116 |     if position:
117 |         df['position'] = df['position'].round(2)
118 |     df.sort_values('clicks',inplace=True,ascending=False)
119 |     return df
120 | 
121 | # Read All csvs to a combined dataframe
122 | def return_df(site,filename):
123 |     folder = get_domain_name(site)
124 |     df = read_all_dfs(folder,filename)  
125 |     return df


--------------------------------------------------------------------------------
/Reddit/prawbot.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | pip install praw
  3 | pip install feedparser
  4 | '''
  5 | import feedparser
  6 | import json
  7 | import praw
  8 | import random
  9 | import requests
 10 | import time 
 11 | 
 12 | from bs4 import BeautifulSoup
 13 | 
 14 | subr = 'pythonsandlot'
 15 | credentials = 'client_secrets.json'
 16 | 
 17 | with open(credentials) as f:
 18 |     creds = json.load(f)
 19 | 
 20 | 
 21 | # Option 1: PRAW stores your password
 22 | # reddit = praw.Reddit(client_id=creds['client_id'],
 23 | #                      client_secret=creds['client_secret'],
 24 | #                      user_agent=creds['user_agent'],
 25 | #                      username=creds['username'],
 26 | #                      password=creds['password'])
 27 | 
 28 | # Option 2: Use Refresh token (recommended)
 29 | reddit = praw.Reddit(client_id=creds['client_id'],
 30 |                      client_secret=creds['client_secret'],
 31 |                      user_agent=creds['user_agent'],
 32 |                      redirect_uri=creds['redirect_uri'],
 33 |                      refresh_token=creds['refresh_token'])
 34 | 
 35 | print(reddit.user.me())
 36 | 
 37 | subreddit = reddit.subreddit(subr)
 38 | requirements = subreddit.post_requirements()['is_flair_required']
 39 | 
 40 | subs = []
 41 | for sub in subreddit.hot(limit=500):
 42 |     print(f'sub.title: {sub.title}')
 43 |     print(f'sub.score: {sub.score}') 
 44 |     print(f'sub.selftext: {sub.selftext}')
 45 |     print(f'sub.url: {sub.url}')
 46 |     print(f'sub.view_count: {sub.view_count}')
 47 |     #print(f'sub.upvote: {sub.upvote}')
 48 |     print(f'sub.upvote_ratio: {sub.upvote_ratio}')
 49 |     print(f'sub.num_comments: {sub.num_comments}')
 50 |     print(f'sub.subreddit: {sub.subreddit}')
 51 |     print(f'sub.subreddit_id: {sub.subreddit_id}')
 52 |     print(f'sub.subreddit_name_prefixed: {sub.subreddit_name_prefixed}')
 53 |     print(f'sub.subreddit_subscribers: {sub.subreddit_subscribers}')
 54 | 
 55 | 
 56 | 
 57 | #search subreddit for keywords
 58 | subreddit = reddit.subreddit('all')
 59 | 
 60 | 
 61 | subs = []
 62 | for sub in subreddit.search('python+seo',time_filter='month'):
 63 |     subs.append({'id':sub.id,'score':sub.score,'title':sub.title,'url':sub.url,'subreddit_name_prefixed':sub.subreddit_name_prefixed,'num_comments':sub.num_comments})
 64 | 
 65 | subs = sorted(subs, key = lambda x: x['score'], reverse=True)
 66 | subs[:10]
 67 | sorted(subs.items(), key=lambda x: x['score'], reverse=True)
 68 | 
 69 | 
 70 | sub_ids = []
 71 | for submission in subreddit.hot(limit = 5): # Define the limit here and filter method
 72 |     sub_ids.append(submission.id)
 73 | 
 74 | top_level_comments = []
 75 | second_level_comments = []
 76 | title = []
 77 | selftext = []
 78 | for sub_id in sub_ids:
 79 |     submission = reddit.submission(id = sub_id)
 80 |     title.append(submission.title) # Get submission title
 81 |     selftext.append(submission.selftext) # Get submission content
 82 |     submission.comments.replace_more(limit = None)
 83 |     for top_level_comment in submission.comments:
 84 |         top_level_comments.append(top_level_comment.body) # Get top-level comments
 85 |         for second_level_comment in top_level_comment.replies:
 86 |             second_level_comments.append(second_level_comment.body) # Ge
 87 | 
 88 | for submission in reddit.subreddit("redditdev+learnpython").top("all"):
 89 |     print(submission)
 90 | 
 91 | for submission in reddit.subreddit("all-redditdev").new():
 92 |     print(submission)
 93 | 
 94 | 
 95 | 
 96 | 
 97 | ##########
 98 | 
 99 | # Parse feed to get post
100 | xml = feedparser.parse('https://www.jcchouinard.com/feed/')
101 | entry = xml.entries
102 | 
103 | posts = {}
104 | for i in range(len(entry)):
105 |     posts[entry[i].link] = {'title':entry[i].title,'description':''}
106 | 
107 | for k in posts.keys():
108 |     try:
109 |         r = requests.get(k)
110 |     except Exception as e:
111 |         print(e)
112 | 
113 |     soup = BeautifulSoup(r.text, 'html.parser')
114 |     metatags = soup.find_all('meta', {'name': 'description'})
115 |     posts[k]['description'] = metatags[0]['content']
116 | 
117 | for k,v in posts.items():
118 |     appender = [
119 |         'Here is the link to the post ',
120 |         'You can read the article here ',
121 |         'Read the story on '
122 |     ]
123 |     text = f'. {appender[random.randint(0,2)]} {k}.'
124 |     selftext = v['description'] + text
125 |     title = v['title']
126 |     reddit.subreddit(subr).submit(title,selftext=selftext)
127 |     time.sleep(3)
128 | 
129 | 
130 | # Extract GSC for top keywords
131 | # Search Reddit for Rising Sub containing each keyword
132 | # Send alert


--------------------------------------------------------------------------------
/GSC_Tutorial/date_manip.py:
--------------------------------------------------------------------------------
  1 | import re 
  2 | import datetime
  3 | from dateutil import relativedelta
  4 | import pandas as pd
  5 | 
  6 | 
  7 | ### DATE CONVERSION FUNCTIONS ###
  8 | # Convert date to a string with format YYYY-MM-DD
  9 | def date_to_str(date):
 10 |     if isinstance(date, datetime.datetime) or isinstance(date, datetime.date):
 11 |         date = datetime.datetime.strftime(date,'%Y-%m-%d')
 12 |     return date
 13 | 
 14 | # Convert date to a string with format YYYY-MM-DD
 15 | def str_to_date(date):
 16 |     if isinstance(date, str):
 17 |         date = datetime.datetime.strptime(date,'%Y-%m-%d')
 18 |     return date
 19 | 
 20 | # Convert Date to YYYY-MM
 21 | def date_to_YM(date):
 22 |     dt = str_to_date(date)
 23 |     return datetime.datetime.strftime(dt,'%Y-%m')
 24 | 
 25 | # Convert YYYYWW to Start and End Date
 26 | def iso_to_dates(pos): # 1 for start_date, 0 for end_date
 27 |     return lambda x:datetime.datetime.strptime(str(x)+'-'+str(pos), "%G%V-%w").date()
 28 | 
 29 | 
 30 | ### FUNCTIONS TO LIST DATES ###
 31 | # List dates between two dates
 32 | def list_dates(startDate,endDate):
 33 |     start_date = str_to_date(startDate)
 34 |     end_date = str_to_date(endDate)
 35 |     delta = end_date - start_date
 36 |     days = []
 37 |     for i in range(delta.days + 1):
 38 |         day = start_date + datetime.timedelta(days=i)
 39 |         days.append(date_to_str(day))
 40 |     return days
 41 | 
 42 | # Compare two lists to find missing dates.
 43 | def get_missing_dates(previous_dt,new_dt):
 44 |     if not previous_dt:
 45 |         dates = new_dt
 46 |     elif not new_dt:
 47 |         dates = print('No new dates to extract')
 48 |     else:
 49 |         previous_dt = set(previous_dt)
 50 |         new_dt = set(new_dt)
 51 |         ms_new_dates = list(sorted(new_dt - previous_dt))
 52 |         ms_old_date = list(sorted(previous_dt - new_dt))
 53 |         dates = list(sorted(set(ms_new_dates + ms_old_date)))
 54 |     return ms_new_dates, ms_old_date, dates
 55 | 
 56 | # Get last day of a specified month
 57 | def last_day_of_month(date):
 58 |     next_month = str_to_date(date).replace(day=28) + datetime.timedelta(days=4)  # this will never fail
 59 |     return date_to_str(next_month - datetime.timedelta(days=next_month.day))
 60 | 
 61 | # Get first and last day of a specified month
 62 | def month_start_end_dates(month,year=datetime.date.today().year):
 63 |     month = month_to_int(month)
 64 |     startDate = f'{year}-{month}-01'
 65 |     last_day = last_day_of_month(startDate)
 66 |     endDate = f'{last_day}'
 67 |     return startDate, endDate
 68 | 
 69 | # Get days since the beginning of the month
 70 | # If dt is not defined, use current month, else use month of the specified date.
 71 | def get_dates(chosen_date):
 72 |     print(f'Fetching start and end dates from {chosen_date}')
 73 |     today = datetime.date.today()
 74 |     days = relativedelta.relativedelta(days=3) # GSC does not permit date earlier than 3 days
 75 |     end_date = today - days  
 76 |     if chosen_date is '': 
 77 |         delta = end_date - today.replace(day=1) # Get first day of the month
 78 |         start_date = end_date - delta # count difference between end date and first day of the month
 79 |     else:
 80 |         delta = end_date - datetime.datetime.strptime(chosen_date,'%Y-%m-%d').date() # Get first day of the month
 81 |         start_date = end_date - delta # count difference between end date and first day of the month
 82 |     YM_date = date_to_YM(start_date)
 83 |     print(f'start_date: {start_date}, end_date: {end_date}')
 84 |     return YM_date, start_date, end_date
 85 | 
 86 | ### OTHER FUNCTIONS ###
 87 | # Convert Month to a two digits str
 88 | def month_to_int(month):
 89 |     month = str(month)
 90 |     if re.search('Jan(urary)?|(0)?1', month, re.IGNORECASE):
 91 |         month = '01'
 92 |     elif re.search('Feb(urary)?|(0)?2', month, re.IGNORECASE):
 93 |         month = '02'
 94 |     elif re.search('Mar(ch)?|(0)?3', month, re.IGNORECASE):
 95 |         month = '03'
 96 |     elif re.search('Apr(il)?|(0)?4', month, re.IGNORECASE):
 97 |         month = '04'
 98 |     elif re.search('M(ay)?|(0)?5', month, re.IGNORECASE):
 99 |         month = '05'
100 |     elif re.search('Jun(e)?|(0)?6', month, re.IGNORECASE):
101 |         month = '06'
102 |     elif re.search('Jul(y)?|(0)?7', month, re.IGNORECASE):
103 |         month = '07'
104 |     elif re.search('Aug(ust)?|(0)?8', month, re.IGNORECASE):
105 |         month = '08'
106 |     elif re.search('Sep(tember)?|(0)?9', month, re.IGNORECASE):
107 |         month = '09'
108 |     elif re.search('Oct(ober)?|10', month, re.IGNORECASE):
109 |         month = '10'
110 |     elif re.search('Nov(ember)?|11', month, re.IGNORECASE):
111 |         month = '11'
112 |     elif re.search('Dec(ember)?|12', month, re.IGNORECASE):
113 |         month = '12'
114 |     else: 
115 |         return print('Not a valid Month')
116 |     return month
117 | 


--------------------------------------------------------------------------------
/Google trends/gsc_to_trends.py:
--------------------------------------------------------------------------------
  1 | from datetime import datetime
  2 | from dateutil.relativedelta import relativedelta
  3 | import pandas as pd 
  4 | 
  5 | import plotly.express as px
  6 | import plotly.graph_objects as go
  7 | from plotly.subplots import make_subplots
  8 | from pytrends.request import TrendReq
  9 | import searchconsole
 10 | 
 11 | 
 12 | property_url = 'https://www.jcchouinard.com/'
 13 | geo = ''
 14 | # max 5 topics
 15 | # https://medium.com/analytics-vidhya/compare-more-than-5-keywords-in-google-trends-search-using-pytrends-3462d6b5ad62
 16 | topics = ['python seo']#['/m/019qb_']#
 17 | 
 18 | def authenticate(secret='client_secrets.json', token='credentials.json'):
 19 |     if os.path.isfile('credentials.json'):
 20 |         account = searchconsole.authenticate(client_config=secret,
 21 |                                             credentials=token)
 22 |     else:
 23 |         account = searchconsole.authenticate(client_config=secret,
 24 |                                         serialize=token)
 25 |     return account
 26 | 
 27 | 
 28 | def extract_gsc(property_url):
 29 |     print(f'Extracting GSC data: {property_url}')
 30 |     webproperty = account[property_url]
 31 |     report = webproperty.query\
 32 |             .range('today', months=-16)\
 33 |             .dimension('date')\
 34 |             .get()
 35 |     return report.to_dataframe()
 36 | 
 37 | 
 38 | def daily_to_weekly(df):
 39 |     # bring daily data to weekly format
 40 |     print('Converting dates to weekly format')
 41 |     df['date'] = pd.to_datetime(df['date']) - pd.to_timedelta(7, unit='d')
 42 |     df = df.groupby(pd.Grouper(key='date', freq='W-SUN'))['clicks','impressions'].sum()
 43 |     df = df.reset_index().sort_values('date')
 44 |     return df
 45 | 
 46 | 
 47 | def extract_google_trend(**kwargs):
 48 |     # Only need to run this once, 
 49 |     # other requests will use the same session.
 50 |     print('Extracting Google Trends data')
 51 |     pytrend = TrendReq()
 52 |     pytrend.build_payload(**kwargs)
 53 |     trend_data = pytrend.interest_over_time()
 54 |     trend_data['interest'] = trend_data[topics].sum(axis=1)
 55 |     return trend_data.reset_index()
 56 | 
 57 | 
 58 | def combine_gsc_to_gtrends(gsc_df, gtrends_df):
 59 |     # combine GSC data to google trend 
 60 |     print('Combining GSC data to Trends')
 61 |     g = pd.merge(gsc_df, gtrends_df, on='date')
 62 |     g.drop(g.tail(1).index, inplace=True)
 63 |     return g
 64 | 
 65 | 
 66 | def compute_seasonality(df, col):
 67 |     print('Computing seasonality')
 68 |     df['seasonnality_index'] = df[col] / df[col].mean()
 69 |     df['WoW_perc'] = df['seasonnality_index'].pct_change()
 70 |     df['WoW_perc'] = df['WoW_perc'].fillna(0)
 71 |     return df
 72 | 
 73 | 
 74 | 
 75 | account = authenticate()
 76 | 
 77 | df = extract_gsc(property_url)
 78 | df = daily_to_weekly(df)
 79 | 
 80 | end_date = datetime.today()
 81 | start_date = end_date - relativedelta(months=16) 
 82 | end_date = end_date.strftime('%Y-%m-%d')
 83 | start_date = start_date.strftime('%Y-%m-%d')
 84 | timeframe = f'{start_date} {end_date}'
 85 | if not geo: 
 86 |     trend_data = extract_google_trend(
 87 |         kw_list = topics,
 88 |         timeframe = timeframe
 89 |         )
 90 | else:
 91 |     trend_data = extract_google_trend(
 92 |         kw_list = topics,
 93 |         geo = geo,
 94 |         timeframe = timeframe
 95 |         )
 96 | 
 97 | g = combine_gsc_to_gtrends(df, trend_data)
 98 | 
 99 | g = compute_seasonality(g, 'interest')
100 | 
101 | # dropping first and las row as they are often and incomplete week
102 | # g = g.drop(0).reset_index(drop=True)
103 | # g.drop(g.tail(1).index,inplace=True)
104 | # make prediction
105 | g['prediction'] = g['clicks']
106 | for i in range(len(g)):
107 |     if i == 0:
108 |         g.loc[i,'prediction'] = g.loc[i,'clicks']
109 |     else:
110 |         prev_clicks = g.loc[i-1,'prediction'] 
111 |         perc_variation = g.loc[i,'seasonnality_index']
112 |         pred = prev_clicks * perc_variation
113 |         # print(i, g.loc[i,'date'], g.loc[i,'clicks'], g.loc[i,'interest'])
114 |         # print(prev_clicks, perc_variation, pred) 
115 |         g.loc[i,'prediction'] = pred
116 | 
117 | # plot the graph
118 | fig = make_subplots(specs=[[{"secondary_y": True}]])
119 | fig.add_trace(
120 |     go.Scatter(
121 |         x=g['date'], 
122 |         y=g['prediction'], 
123 |         name='Prediction'
124 |         ),
125 |         secondary_y=False
126 |     )
127 | fig.add_trace(
128 |     go.Scatter(
129 |         x=g['date'],
130 |         y=g['clicks'],
131 |         name='Clicks'),
132 |     secondary_y=False
133 |     )
134 | fig.add_trace(
135 |     go.Scatter(
136 |         x=g['date'],
137 |         y=g['interest'],
138 |         name='Trend'),
139 |     secondary_y=True
140 |     )
141 | fig.update_layout(title_text=f'Clicks Vs Prediction - {property_url}')
142 | fig.update_xaxes(title_text='Date')
143 | fig.update_yaxes(title_text="<b>Traffic:</b> Pred VS Actual", secondary_y=False)
144 | fig.update_yaxes(title_text="<b>Google Trends</b> data", secondary_y=True)
145 | 
146 | fig.show()
147 | 


--------------------------------------------------------------------------------
/Causal-impact/causal.py:
--------------------------------------------------------------------------------
  1 | from causalimpact import CausalImpact
  2 | from functools import reduce
  3 | import numpy as np
  4 | import pandas as pd 
  5 | import searchconsole
  6 | 
  7 | 
  8 | class Causal:
  9 | 
 10 | 
 11 |     def __init__(self, account, intervention, test_sites, control_sites='', months=-16, metric='clicks', dimension='date'):
 12 |         self.account = account
 13 |         self.test_sites = test_sites
 14 |         self.control_sites = control_sites if control_sites else None
 15 |         self.intervention = intervention
 16 |         self.metric = metric
 17 |         self.months = months
 18 |         self.dimension = dimension
 19 | 
 20 | 
 21 |     def run_causal(self):
 22 |         """Combines all the functions together
 23 | 
 24 |         Returns:
 25 |             [df]: dataframe on which CI was run
 26 |         """        
 27 |         data = self.create_master_df()
 28 |         pre_period, post_period = self.get_pre_post(data)
 29 |         self.make_report(data, pre_period, post_period)
 30 |         return data
 31 | 
 32 | 
 33 |     def extract_from_list(self, sites):
 34 |         """Extract GSC data from a list of sites
 35 | 
 36 |         Args:
 37 |             sites (list): list of properties validated in GSC
 38 | 
 39 |         Returns:
 40 |             [list]: List of dataframes extracted from GSC
 41 |         """        
 42 |         print(f'Extracting data for {sites}')
 43 |         dfs = []
 44 |         for site in sites:
 45 |             print(f'Extracting: {site}')
 46 |             webproperty = self.account[site]
 47 |             report = webproperty.query\
 48 |                     .range('today', months=self.months)\
 49 |                     .dimension(self.dimension)\
 50 |                     .get()
 51 |             df = report.to_dataframe()
 52 |             dfs.append(df)
 53 |         return dfs
 54 | 
 55 | 
 56 |     def concat_test(self, dfs):
 57 |         """Concatenate the dataframes used for testing
 58 | 
 59 |         Args:
 60 |             dfs (list): List of dataframes extracted from GSC
 61 | 
 62 |         Returns:
 63 |             dataframe: merged test dataframes summed together 
 64 |         """        
 65 |         concat_df = pd.concat(dfs)
 66 |         test = concat_df.groupby('date')[['clicks', 'impressions']].sum()
 67 |         test = test.reset_index()
 68 |         test['ctr'] = test['clicks'] / test['impressions']
 69 |         return test
 70 | 
 71 | 
 72 |     def concat_control(self, dfs):
 73 |         """Concatenate the dataframes used for control
 74 | 
 75 |         Args:
 76 |             dfs (list): List of dataframes extracted from GSC
 77 | 
 78 |         Returns:
 79 |             dataframe: merged control dataframes. 1 metric column by df 
 80 |         """        
 81 |         control_data = []
 82 |         for i in range(len(dfs)):
 83 |             df = dfs[i][['date', self.metric]]
 84 |             df = df.rename(columns={self.metric: f'X{i}'})
 85 |             control_data.append(df)
 86 |         control = reduce(
 87 |                 lambda left, right: pd.merge(
 88 |                         left, right, on=['date'],
 89 |                         how='outer'),
 90 |                 control_data
 91 |                 )
 92 |         return control
 93 | 
 94 | 
 95 |     def create_master_df(self):
 96 |         """Create a master df for a given metric with:
 97 |         y = test (target)
 98 |         Xn = control (features)
 99 | 
100 |         Returns:
101 |             dataframe: df with target and features based on list of sites
102 |         """        
103 |         test = self.extract_from_list(self.test_sites)
104 |         test = self.concat_test(test)
105 |         y = test[['date', self.metric]].rename(columns={self.metric:'y'})
106 | 
107 |         if self.control_sites:
108 |             control = self.extract_from_list(self.control_sites)
109 |             X = self.concat_control(control)
110 |             data = y.merge(X, on='date', how='left')
111 |         else:
112 |             data = y
113 |         return data.sort_values(by='date').reset_index(drop=True)
114 | 
115 | 
116 |     def get_pre_post(self, data):
117 |         """Get pre-post periods based on the intervention date
118 | 
119 |         Args:
120 |             data (dataframe): df comming from create_master_df()
121 | 
122 |         Returns:
123 |             tuple: tuple of lists showing index edges of period before and after intervention
124 |         """        
125 |         pre_start = min(data.index)
126 |         pre_end = int(data[data['date'] == self.intervention].index.values)
127 |         post_start = pre_end + 1
128 |         post_end = max(data.index)
129 | 
130 |         pre_period = [pre_start, pre_end] 
131 |         post_period = [post_start, post_end]
132 |         return pre_period, post_period
133 | 
134 | 
135 |     def make_report(self, data, pre_period, post_period):
136 |         """Creates the built-in CausalImpact report
137 | 
138 |         Args:
139 |             data (dataframe): df comming from create_master_df()
140 |             pre_period (list): list coming from get_pre_post()
141 |             post_period (list): list coming from get_pre_post()
142 |         """        
143 |         ci = CausalImpact(data.drop(['date'], axis=1), pre_period, post_period)
144 |         print(ci.summary())
145 |         print(ci.summary(output='report'))
146 |         ci.plot()
147 | 


--------------------------------------------------------------------------------
/Google-Search-Console/gscbackups.py:
--------------------------------------------------------------------------------
  1 | """
  2 | This Script will backup your Google Search Console Data automatically in a MySQL database using Python.
  3 | 
  4 | @author: Jean-Christophe Chouinard: Technical SEO / Data Scientist. 
  5 | Twitter > https://www.twitter.com/ChouinardJC, 
  6 | LinkedIn > https://www.linkedin.com/in/jeanchristophechouinard/, 
  7 | Blog > https://www.jcchouinard.com/
  8 | 
  9 | """
 10 | 
 11 | import pandas as pd
 12 | import datetime
 13 | import httplib2
 14 | from apiclient.discovery import build
 15 | from collections import defaultdict
 16 | from dateutil import relativedelta
 17 | import argparse
 18 | from oauth2client import client
 19 | from oauth2client import file
 20 | from oauth2client import tools
 21 | import pymysql.cursors
 22 | import sqlalchemy
 23 | 
 24 | """
 25 | Step 1: Create your db (the first time only)
 26 | connection = pymysql.connect(host='localhost',
 27 |                              user='root',
 28 |                              port='',
 29 |                              password='')
 30 | 
 31 | # Simulate the CREATE DATABASE function of mySQL
 32 | try:
 33 |     with connection.cursor() as cursor:
 34 |         cursor.execute('CREATE DATABASE gsc_db')
 35 | 
 36 | finally:
 37 |     connection.close()
 38 | 
 39 | 
 40 | 
 41 | Step 2: Create your table and/or connect to it
 42 | """
 43 | connection = pymysql.connect(host='localhost',
 44 |                              user='root',
 45 |                              port='',
 46 |                              password='',
 47 |                              db='gsc_db',
 48 |                              cursorclass=pymysql.cursors.DictCursor)
 49 | 
 50 | try:
 51 |     with connection.cursor() as cursor:
 52 |         sqlQuery = '''CREATE TABLE IF NOT EXISTS backup_gsc(Date DATE, 
 53 |                                                             Page LONGTEXT, 
 54 |                                                             Query LONGTEXT, 
 55 |                                                             Clicks INT, 
 56 |                                                             Impressions INT, 
 57 |                                                             Ctr DECIMAL(10,2), 
 58 |                                                             Position DECIMAL(3,2))'''
 59 |         cursor.execute(sqlQuery)
 60 | finally:
 61 |     connection.close()
 62 | 
 63 | 
 64 | """
 65 | Step 3: Connect to the API
 66 | """
 67 | site = 'https://www.yoursite.com'
 68 | 
 69 | SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly']
 70 | DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/customsearch/v1/rest')
 71 | 
 72 | CLIENT_SECRETS_PATH = r'C:\Users\YOUR-PATH\client_secrets.json' # Path to client_secrets.json file.
 73 | 
 74 | parser = argparse.ArgumentParser(
 75 |     formatter_class=argparse.RawDescriptionHelpFormatter,
 76 |     parents=[tools.argparser])
 77 | flags = parser.parse_args([])
 78 | 
 79 | 
 80 | flow = client.flow_from_clientsecrets(
 81 |     CLIENT_SECRETS_PATH, scope=SCOPES,
 82 |     message=tools.message_if_missing(CLIENT_SECRETS_PATH))
 83 | 
 84 | storage = file.Storage('searchconsole.dat')
 85 | credentials = storage.get()
 86 | 
 87 | if credentials is None or credentials.invalid:
 88 |   credentials = tools.run_flow(flow, storage, flags)
 89 | http = credentials.authorize(http=httplib2.Http())
 90 | 
 91 | webmasters_service = build('webmasters', 'v3', http=http)
 92 | 
 93 | """
 94 | Step 4: Query All Your Search Traffic
 95 | """
 96 | #Set Date
 97 | end_date = datetime.date.today()-relativedelta.relativedelta(days=3)
 98 | start_date = end_date 
 99 | 
100 | #Execute your API Request
101 | def execute_request(service, property_uri, request):
102 |     return service.searchanalytics().query(siteUrl=property_uri, body=request).execute()
103 | 
104 | #Get more than 25000 rows
105 | maxRows = 25000; 
106 | i = 0;
107 | numRows = maxRows;
108 | scDict = defaultdict(list);
109 | 
110 | while (numRows == 25000 and i < 40) : # Limit to 1M rows
111 |     request = {
112 |         'startDate': datetime.datetime.strftime(start_date,"%Y-%m-%d"),
113 |         'endDate': datetime.datetime.strftime(end_date,'%Y-%m-%d'),
114 |         'dimensions': ['date','page','query'],
115 |         'rowLimit': maxRows, 
116 |         'startRow': i*maxRows
117 |     }
118 |     response = execute_request(webmasters_service, site, request)
119 | #Process the response
120 |     for row in response['rows']:
121 |         scDict['date'].append(row['keys'][0] or 0)    
122 |         scDict['page'].append(row['keys'][1] or 0)
123 |         scDict['query'].append(row['keys'][2] or 0)
124 |         scDict['clicks'].append(row['clicks'] or 0)
125 |         scDict['ctr'].append(row['ctr'] or 0)
126 |         scDict['impressions'].append(row['impressions'] or 0)
127 |         scDict['position'].append(row['position'] or 0)
128 | 
129 |   #Add response to dataframe 
130 |     df = pd.DataFrame(data = scDict)
131 |     df['clicks'] = df['clicks'].astype('int')
132 |     df['ctr'] = df['ctr']*100
133 |     df['impressions'] = df['impressions'].astype('int')
134 |     df['position'] = df['position'].round(2)
135 |     df.sort_values('clicks',inplace=True,ascending=False)
136 |     numRows=len(response['rows'])
137 |     i=i+1
138 | 
139 | 
140 | """
141 | Step 5: Save your data to the MySQL database
142 | """
143 | 
144 | engine = sqlalchemy.create_engine('mysql+pymysql://root:@localhost/gsc_db')
145 | df.columns
146 | df.to_sql(
147 |         name = 'backup_gsc',
148 |         con = engine,
149 |         index = False,
150 |         if_exists = 'append')
151 | 


--------------------------------------------------------------------------------
/GSC_Tutorial/gsc_to_csv_by_month.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import os
  3 | import pandas as pd
  4 | import re
  5 | import time
  6 | 
  7 | from collections import defaultdict
  8 | import datetime
  9 | from datetime import date, timedelta
 10 | from dateutil import relativedelta
 11 | 
 12 | import date_manip as dm
 13 | import file_manip as fm
 14 | from oauth import authorize_creds, execute_request
 15 |                       
 16 | today = datetime.date.today()
 17 | days = relativedelta.relativedelta(days=3)
 18 | default_end = today - days 
 19 | 
 20 | # Create function to extract all the data
 21 | def gsc_to_csv(site,output,creds,start_date,end_date=default_end,storage='authorizedcreds.dat'):
 22 |     get_path = fm.get_full_path(site,output,start_date)
 23 |     domain_name = get_path[1]                       # Get Domain From URL
 24 |     output_path = get_path[3]                       # Folder created with your domain name
 25 |     fm.create_project(domain_name)                  # Create a new project folder
 26 |     csv_dt = fm.get_dates_csvs(output_path,site,output)   # Read existing CSV
 27 |     webmasters_service = authorize_creds(creds)     # Get credentials to log in the api
 28 | 
 29 |     # Set up Dates
 30 |     dates = dm.get_dates(start_date)
 31 |     start_date = dates[1]                           # Start first day of the month.
 32 |     end_date = end_date                             # End 3 days in the past, since GSC don't show latest data.
 33 |     delta = datetime.timedelta(days=1)              # This will let us loop one day at the time
 34 |     scDict = defaultdict(list)                      # initialize empty Dict to store data
 35 |     while start_date <= end_date:                   # Loop through all dates until start_date is equal to end_date.
 36 |         curr_month = dm.date_to_YM(start_date)
 37 |         full_path = os.path.join(output_path + curr_month + '_' + output)
 38 |         print(full_path)
 39 |         # If a GSC csv file exists from previous extraction
 40 |         # and dates in the file match to dates we are extracting...
 41 |         if csv_dt is not None and \
 42 |             dm.date_to_str(start_date) in csv_dt:
 43 |             print('Existing Date: %s' % start_date) #... Print the date
 44 |             start_date += delta                     #... and increment without extraction  
 45 |         else:                                       # If the file doesn't exist, or date don't match...
 46 |             # ... Print and start the extraction
 47 |             print('Start date at beginning: %s' % start_date)
 48 | 
 49 |             maxRows = 25000 # Maximum 25K per call 
 50 |             numRows = 0     # Start at Row Zero
 51 |             status = ''     # Initialize status of extraction
 52 | 
 53 |             while (status != 'Finished') : # As long as today's data have not been fully extracted.
 54 |                 # Extract this information from GSC
 55 |                 dt = dm.date_to_str(start_date)
 56 |                 print(f'date = {dt}')
 57 |                 request = {
 58 |                     'startDate': dt,                        # Get today's date (while loop)
 59 |                     'endDate': dt,                          # Get today's date (while loop)
 60 |                     'dimensions': ['date','page','query'],  # Extract This information
 61 |                     'rowLimit': maxRows,                    # Set number of rows to extract at once (max 25k)
 62 |                     'startRow': numRows                     # Start at row 0, then row 25k, then row 50k... until with all.
 63 |                 }
 64 |                 response = execute_request(webmasters_service, site, request)
 65 |                 #Process the response
 66 |                 try:
 67 |                     for row in response['rows']:
 68 |                         scDict['date'].append(row['keys'][0] or 0)    
 69 |                         scDict['page'].append(row['keys'][1] or 0)
 70 |                         scDict['query'].append(row['keys'][2] or 0)
 71 |                         scDict['clicks'].append(row['clicks'] or 0)
 72 |                         scDict['ctr'].append(row['ctr'] or 0)
 73 |                         scDict['impressions'].append(row['impressions'] or 0)
 74 |                         scDict['position'].append(row['position'] or 0)
 75 |                     print('successful at %i' % numRows)
 76 | 
 77 |                 except:
 78 |                     print('error occurred at %i' % numRows)
 79 | 
 80 |                 # Add response to dataframe 
 81 |                 df = pd.DataFrame(data = scDict)
 82 |                 df['clicks'] = df['clicks'].astype('int')
 83 |                 df['ctr'] = df['ctr']*100
 84 |                 df['impressions'] = df['impressions'].astype('int')
 85 |                 df['position'] = df['position'].round(2)
 86 | 
 87 |                 # Increment the 'start_row'
 88 |                 print('Numrows at the start of loop: %i' % numRows)
 89 |                 try: 
 90 |                     numRows = numRows + len(response['rows'])
 91 |                 except:
 92 |                     status = 'Finished'                 # If no response left, change status
 93 |                 print('Numrows at the end of loop: %i' % numRows)
 94 |                 if numRows % maxRows != 0:              # If numRows not divisible by 25k...
 95 |                     status = 'Finished'                 # change status, you have covered all lines.                
 96 |             #print(f'DF to write {df.head()}')
 97 |             to_write = df[df['date'].str.contains(dm.date_to_str(start_date))]
 98 |             fm.write_to_csv(to_write,full_path)
 99 |             start_date += delta                         # Increment start_date to continue the loop
100 |     print(f'Done extracting {site}')


--------------------------------------------------------------------------------
/Sitemaps/Sitemap From List of URLs/sitemap_0.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  3 |     
  4 |     <url>
  5 |         <loc>https://www.jcchouinard.com/category/analytics/</loc>
  6 |         <lastmod>2021-06-28</lastmod>
  7 |         <changefreq>never</changefreq>
  8 |         <priority>1.0</priority>        
  9 |     </url>
 10 |     
 11 |     <url>
 12 |         <loc>https://www.jcchouinard.com/category/analytics/google-analytics/</loc>
 13 |         <lastmod>2021-06-28</lastmod>
 14 |         <changefreq>never</changefreq>
 15 |         <priority>1.0</priority>        
 16 |     </url>
 17 |     
 18 |     <url>
 19 |         <loc>https://www.jcchouinard.com/category/automation/</loc>
 20 |         <lastmod>2021-06-28</lastmod>
 21 |         <changefreq>never</changefreq>
 22 |         <priority>1.0</priority>        
 23 |     </url>
 24 |     
 25 |     <url>
 26 |         <loc>https://www.jcchouinard.com/category/coding/automation/</loc>
 27 |         <lastmod>2021-06-28</lastmod>
 28 |         <changefreq>never</changefreq>
 29 |         <priority>1.0</priority>        
 30 |     </url>
 31 |     
 32 |     <url>
 33 |         <loc>https://www.jcchouinard.com/category/coding/github/</loc>
 34 |         <lastmod>2021-06-28</lastmod>
 35 |         <changefreq>never</changefreq>
 36 |         <priority>1.0</priority>        
 37 |     </url>
 38 |     
 39 |     <url>
 40 |         <loc>https://www.jcchouinard.com/category/coding/python/</loc>
 41 |         <lastmod>2021-06-28</lastmod>
 42 |         <changefreq>never</changefreq>
 43 |         <priority>1.0</priority>        
 44 |     </url>
 45 |     
 46 |     <url>
 47 |         <loc>https://www.jcchouinard.com/category/coding/sql/</loc>
 48 |         <lastmod>2021-06-28</lastmod>
 49 |         <changefreq>never</changefreq>
 50 |         <priority>1.0</priority>        
 51 |     </url>
 52 |     
 53 |     <url>
 54 |         <loc>https://www.jcchouinard.com/category/github/</loc>
 55 |         <lastmod>2021-06-28</lastmod>
 56 |         <changefreq>never</changefreq>
 57 |         <priority>1.0</priority>        
 58 |     </url>
 59 |     
 60 |     <url>
 61 |         <loc>https://www.jcchouinard.com/category/python/</loc>
 62 |         <lastmod>2021-06-28</lastmod>
 63 |         <changefreq>never</changefreq>
 64 |         <priority>1.0</priority>        
 65 |     </url>
 66 |     
 67 |     <url>
 68 |         <loc>https://www.jcchouinard.com/category/seo/</loc>
 69 |         <lastmod>2021-06-28</lastmod>
 70 |         <changefreq>never</changefreq>
 71 |         <priority>1.0</priority>        
 72 |     </url>
 73 |     
 74 |     <url>
 75 |         <loc>https://www.jcchouinard.com/category/seo/a-b-split-testing/</loc>
 76 |         <lastmod>2021-06-28</lastmod>
 77 |         <changefreq>never</changefreq>
 78 |         <priority>1.0</priority>        
 79 |     </url>
 80 |     
 81 |     <url>
 82 |         <loc>https://www.jcchouinard.com/category/social-medias/facebook/</loc>
 83 |         <lastmod>2021-06-28</lastmod>
 84 |         <changefreq>never</changefreq>
 85 |         <priority>1.0</priority>        
 86 |     </url>
 87 |     
 88 |     <url>
 89 |         <loc>https://www.jcchouinard.com/category/web-dev/</loc>
 90 |         <lastmod>2021-06-28</lastmod>
 91 |         <changefreq>never</changefreq>
 92 |         <priority>1.0</priority>        
 93 |     </url>
 94 |     
 95 |     <url>
 96 |         <loc>https://www.jcchouinard.com/category/analytics/google-search-console/</loc>
 97 |         <lastmod>2021-06-28</lastmod>
 98 |         <changefreq>never</changefreq>
 99 |         <priority>1.0</priority>        
100 |     </url>
101 |     
102 |     <url>
103 |         <loc>https://www.jcchouinard.com/page/1/</loc>
104 |         <lastmod>2021-06-28</lastmod>
105 |         <changefreq>never</changefreq>
106 |         <priority>1.0</priority>        
107 |     </url>
108 |     
109 |     <url>
110 |         <loc>https://www.jcchouinard.com/page/2/</loc>
111 |         <lastmod>2021-06-28</lastmod>
112 |         <changefreq>never</changefreq>
113 |         <priority>1.0</priority>        
114 |     </url>
115 |     
116 |     <url>
117 |         <loc>https://www.jcchouinard.com/page/3/</loc>
118 |         <lastmod>2021-06-28</lastmod>
119 |         <changefreq>never</changefreq>
120 |         <priority>1.0</priority>        
121 |     </url>
122 |     
123 |     <url>
124 |         <loc>https://www.jcchouinard.com/page/4/</loc>
125 |         <lastmod>2021-06-28</lastmod>
126 |         <changefreq>never</changefreq>
127 |         <priority>1.0</priority>        
128 |     </url>
129 |     
130 |     <url>
131 |         <loc>https://www.jcchouinard.com/page/5/</loc>
132 |         <lastmod>2021-06-28</lastmod>
133 |         <changefreq>never</changefreq>
134 |         <priority>1.0</priority>        
135 |     </url>
136 |     
137 |     <url>
138 |         <loc>https://www.jcchouinard.com/page/6/</loc>
139 |         <lastmod>2021-06-28</lastmod>
140 |         <changefreq>never</changefreq>
141 |         <priority>1.0</priority>        
142 |     </url>
143 |     
144 |     <url>
145 |         <loc>https://www.jcchouinard.com/30-days-of-pandas/</loc>
146 |         <lastmod>2021-06-28</lastmod>
147 |         <changefreq>never</changefreq>
148 |         <priority>1.0</priority>        
149 |     </url>
150 |     
151 |     <url>
152 |         <loc>https://www.jcchouinard.com/a-tribute-to-hamlet-batista/</loc>
153 |         <lastmod>2021-06-28</lastmod>
154 |         <changefreq>never</changefreq>
155 |         <priority>1.0</priority>        
156 |     </url>
157 |     
158 |     <url>
159 |         <loc>https://www.jcchouinard.com/a-year-in-review/</loc>
160 |         <lastmod>2021-06-28</lastmod>
161 |         <changefreq>never</changefreq>
162 |         <priority>1.0</priority>        
163 |     </url>
164 |     
165 |     <url>
166 |         <loc>https://www.jcchouinard.com/ab-split-testing-resources/</loc>
167 |         <lastmod>2021-06-28</lastmod>
168 |         <changefreq>never</changefreq>
169 |         <priority>1.0</priority>        
170 |     </url>
171 |     
172 |     <url>
173 |         <loc>https://www.jcchouinard.com/web-stories/getting-started-with-django/</loc>
174 |         <lastmod>2021-06-28</lastmod>
175 |         <changefreq>never</changefreq>
176 |         <priority>1.0</priority>        
177 |     </url>
178 |     
179 |     <url>
180 |         <loc>https://www.jcchouinard.com/best-seo-posts/</loc>
181 |         <lastmod>2021-06-28</lastmod>
182 |         <changefreq>never</changefreq>
183 |         <priority>1.0</priority>        
184 |     </url>
185 |     
186 | </urlset>


--------------------------------------------------------------------------------
/Google-Search-Console/Simple Google Search Console API Call With Filters.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "nbformat": 4,
  3 |  "nbformat_minor": 2,
  4 |  "metadata": {
  5 |   "language_info": {
  6 |    "name": "python",
  7 |    "codemirror_mode": {
  8 |     "name": "ipython",
  9 |     "version": 3
 10 |    }
 11 |   },
 12 |   "orig_nbformat": 2,
 13 |   "file_extension": ".py",
 14 |   "mimetype": "text/x-python",
 15 |   "name": "python",
 16 |   "npconvert_exporter": "python",
 17 |   "pygments_lexer": "ipython3",
 18 |   "version": 3
 19 |  },
 20 |  "cells": [
 21 |   {
 22 |    "cell_type": "markdown",
 23 |    "metadata": {},
 24 |    "source": [
 25 |     "# Connect to Google Search Console API using Python\n",
 26 |     "This notebook shows you how to make a simple Google Search Console API call with filters using Python.\n",
 27 |     "\n",
 28 |     "@author: Jean-Christophe Chouinard: Senior SEO Specialis at Seek.\n",
 29 |     "\n",
 30 |     "[LinkedIn](https://www.linkedin.com/in/jeanchristophechouinard), [Twitter.com/@ChouinardJC](https://twitter.com/chouinardjc), Blog > [jcchouinard.com](https://www.jcchouinard.com)\n",
 31 |     "\n",
 32 |     "Before you can run this code, make sure that you know how to get your [Google Search Console API Keys](https://www.jcchouinard.com/how-to-get-google-search-console-api-keys/)"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "code",
 37 |    "execution_count": 1,
 38 |    "metadata": {},
 39 |    "outputs": [],
 40 |    "source": [
 41 |     "import pandas as pd\n",
 42 |     "import datetime\n",
 43 |     "import httplib2\n",
 44 |     "from apiclient.discovery import build\n",
 45 |     "from oauth2client.client import OAuth2WebServerFlow\n",
 46 |     "from collections import defaultdict\n",
 47 |     "from dateutil import relativedelta\n",
 48 |     "import argparse\n",
 49 |     "from oauth2client import client\n",
 50 |     "from oauth2client import file\n",
 51 |     "from oauth2client import tools\n",
 52 |     "import re\n",
 53 |     "import os"
 54 |    ]
 55 |   },
 56 |   {
 57 |    "cell_type": "markdown",
 58 |    "metadata": {},
 59 |    "source": [
 60 |     "## Get Your Path and Set-Up Your GSC Domain"
 61 |    ]
 62 |   },
 63 |   {
 64 |    "cell_type": "code",
 65 |    "execution_count": 2,
 66 |    "metadata": {},
 67 |    "outputs": [],
 68 |    "source": [
 69 |     "path = os.getcwd()\n",
 70 |     "site = 'https://www.example.com'"
 71 |    ]
 72 |   },
 73 |   {
 74 |    "cell_type": "markdown",
 75 |    "metadata": {},
 76 |    "source": [
 77 |     "## Set Credentials\n",
 78 |     "To run this, you need to add your `client_secrets.json` file to your current working directory. If you don't know how, just check how you can [download your Google API keys](https://www.jcchouinard.com/how-to-get-google-search-console-api-keys/).\n",
 79 |     "\n",
 80 |     "The first time your run this, you will get an error message, because it will add the `/searchconsolereporting.dat` file to your current working directory."
 81 |    ]
 82 |   },
 83 |   {
 84 |    "cell_type": "code",
 85 |    "execution_count": null,
 86 |    "metadata": {},
 87 |    "outputs": [],
 88 |    "source": [
 89 |     "SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly'] # Variable parameter that controls the set of resources that the access token permits.\n",
 90 |     "\n",
 91 |     "CLIENT_SECRETS_PATH = path + '/client_secrets.json' # Path to client_secrets.json file.\n",
 92 |     " \n",
 93 |     "parser = argparse.ArgumentParser(\n",
 94 |     "    formatter_class = argparse.RawDescriptionHelpFormatter,\n",
 95 |     "    parents = [tools.argparser])\n",
 96 |     "flags = parser.parse_args([])\n",
 97 |     "\n",
 98 |     "flow = client.flow_from_clientsecrets(\n",
 99 |     "    CLIENT_SECRETS_PATH, scope = SCOPES,\n",
100 |     "    message = tools.message_if_missing(CLIENT_SECRETS_PATH))\n",
101 |     "\n",
102 |     "storage = file.Storage(path + '/searchconsolereporting.dat')\n",
103 |     "credentials = storage.get()\n"
104 |    ]
105 |   },
106 |   {
107 |    "cell_type": "code",
108 |    "execution_count": null,
109 |    "metadata": {},
110 |    "outputs": [],
111 |    "source": [
112 |     "if credentials is None or credentials.invalid:\n",
113 |     "  credentials = tools.run_flow(flow, storage, flags)\n",
114 |     "http = credentials.authorize(http=httplib2.Http())\n",
115 |     "\n",
116 |     "webmasters_service = build('webmasters', 'v3', http=http)"
117 |    ]
118 |   },
119 |   {
120 |    "cell_type": "markdown",
121 |    "metadata": {},
122 |    "source": [
123 |     "## Set-Up Parameters"
124 |    ]
125 |   },
126 |   {
127 |    "cell_type": "code",
128 |    "execution_count": null,
129 |    "metadata": {},
130 |    "outputs": [],
131 |    "source": [
132 |     "start_date = datetime.date.today() - relativedelta.relativedelta(days=3)\n",
133 |     "end_date = start_date"
134 |    ]
135 |   },
136 |   {
137 |    "cell_type": "code",
138 |    "execution_count": null,
139 |    "metadata": {},
140 |    "outputs": [],
141 |    "source": [
142 |     "maxRows = 25000; \n",
143 |     "scDict = defaultdict(list)\n",
144 |     "print(scDict)"
145 |    ]
146 |   },
147 |   {
148 |    "cell_type": "markdown",
149 |    "metadata": {},
150 |    "source": [
151 |     "## Execute your API Request"
152 |    ]
153 |   },
154 |   {
155 |    "cell_type": "code",
156 |    "execution_count": null,
157 |    "metadata": {},
158 |    "outputs": [],
159 |    "source": [
160 |     "def execute_request(service, property_uri, request):\n",
161 |     "    return service.searchanalytics().query(siteUrl=property_uri, body=request).execute()"
162 |    ]
163 |   },
164 |   {
165 |    "cell_type": "markdown",
166 |    "metadata": {},
167 |    "source": [
168 |     "## Run Request"
169 |    ]
170 |   },
171 |   {
172 |    "cell_type": "code",
173 |    "execution_count": null,
174 |    "metadata": {},
175 |    "outputs": [],
176 |    "source": [
177 |     "request = {\n",
178 |     "        'startDate': datetime.datetime.strftime(start_date,\"%Y-%m-%d\"),\n",
179 |     "        'endDate': datetime.datetime.strftime(end_date,'%Y-%m-%d'),\n",
180 |     "        'dimensions': ['date','page','query'],  #country, device, page, query, searchAppearance\n",
181 |     "        'dimensionFilterGroups': [{\n",
182 |     "          'filters': [{\n",
183 |     "              'dimension': 'page',              \n",
184 |     "              'operator': 'contains',           #contains, equals, notEquals, notContains\n",
185 |     "              'expression': '/job/'\n",
186 |     "          }]\n",
187 |     "         }],\n",
188 |     "        'rowLimit': maxRows \n",
189 |     "    }\n",
190 |     "response = execute_request(webmasters_service, site, request)\n"
191 |    ]
192 |   },
193 |   {
194 |    "cell_type": "code",
195 |    "execution_count": null,
196 |    "metadata": {},
197 |    "outputs": [],
198 |    "source": [
199 |     "for row in response['rows']:\n",
200 |     "        scDict['date'].append(row['keys'][0] or 0)    \n",
201 |     "        scDict['page'].append(row['keys'][1] or 0)\n",
202 |     "        scDict['query'].append(row['keys'][2] or 0)\n",
203 |     "        scDict['clicks'].append(row['clicks'] or 0)\n",
204 |     "        scDict['ctr'].append(row['ctr'] or 0)\n",
205 |     "        scDict['impressions'].append(row['impressions'] or 0)\n",
206 |     "        scDict['position'].append(row['position'] or 0)"
207 |    ]
208 |   },
209 |   {
210 |    "cell_type": "code",
211 |    "execution_count": null,
212 |    "metadata": {},
213 |    "outputs": [],
214 |    "source": [
215 |     "df = pd.DataFrame(data = scDict)\n",
216 |     "df['clicks'] = df['clicks'].astype('int')\n",
217 |     "df['ctr'] = df['ctr']*100\n",
218 |     "df['impressions'] = df['impressions'].astype('int')\n",
219 |     "df['position'] = df['position'].round(2)\n",
220 |     "df.sort_values('clicks',inplace=True,ascending=False)\n",
221 |     "df"
222 |    ]
223 |   }
224 |  ]
225 | }


--------------------------------------------------------------------------------
/Google-Search-Console/GSC_to_CSV_All_keywords.py:
--------------------------------------------------------------------------------
  1 | import pandas as pd
  2 | import datetime
  3 | from datetime import date, timedelta
  4 | import httplib2
  5 | from googleapiclient.discovery import build
  6 | from oauth2client.client import OAuth2WebServerFlow
  7 | from collections import defaultdict
  8 | from dateutil import relativedelta
  9 | import argparse
 10 | from oauth2client import client
 11 | from oauth2client import file
 12 | from oauth2client import tools
 13 | import re
 14 | import os
 15 | from urllib.parse import urlparse
 16 | 
 17 | site = 'https://au.jora.com'    # Property to extract
 18 | num_days = 8                    # Number of Days, Months to Extract
 19 | creds = 'client_secrets.json'   # Credential file from GSC
 20 | output = 'gsc_data.csv'
 21 | 
 22 | # Get Domain Name to Create a Project
 23 | def get_domain_name(start_url):
 24 |     domain_name = '{uri.netloc}'.format(uri=urlparse(start_url))  # Get Domain Name To Name Project
 25 |     domain_name = domain_name.replace('.','_')
 26 |     return domain_name
 27 | 
 28 | 
 29 | # Create a project Directory for this website
 30 | def create_project(directory):
 31 |     if not os.path.exists(directory):
 32 |         print('Create project: '+ directory)
 33 |         os.makedirs(directory)
 34 | 
 35 | def authorize_creds(creds):
 36 |     # Variable parameter that controls the set of resources that the access token permits.
 37 |     SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly'] 
 38 | 
 39 |     # Path to client_secrets.json file
 40 |     CLIENT_SECRETS_PATH = creds
 41 | 
 42 |     # Create a parser to be able to open browser for Authorization
 43 |     parser = argparse.ArgumentParser(
 44 |         formatter_class=argparse.RawDescriptionHelpFormatter,
 45 |         parents=[tools.argparser])
 46 |     flags = parser.parse_args([])
 47 | 
 48 |     flow = client.flow_from_clientsecrets(
 49 |         CLIENT_SECRETS_PATH, scope = SCOPES,
 50 |         message = tools.message_if_missing(CLIENT_SECRETS_PATH))
 51 | 
 52 |     # Prepare credentials and authorize HTTP
 53 |     # If they exist, get them from the storage object
 54 |     # credentials will get written back to a file.
 55 |     storage = file.Storage('authorizedcreds.dat')
 56 |     credentials = storage.get()
 57 | 
 58 |     # If authenticated credentials don't exist, open Browser to authenticate
 59 |     if credentials is None or credentials.invalid:
 60 |         credentials = tools.run_flow(flow, storage, flags)
 61 |     http = credentials.authorize(http=httplib2.Http())
 62 |     webmasters_service = build('webmasters', 'v3', http=http)
 63 |     return webmasters_service
 64 | 
 65 | # Create Function to execute your API Request
 66 | def execute_request(service, property_uri, request):
 67 |     return service.searchanalytics().query(siteUrl=property_uri, body=request).execute()
 68 | 
 69 | # Create function to write to CSV
 70 | def write_to_csv(data,filename):
 71 |     if not os.path.isfile(filename):
 72 |         data.to_csv(filename)
 73 |     else: # else it exists so append without writing the header
 74 |         data.to_csv(filename, mode='a', header=False)
 75 | 
 76 | # Read CSV if it exists to find dates that have already been processed.
 77 | def get_dates_from_csv(path):
 78 |     if os.path.isfile(path):
 79 |         data = pd.read_csv(path)
 80 |         data = pd.Series(data['date'].unique())
 81 |         return data
 82 |     else:
 83 |         pass
 84 | 
 85 | def dt_to_str(date):
 86 |     return datetime.datetime.strftime(date,'%Y-%m-%d')
 87 | 
 88 | # Create function to extract all the data
 89 | def extract_data(site,creds,num_days,output):
 90 |     domain_name = get_domain_name(site)             # Get Domain From URL
 91 |     create_project(domain_name)                     # Create a new project folder
 92 |     full_path = domain_name + '/' + output          # Get Path to add all files
 93 |     csv_dt = get_dates_from_csv(full_path)          # Read existing CSV
 94 |     webmasters_service = authorize_creds(creds)     # Get credentials to log in the api
 95 | 
 96 |     # Set up Dates
 97 |     end_date = datetime.date.today() -\
 98 |          relativedelta.relativedelta(days=3)        # Start 3 days in the past, since GSC don't show latest data.
 99 |     start_date = end_date -\
100 |          relativedelta.relativedelta(days=num_days) # Get end date minus the number of days to go back in time.
101 |     delta = datetime.timedelta(days=1)              # This will let us loop one day at the time
102 |     scDict = defaultdict(list)                      # initialize empty Dict to store data
103 | 
104 |     while start_date <= end_date:                   # Loop through all dates until start_date is equal to end_date.
105 |         
106 |         # If a GSC csv file exists from previous extraction
107 |         # and dates in the file match to dates we are extracting...
108 |         if csv_dt is not None and \
109 |             csv_dt.str.contains(dt_to_str(start_date)).any(): 
110 |             print('Existing Date: %s' % start_date) #... Print the date
111 |             start_date += delta                     #... and increment without extraction  
112 |         else:                                       # If the file doesn't exist, or date don't match...
113 |             # ... Print and start the extraction
114 |             print('Start date at beginning: %s' % start_date)
115 | 
116 |             maxRows = 25000 # Maximum 25K per call 
117 |             numRows = 0     # Start at Row Zero
118 |             status = ''     # Initialize status of extraction
119 | 
120 | 
121 |             while (status != 'Finished') : # As long as today's data have not been fully extracted.
122 |                 # Extract this information from GSC
123 |                 request = {
124 |                     'startDate': dt_to_str(start_date),     # Get today's date (while loop)
125 |                     'endDate': dt_to_str(start_date),       # Get today's date (while loop)
126 |                     'dimensions': ['date','page','query'],  # Extract This information
127 |                     'rowLimit': maxRows,                    # Set number of rows to extract at once (max 25k)
128 |                     'startRow': numRows                     # Start at row 0, then row 25k, then row 50k... until with all.
129 |                 }
130 | 
131 |                 response = execute_request(webmasters_service, site, request)
132 |                 
133 |                 #Process the response
134 |                 try:
135 |                     for row in response['rows']:
136 |                         scDict['date'].append(row['keys'][0] or 0)    
137 |                         scDict['page'].append(row['keys'][1] or 0)
138 |                         scDict['query'].append(row['keys'][2] or 0)
139 |                         scDict['clicks'].append(row['clicks'] or 0)
140 |                         scDict['ctr'].append(row['ctr'] or 0)
141 |                         scDict['impressions'].append(row['impressions'] or 0)
142 |                         scDict['position'].append(row['position'] or 0)
143 |                     print('successful at %i' % numRows)
144 | 
145 |                 except:
146 |                     print('error occurred at %i' % numRows)
147 | 
148 |                 # Add response to dataframe 
149 |                 df = pd.DataFrame(data = scDict)
150 |                 df['clicks'] = df['clicks'].astype('int')
151 |                 df['ctr'] = df['ctr']*100
152 |                 df['impressions'] = df['impressions'].astype('int')
153 |                 df['position'] = df['position'].round(2)
154 | 
155 |                 # Increment the 'start_row'
156 |                 print('Numrows at the start of loop: %i' % numRows)
157 |                 try: 
158 |                     numRows = numRows + len(response['rows'])
159 |                 except:
160 |                     status = 'Finished'                 # If no response left, change status
161 |                 print('Numrows at the end of loop: %i' % numRows)
162 |                 if numRows % maxRows != 0:              # If numRows not divisible by 25k...
163 |                     status = 'Finished'                 # change status, you have covered all lines.                
164 |         
165 |             start_date += delta                         # Increment start_date to continue the loop
166 |             print('Start date at end: %s' % start_date) 
167 |             write_to_csv(df,full_path)                  # Write today's data to CSV file.
168 |         df = pd.read_csv(full_path)
169 |     return df
170 | 
171 | df = extract_data(site,creds,num_days,output)
172 | df.sort_values('clicks',ascending=False)


--------------------------------------------------------------------------------
/Reddit/Use Reddit API With Python.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Use Reddit API With Python and Pushshift.io\n",
  8 |     "This post will show you how to make an API call with Reddit API using Pushshift.io.\n",
  9 |     "\n",
 10 |     "@author: Jean-Christophe Chouinard: Technical SEO / Data Scientist > [LinkedIn](https://www.linkedin.com/in/jeanchristophechouinard/) > [@ChouinardJC](https://twitter.com/ChouinardJC) > Blog > [jcchouinard.com](https://www.jcchouinard.com/)\n",
 11 |     "View the post > [How to use Reddit API With Python](https://www.jcchouinard.com/how-to-use-reddit-api-with-python/) \n",
 12 |     "\n",
 13 |     "\n",
 14 |     "This work must be attributed to [Duarte O.Carmo](https://duarteocarmo.com/)."
 15 |    ]
 16 |   },
 17 |   {
 18 |    "cell_type": "markdown",
 19 |    "metadata": {},
 20 |    "source": [
 21 |     "In this post, I will show you how to extract data from Reddit API to find out which subreddit have the most activity for your search term. "
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "markdown",
 26 |    "metadata": {},
 27 |    "source": [
 28 |     "\n",
 29 |     "## How to Extract Data From Reddit Using Pushshift.io?\n",
 30 |     "\n",
 31 |     "\n",
 32 |     "## Install Dependent Libraries"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "code",
 37 |    "execution_count": null,
 38 |    "metadata": {},
 39 |    "outputs": [],
 40 |    "source": [
 41 |     "# Install packages if you don't have them already installed in the current Jupyter kernel\n",
 42 |     "import sys\n",
 43 |     "!{sys.executable} -m pip install plotly==4.4.1\n",
 44 |     "!{sys.executable} -m pip install pandas\n",
 45 |     "!{sys.executable} -m pip install requests"
 46 |    ]
 47 |   },
 48 |   {
 49 |    "cell_type": "markdown",
 50 |    "metadata": {},
 51 |    "source": [
 52 |     "## Load Libraries"
 53 |    ]
 54 |   },
 55 |   {
 56 |    "cell_type": "code",
 57 |    "execution_count": null,
 58 |    "metadata": {},
 59 |    "outputs": [],
 60 |    "source": [
 61 |     "import pandas as pd\n",
 62 |     "import requests"
 63 |    ]
 64 |   },
 65 |   {
 66 |    "cell_type": "markdown",
 67 |    "metadata": {},
 68 |    "source": [
 69 |     "## Import the JSON Data With Requests"
 70 |    ]
 71 |   },
 72 |   {
 73 |    "cell_type": "code",
 74 |    "execution_count": null,
 75 |    "metadata": {},
 76 |    "outputs": [],
 77 |    "source": [
 78 |     "query=\"python\" \n",
 79 |     "url = f\"https://api.pushshift.io/reddit/search/comment/?q={query}\"\n",
 80 |     "request = requests.get(url)\n",
 81 |     "json_response = request.json()\n",
 82 |     "json_response"
 83 |    ]
 84 |   },
 85 |   {
 86 |    "cell_type": "markdown",
 87 |    "metadata": {},
 88 |    "source": [
 89 |     "## Set-up Your Parameters"
 90 |    ]
 91 |   },
 92 |   {
 93 |    "cell_type": "code",
 94 |    "execution_count": null,
 95 |    "metadata": {},
 96 |    "outputs": [],
 97 |    "source": [
 98 |     "data_type=\"comment\"     # give me comments, use \"submission\" to publish something\n",
 99 |     "query=\"python\"          # Add your query\n",
100 |     "duration=\"30d\"          # Select the timeframe. Epoch value or Integer + \"s,m,h,d\" (i.e. \"second\", \"minute\", \"hour\", \"day\")\n",
101 |     "size=1000               # maximum 1000 comments\n",
102 |     "sort_type=\"score\"       # Sort by score (Accepted: \"score\", \"num_comments\", \"created_utc\")\n",
103 |     "sort=\"desc\"             # sort descending\n",
104 |     "aggs=\"subreddit\"        #\"author\", \"link_id\", \"created_utc\", \"subreddit\""
105 |    ]
106 |   },
107 |   {
108 |    "cell_type": "markdown",
109 |    "metadata": {},
110 |    "source": [
111 |     "## Make a Function to Call The API"
112 |    ]
113 |   },
114 |   {
115 |    "cell_type": "code",
116 |    "execution_count": null,
117 |    "metadata": {},
118 |    "outputs": [],
119 |    "source": [
120 |     "def get_pushshift_data(data_type, **kwargs):\n",
121 |     "    \"\"\"\n",
122 |     "    Gets data from the pushshift api.\n",
123 |     "\n",
124 |     "    data_type can be 'comment' or 'submission'\n",
125 |     "    The rest of the args are interpreted as payload.\n",
126 |     "\n",
127 |     "    Read more: https://github.com/pushshift/api\n",
128 |     "    \"\"\"\n",
129 |     "\n",
130 |     "    base_url = f\"https://api.pushshift.io/reddit/search/{data_type}/\"\n",
131 |     "    payload = kwargs\n",
132 |     "    request = requests.get(base_url, params=payload)\n",
133 |     "    return request.json()"
134 |    ]
135 |   },
136 |   {
137 |    "cell_type": "code",
138 |    "execution_count": null,
139 |    "metadata": {},
140 |    "outputs": [],
141 |    "source": [
142 |     "get_pushshift_data(data_type=data_type,     \n",
143 |     "                   q=query,                 \n",
144 |     "                   after=duration,          \n",
145 |     "                   size=size,               \n",
146 |     "                   sort_type=sort_type,\n",
147 |     "                   sort=sort)             "
148 |    ]
149 |   },
150 |   {
151 |    "cell_type": "code",
152 |    "execution_count": null,
153 |    "metadata": {},
154 |    "outputs": [],
155 |    "source": [
156 |     "data = get_pushshift_data(data_type=data_type,\n",
157 |     "                          q=query,\n",
158 |     "                          after=duration,\n",
159 |     "                          size=size,\n",
160 |     "                          aggs=aggs)"
161 |    ]
162 |   },
163 |   {
164 |    "cell_type": "code",
165 |    "execution_count": null,
166 |    "metadata": {},
167 |    "outputs": [],
168 |    "source": [
169 |     "data = data.get(\"aggs\").get(aggs)"
170 |    ]
171 |   },
172 |   {
173 |    "cell_type": "code",
174 |    "execution_count": null,
175 |    "metadata": {},
176 |    "outputs": [],
177 |    "source": [
178 |     "df = pd.DataFrame.from_records(data)[0:10]\n",
179 |     "df"
180 |    ]
181 |   },
182 |   {
183 |    "cell_type": "code",
184 |    "execution_count": null,
185 |    "metadata": {},
186 |    "outputs": [],
187 |    "source": [
188 |     "import plotly.express as px\n",
189 |     "\n",
190 |     "px.bar(df,              # our dataframe\n",
191 |     "       x=\"key\",         # x will be the 'key' column of the dataframe\n",
192 |     "       y=\"doc_count\",   # y will be the 'doc_count' column of the dataframe\n",
193 |     "       title=f'Subreddits with most activity - comments with \"{query}\" in the last \"{duration}\"',\n",
194 |     "       labels={\"doc_count\": \"# comments\",\"key\": \"Subreddits\"}, # the axis names\n",
195 |     "       color_discrete_sequence=[\"blueviolet\"], # the colors used\n",
196 |     "       height=500,\n",
197 |     "       width=800)"
198 |    ]
199 |   },
200 |   {
201 |    "cell_type": "code",
202 |    "execution_count": null,
203 |    "metadata": {},
204 |    "outputs": [],
205 |    "source": [
206 |     "def make_clickable(val):\n",
207 |     "    \"\"\" Makes a pandas column clickable by wrapping it in some html.\n",
208 |     "    \"\"\"\n",
209 |     "    return '<a href=\"{}\">Link</a>'.format(val,val)"
210 |    ]
211 |   },
212 |   {
213 |    "cell_type": "code",
214 |    "execution_count": null,
215 |    "metadata": {},
216 |    "outputs": [],
217 |    "source": [
218 |     "# Call the API\n",
219 |     "data = get_pushshift_data(data_type=data_type,\n",
220 |     "                          q=query,\n",
221 |     "                          after=\"7d\",\n",
222 |     "                          size=10,\n",
223 |     "                          sort_type=sort_type,\n",
224 |     "                          sort=sort).get(\"data\")\n",
225 |     "\n",
226 |     "# Select the columns you care about\n",
227 |     "df = pd.DataFrame.from_records(data)[[\"author\", \"subreddit\", \"score\", \"body\", \"permalink\"]]\n",
228 |     "\n",
229 |     "# Keep the first 400 characters\n",
230 |     "df['body'] = df['body'].str[0:400] + \"...\"\n",
231 |     "\n",
232 |     "# Append the string to all the permalink entries so that we have a link to the comment\n",
233 |     "df['permalink'] = \"https://reddit.com\" + df['permalink'].astype(str)\n",
234 |     "\n",
235 |     "\n",
236 |     "# Create a function to make the link to be clickable and style the last column\n",
237 |     "def make_clickable(val):\n",
238 |     "    \"\"\" Makes a pandas column clickable by wrapping it in some html.\n",
239 |     "    \"\"\"\n",
240 |     "    return '<a href=\"{}\">Link</a>'.format(val,val)\n",
241 |     "\n",
242 |     "\n",
243 |     "df.style.format({'permalink': make_clickable})"
244 |    ]
245 |   },
246 |   {
247 |    "cell_type": "code",
248 |    "execution_count": null,
249 |    "metadata": {},
250 |    "outputs": [],
251 |    "source": []
252 |   }
253 |  ],
254 |  "metadata": {
255 |   "kernelspec": {
256 |    "display_name": "Python 3",
257 |    "language": "python",
258 |    "name": "python3"
259 |   },
260 |   "language_info": {
261 |    "codemirror_mode": {
262 |     "name": "ipython",
263 |     "version": 3
264 |    },
265 |    "file_extension": ".py",
266 |    "mimetype": "text/x-python",
267 |    "name": "python",
268 |    "nbconvert_exporter": "python",
269 |    "pygments_lexer": "ipython3",
270 |    "version": "3.7.4"
271 |   }
272 |  },
273 |  "nbformat": 4,
274 |  "nbformat_minor": 2
275 | }
276 | 


--------------------------------------------------------------------------------
/Google-Search-Console/GSC_to_CSV_by_month.py:
--------------------------------------------------------------------------------
  1 | import pandas as pd
  2 | import datetime
  3 | from datetime import date, timedelta
  4 | from dateutil import relativedelta
  5 | import httplib2
  6 | from googleapiclient.discovery import build
  7 | from oauth2client.client import OAuth2WebServerFlow
  8 | from collections import defaultdict
  9 | import argparse
 10 | from oauth2client import client
 11 | from oauth2client import file
 12 | from oauth2client import tools
 13 | import re
 14 | import os
 15 | from urllib.parse import urlparse
 16 | import time
 17 | 
 18 | 
 19 | site = 'https://www.example.com'    # Property to extract                   # Number of Days, Months to Extract
 20 | path = os.getcwd()
 21 | creds = 'client_secrets.json'   # Credential file from GSC
 22 | filename = 'gsc_data.csv'
 23 | man_st_dt = '2020-04-23' # Leave empty to start at beginning of the month
 24 | 
 25 | # Get Domain Name to Create a Project
 26 | def get_domain_name(start_url):
 27 |     domain_name = '{uri.netloc}'.format(uri=urlparse(start_url))  # Get Domain Name To Name Project
 28 |     domain_name = domain_name.replace('.','_')
 29 |     return domain_name
 30 | 
 31 | 
 32 | # Create a project Directory for this website
 33 | def create_project(directory):
 34 |     if not os.path.exists(directory):
 35 |         print('Create project: '+ directory)
 36 |         os.makedirs(directory)
 37 | 
 38 | def authorize_creds(creds):
 39 |     # Variable parameter that controls the set of resources that the access token permits.
 40 |     SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly'] 
 41 | 
 42 |     # Path to client_secrets.json file
 43 |     CLIENT_SECRETS_PATH = path + '/' + creds
 44 | 
 45 |     # Create a parser to be able to open browser for Authorization
 46 |     parser = argparse.ArgumentParser(
 47 |         formatter_class=argparse.RawDescriptionHelpFormatter,
 48 |         parents=[tools.argparser])
 49 |     flags = parser.parse_args([])
 50 | 
 51 |     flow = client.flow_from_clientsecrets(
 52 |         CLIENT_SECRETS_PATH, scope = SCOPES,
 53 |         message = tools.message_if_missing(CLIENT_SECRETS_PATH))
 54 | 
 55 |     # Prepare credentials and authorize HTTP
 56 |     # If they exist, get them from the storage object
 57 |     # credentials will get written back to a file.
 58 |     storage = file.Storage(path + '/authorizedcreds.dat')
 59 |     credentials = storage.get()
 60 | 
 61 |     # If authenticated credentials don't exist, open Browser to authenticate
 62 |     if credentials is None or credentials.invalid:
 63 |         credentials = tools.run_flow(flow, storage, flags)
 64 |     http = credentials.authorize(http=httplib2.Http())
 65 |     webmasters_service = build('webmasters', 'v3', http=http)
 66 |     return webmasters_service
 67 | 
 68 | # Create Function to execute your API Request
 69 | def execute_request(service, property_uri, request):
 70 |     return service.searchanalytics().query(siteUrl=property_uri, body=request).execute()
 71 | 
 72 | # Convert dates to strings
 73 | def dt_to_str(date):
 74 |     full_date = datetime.datetime.strftime(date,'%Y-%m-%d')
 75 |     yr_m = datetime.datetime.strftime(date,'%Y-%m')
 76 |     return full_date, yr_m
 77 | 
 78 | # Get Days since start of month
 79 | def get_dates(man_st_dt):
 80 |     end_date = datetime.date.today() - relativedelta.relativedelta(days=3)  
 81 |     if man_st_dt is '':
 82 |         delta = end_date - datetime.date.today().replace(day=1) # Get first day of the month
 83 |         start_date = end_date - delta # count difference between end date and first day of the month
 84 |     else:
 85 |         delta = end_date - datetime.datetime.strptime(man_st_dt,'%Y-%m-%d').date() # Get first day of the month
 86 |         start_date = end_date - delta # count difference between end date and first day of the month
 87 |     str_date = dt_to_str(start_date)[1]
 88 |     return str_date, start_date, end_date
 89 | 
 90 | # Create function to write to CSV
 91 | def write_to_csv(data,filename):
 92 |     if not os.path.isfile(filename):
 93 |         data.to_csv(filename, index=False)
 94 |     else: # else it exists so append without writing the header
 95 |         data.to_csv(filename, mode='a', header=False, index=False)
 96 | 
 97 | 
 98 | def get_full_path(site,filename):
 99 |     output = os.path.join(get_dates(man_st_dt)[0] + '_' + filename)
100 |     domain_name = get_domain_name(site) 
101 |     data_path = domain_name + '/' 
102 |     full_path = data_path + output  
103 |     return output, domain_name, full_path, data_path
104 | 
105 | def loop_csv(filename):
106 |     path = get_full_path(site,filename)[3]
107 |     file_list = []
108 |     for file in os.listdir(path):
109 |         if file.endswith(os.path.join('_'+ filename)):
110 |             file_list.append(file)
111 |             file_list.sort()
112 |     return file_list
113 | 
114 | 
115 | # Read CSV if it exists to find dates that have already been processed.
116 | def get_dates_from_csv(path):
117 |     if os.path.isfile(path):
118 |         data = pd.read_csv(path)
119 |         data = pd.Series(data['date'].unique())
120 |         return data
121 |     else:
122 |         pass
123 | 
124 | 
125 | def get_dates_csvs(site,filename):
126 |     full_path = get_full_path(site,filename)[3]
127 |     dset = set()
128 |     csvs = loop_csv(filename)
129 |     for csv in csvs:
130 |         path = os.path.join(full_path + csv)
131 |         dates = get_dates_from_csv(path)
132 |         for date in dates:
133 |             dset.add(date)
134 |     return dset
135 | 
136 | # Create function to extract all the data
137 | def gsc_to_csv(site,creds,filename):
138 |     domain_name = get_full_path(site,filename)[1]   # Get Domain From URL
139 |     #full_path = get_full_path(site,filename)[2]     # Get Path to add all files
140 |     #print(full_path)
141 |     create_project(domain_name)                     # Create a new project folder
142 |     csv_dt = get_dates_csvs(site,filename)
143 |     #csv_dt = get_dates_from_csv(full_path)          # Read existing CSV
144 |     webmasters_service = authorize_creds(creds)     # Get credentials to log in the api
145 | 
146 |     # Set up Dates
147 |     start_date = get_dates(man_st_dt)[1]                     # Start first day of the month.
148 |     end_date = get_dates(man_st_dt)[2]                       # End 3 days in the past, since GSC don't show latest data.
149 |     delta = datetime.timedelta(days=1)              # This will let us loop one day at the time
150 |     scDict = defaultdict(list)                      # initialize empty Dict to store data
151 |     while start_date <= end_date:                   # Loop through all dates until start_date is equal to end_date.
152 |         curr_month = dt_to_str(start_date)[1]
153 |         full_path = os.path.join(get_full_path(site,filename)[3] + curr_month + '_' + filename)
154 |         print(full_path)
155 |         # If a GSC csv file exists from previous extraction
156 |         # and dates in the file match to dates we are extracting...
157 |         if csv_dt is not None and \
158 |             dt_to_str(start_date)[0] in csv_dt:
159 |             #csv_dt.str.contains(dt_to_str(start_date)[0]).any(): 
160 |             print('Existing Date: %s' % start_date) #... Print the date
161 |             start_date += delta                     #... and increment without extraction  
162 |         else:                                       # If the file doesn't exist, or date don't match...
163 |             # ... Print and start the extraction
164 |             print('Start date at beginning: %s' % start_date)
165 | 
166 |             maxRows = 25000 # Maximum 25K per call 
167 |             numRows = 0     # Start at Row Zero
168 |             status = ''     # Initialize status of extraction
169 | 
170 | 
171 |             while (status != 'Finished') : # As long as today's data have not been fully extracted.
172 |                 # Extract this information from GSC
173 |                 request = {
174 |                     'startDate': dt_to_str(start_date)[0],     # Get today's date (while loop)
175 |                     'endDate': dt_to_str(start_date)[0],       # Get today's date (while loop)
176 |                     'dimensions': ['date','page','query'],  # Extract This information
177 |                     'rowLimit': maxRows,                    # Set number of rows to extract at once (max 25k)
178 |                     'startRow': numRows                     # Start at row 0, then row 25k, then row 50k... until with all.
179 |                 }
180 | 
181 |                 response = execute_request(webmasters_service, site, request)
182 |                 
183 |                 #Process the response
184 |                 try:
185 |                     for row in response['rows']:
186 |                         scDict['date'].append(row['keys'][0] or 0)    
187 |                         scDict['page'].append(row['keys'][1] or 0)
188 |                         scDict['query'].append(row['keys'][2] or 0)
189 |                         scDict['clicks'].append(row['clicks'] or 0)
190 |                         scDict['ctr'].append(row['ctr'] or 0)
191 |                         scDict['impressions'].append(row['impressions'] or 0)
192 |                         scDict['position'].append(row['position'] or 0)
193 |                     print('successful at %i' % numRows)
194 | 
195 |                 except:
196 |                     print('error occurred at %i' % numRows)
197 | 
198 |                 # Add response to dataframe 
199 |                 df = pd.DataFrame(data = scDict)
200 |                 df['clicks'] = df['clicks'].astype('int')
201 |                 df['ctr'] = df['ctr']*100
202 |                 df['impressions'] = df['impressions'].astype('int')
203 |                 df['position'] = df['position'].round(2)
204 | 
205 |                 # Increment the 'start_row'
206 |                 print('Numrows at the start of loop: %i' % numRows)
207 |                 try: 
208 |                     numRows = numRows + len(response['rows'])
209 |                 except:
210 |                     status = 'Finished'                 # If no response left, change status
211 |                 print('Numrows at the end of loop: %i' % numRows)
212 |                 if numRows % maxRows != 0:              # If numRows not divisible by 25k...
213 |                     status = 'Finished'                 # change status, you have covered all lines.                
214 |         
215 |             start_date += delta                         # Increment start_date to continue the loop
216 |             print('Start date at end: %s' % start_date) 
217 |             write_to_csv(df[df['date'].str.contains(curr_month)],full_path)                  # Write today's data to CSV file.
218 |         
219 | 
220 | def return_df(site,filename):
221 |     full_path = get_full_path(site,filename)[2]
222 |     df = pd.read_csv(full_path)
223 |     df.sort_values('clicks',ascending=False).reset_index(inplace=True)
224 | 
225 | 
226 | gsc_to_csv(site,creds,filename)
227 | 
228 | print('Done')
229 | #return_df(site,filename)
230 | 
231 | 


--------------------------------------------------------------------------------
/GSC_Tutorial/run.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "metadata": {
  3 |   "language_info": {
  4 |    "codemirror_mode": {
  5 |     "name": "ipython",
  6 |     "version": 3
  7 |    },
  8 |    "file_extension": ".py",
  9 |    "mimetype": "text/x-python",
 10 |    "name": "python",
 11 |    "nbconvert_exporter": "python",
 12 |    "pygments_lexer": "ipython3",
 13 |    "version": 3
 14 |   },
 15 |   "orig_nbformat": 2,
 16 |   "kernelspec": {
 17 |    "name": "python_defaultSpec_1597032642394",
 18 |    "display_name": "Python 3.7.6 64-bit ('base': conda)"
 19 |   }
 20 |  },
 21 |  "nbformat": 4,
 22 |  "nbformat_minor": 2,
 23 |  "cells": [
 24 |   {
 25 |    "cell_type": "code",
 26 |    "execution_count": 1,
 27 |    "metadata": {},
 28 |    "outputs": [],
 29 |    "source": [
 30 |     "from dateutil import relativedelta\n",
 31 |     "\n",
 32 |     "from oauth import authorize_creds, execute_request\n"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "code",
 37 |    "execution_count": 2,
 38 |    "metadata": {},
 39 |    "outputs": [],
 40 |    "source": [
 41 |     "site = 'https://www.jcchouinard.com'  # Property to extract              \n",
 42 |     "creds = 'client_secrets.json'       # Credential file from GSC\n",
 43 |     "storage = 'authorizedcreds.dat'\n",
 44 |     "output = 'gsc_data.csv'\n",
 45 |     "start_date = '2020-07-01'          "
 46 |    ]
 47 |   },
 48 |   {
 49 |    "cell_type": "code",
 50 |    "execution_count": 4,
 51 |    "metadata": {
 52 |     "tags": []
 53 |    },
 54 |    "outputs": [
 55 |     {
 56 |      "output_type": "stream",
 57 |      "name": "stdout",
 58 |      "text": "Authorizing creds\nYour browser has been opened to visit:\n\n    https://accounts.google.com/o/oauth2/auth?client_id=378775270293-cak7p61tk419qn06p7e9atv2a0da09fk.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8090%2F&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fwebmasters.readonly&access_type=offline&response_type=code\n\nIf your browser is on a different machine then exit and re-run this\napplication with the command-line parameter \n\n  --noauth_local_webserver\n\n"
 59 |     },
 60 |     {
 61 |      "output_type": "error",
 62 |      "ename": "AttributeError",
 63 |      "evalue": "'str' object has no attribute 'put'",
 64 |      "traceback": [
 65 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
 66 |       "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
 67 |       "\u001b[0;32m<ipython-input-4-f5d326c692a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mlist_of_urls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msite\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlist_of_urls\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mgsc_by_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist_of_urls\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcreds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstart_date\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'authorizedcreds.dat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
 68 |       "\u001b[0;32m~/Documents/Github/SEO-Projects/RankSense_rstwittorial/gsc_by_url.py\u001b[0m in \u001b[0;36mgsc_by_url\u001b[0;34m(list_of_urls, creds, start_date, end_date, storage)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgsc_by_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist_of_urls\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcreds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstart_date\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mend_date\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdefault_end\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'authorizedcreds.dat'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mwebmasters_service\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauthorize_creds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcreds\u001b[0m\u001b[0;34m)\u001b[0m     \u001b[0;31m# Get credentials to log in the api\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mscDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefaultdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0murl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlist_of_urls\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 69 |       "\u001b[0;32m~/Documents/Github/SEO-Projects/RankSense_rstwittorial/oauth.py\u001b[0m in \u001b[0;36mauthorize_creds\u001b[0;34m(creds, storage)\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0;31m# If authenticated credentials don't exist, open Browser to authenticate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcredentials\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minvalid\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0mcredentials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_flow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'authorizedcreds.dat'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m     \u001b[0mhttp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauthorize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhttplib2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHttp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mwebmasters_service\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'webmasters'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'v3'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhttp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhttp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 70 |       "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/oauth2client/util.py\u001b[0m in \u001b[0;36mpositional_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    138\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# IGNORE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m                     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mpositional_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 71 |       "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/oauth2client/tools.py\u001b[0m in \u001b[0;36mrun_flow\u001b[0;34m(flow, storage, flags, http)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Authentication has failed: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m     \u001b[0mstorage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcredential\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m     \u001b[0mcredential\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_store\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Authentication successful.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 72 |       "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute 'put'"
 73 |      ]
 74 |     }
 75 |    ],
 76 |    "source": [
 77 |     "from gsc_by_url import gsc_by_url\n",
 78 |     "\n",
 79 |     "list_of_urls = ['/python-for-seo','/learn-selenium-python-seo-automation']\n",
 80 |     "list_of_urls = [site + x for x in list_of_urls]\n",
 81 |     "\n",
 82 |     "gsc_by_url(list_of_urls,creds,start_date,storage='authorizedcreds.dat')"
 83 |    ]
 84 |   },
 85 |   {
 86 |    "cell_type": "code",
 87 |    "execution_count": null,
 88 |    "metadata": {},
 89 |    "outputs": [],
 90 |    "source": [
 91 |     "from gsc_with_filters import gsc_with_filters\n",
 92 |     "\n",
 93 |     "gsc_with_filterssite,creds,start_date,end_date=default_end,storage='authorizedcreds.dat')\n"
 94 |    ]
 95 |   },
 96 |   {
 97 |    "cell_type": "code",
 98 |    "execution_count": 8,
 99 |    "metadata": {
100 |     "tags": []
101 |    },
102 |    "outputs": [
103 |     {
104 |      "output_type": "stream",
105 |      "name": "stdout",
106 |      "text": "Fetching start and end dates from 2020-07-01\nstart_date: 2020-07-01, end_date: 2020-08-07\nwww_jcchouinard_com project exists\nChecking existing dates in www_jcchouinard_com/\nAuthorizing creds\nFailed to start a local webserver listening on either port 8080\nor port 8090. Please check your firewall settings and locally\nrunning programs that may be blocking or using those ports.\n\nFalling back to --noauth_local_webserver and continuing with\nauthorization.\n\nGo to the following link in your browser:\n\n    https://accounts.google.com/o/oauth2/auth?client_id=378775270293-cak7p61tk419qn06p7e9atv2a0da09fk.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fwebmasters.readonly&access_type=offline&response_type=code\n\nERROR:root:Internal Python error in the inspect module.\nBelow is the traceback from this internal error.\n\nTraceback (most recent call last):\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/oauth2client/tools.py\", line 231, in run_flow\n    credential = flow.step2_exchange(code, http=http)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/oauth2client/util.py\", line 140, in positional_wrapper\n    return wrapped(*args, **kwargs)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/oauth2client/client.py\", line 2172, in step2_exchange\n    raise FlowExchangeError(error_msg)\noauth2client.client.FlowExchangeError: invalid_requestMissing required parameter: code\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3331, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-deecbc6f4273>\", line 3, in <module>\n    gsc_to_csv(site,output,creds,start_date,storage=storage)\n  File \"/Users/jean.christophe/Documents/Github/SEO-Projects/RankSense_rstwittorial/gsc_to_csv_by_month.py\", line 27, in gsc_to_csv\n    webmasters_service = authorize_creds(creds)     # Get credentials to log in the api\n  File \"/Users/jean.christophe/Documents/Github/SEO-Projects/RankSense_rstwittorial/oauth.py\", line 39, in authorize_creds\n    credentials = tools.run_flow(flow, 'authorizedcreds.dat', flags)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/oauth2client/util.py\", line 140, in positional_wrapper\n    return wrapped(*args, **kwargs)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/oauth2client/tools.py\", line 233, in run_flow\n    sys.exit('Authentication has failed: %s' % e)\nSystemExit: Authentication has failed: invalid_requestMissing required parameter: code\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1151, in get_records\n    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 319, in wrapped\n    return f(*args, **kwargs)\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 353, in _fixed_getinnerframes\n    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n  File \"/Users/jean.christophe/opt/anaconda3/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\nAttributeError: 'tuple' object has no attribute 'tb_frame'\n"
107 |     },
108 |     {
109 |      "output_type": "error",
110 |      "ename": "SystemExit",
111 |      "evalue": "Authentication has failed: invalid_requestMissing required parameter: code",
112 |      "traceback": [
113 |       "An exception has occurred, use %tb to see the full traceback.\n",
114 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
115 |       "\nDuring handling of the above exception, another exception occurred:\n",
116 |       "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m Authentication has failed: invalid_requestMissing required parameter: code\n"
117 |      ]
118 |     }
119 |    ],
120 |    "source": [
121 |     "from gsc_to_csv_by_month import gsc_to_csv\n",
122 |     "\n",
123 |     "gsc_to_csv(site,output,creds,start_date,storage=storage)"
124 |    ]
125 |   },
126 |   {
127 |    "cell_type": "code",
128 |    "execution_count": null,
129 |    "metadata": {},
130 |    "outputs": [],
131 |    "source": []
132 |   }
133 |  ]
134 | }


--------------------------------------------------------------------------------
/Web-Scraping/Web Scraping With Python and Requests-HTML.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Web Scraping With Python and Requests-HTML\n",
  8 |     "\n",
  9 |     "In this Python for SEO tutorial, we will learn how to scrape a website with Python using the `Requests-HTML` library.\n",
 10 |     "\n",
 11 |     "(Code example included)\n",
 12 |     "\n",
 13 |     "@author: Jean-Christophe Chouinard: Technical SEO / Data Scientist > [LinkedIn](https://www.linkedin.com/in/jeanchristophechouinard/) > [@ChouinardJC](https://twitter.com/ChouinardJC) > Blog > [jcchouinard.com](https://www.jcchouinard.com/) > Complete Tutorial > [Web Scraping With Python and Requests-HTML](https://www.jcchouinard.com/web-scraping-with-python-and-requests-html/)\n",
 14 |     "\n",
 15 |     "## What is Web Scraping?\n",
 16 |     "*Web scraping* means the action of *parsing* the content of a webpage to extract specific information.\n",
 17 |     "\n",
 18 |     "*Parsing* means that you analyze a document to describe the syntax (i.e. the HTML structure). Without a *parser*, your HTML document will look like a single block of text.\n",
 19 |     "\n",
 20 |     "When you are scraping a website, you are asking the server to send you an HTML document that you *parse* to understand the building blocks (`<head>`,`<body>`,`<title>`,`<h1>`, etc.). Once the structure is understood, you can pull out any information that you want.\n",
 21 |     "\n",
 22 |     "## What is Requests-HTML Library?\n",
 23 |     "\n",
 24 |     "The `requests-HTML` library is a HTML parser that lets you use *CSS Selectors* and *XPath Selectors* to extract the information that you want from a web page.\n",
 25 |     "\n",
 26 |     "\n",
 27 |     "## Install and load Libraries\n",
 28 |     "\n",
 29 |     "In this tutorial, we will use the `requests` library to \"call\" the URL by making HTTP requests to servers, the `requests-HTML` library to *parse* the data, and the `pandas` library to work with the scraped information. "
 30 |    ]
 31 |   },
 32 |   {
 33 |    "cell_type": "code",
 34 |    "execution_count": null,
 35 |    "metadata": {},
 36 |    "outputs": [],
 37 |    "source": [
 38 |     "!pip install requests\n",
 39 |     "!pip install requests-HTML\n",
 40 |     "!pip install pandas\n",
 41 |     "!pip install regex\n",
 42 |     "!pip install urlparse4"
 43 |    ]
 44 |   },
 45 |   {
 46 |    "cell_type": "markdown",
 47 |    "metadata": {},
 48 |    "source": [
 49 |     "## Call the URL With requests.get()\n",
 50 |     "\n",
 51 |     "Use `HTMLSession()` to initialize the GET requests and the `.get()` function from `requests` to call the URL that you want to scrape.\n",
 52 |     "\n",
 53 |     "Just to make sure that there is no error, I will add a `try` and `except` statement to return an error in any case the code don't work.\n",
 54 |     "\n",
 55 |     "We will store the response in a variable called `response`."
 56 |    ]
 57 |   },
 58 |   {
 59 |    "cell_type": "code",
 60 |    "execution_count": null,
 61 |    "metadata": {},
 62 |    "outputs": [],
 63 |    "source": [
 64 |     "import requests\n",
 65 |     "from requests_html import HTMLSession\n",
 66 |     "\n",
 67 |     "url = \"https://www.searchenginejournal.com/introduction-to-python-seo-spreadsheets/342779/\"\n",
 68 |     "\n",
 69 |     "try:\n",
 70 |     "    session = HTMLSession()\n",
 71 |     "    response = session.get(url)\n",
 72 |     "    \n",
 73 |     "except requests.exceptions.RequestException as error:\n",
 74 |     "    print(error)"
 75 |    ]
 76 |   },
 77 |   {
 78 |    "cell_type": "markdown",
 79 |    "metadata": {},
 80 |    "source": [
 81 |     "## Structure of Scraping Functions\n",
 82 |     "\n",
 83 |     "The structure of the `requests-HTML` parsing call goes like this:\n",
 84 |     "\n",
 85 |     "`variable.attribute.function(*selector*, parameters)`\n",
 86 |     "\n",
 87 |     "The `variable` is the instance that you created using the `.get(url)` function.\n",
 88 |     "\n",
 89 |     "The `attribute` is the type of content that you want to extract (`html` / `lxml`).\n",
 90 |     "\n",
 91 |     "The `requests-HTML` parser also has many useful built-in `methods` for SEOs.\n",
 92 |     "\n",
 93 |     "* **links**: Get all links found on a page (anchors included);\n",
 94 |     "* **absolute_links**: Get all links found on a page  (anchors excluded);\n",
 95 |     "* **find()**: Find a specific element on a page with a CSS Selector;\n",
 96 |     "* **xpath()**: Get elements using Xpath function;"
 97 |    ]
 98 |   },
 99 |   {
100 |    "cell_type": "markdown",
101 |    "metadata": {},
102 |    "source": [
103 |     "### Extract the Title From the Page\n",
104 |     "\n",
105 |     "Here, we are going to use `find()` with the `html` attribute to \"find\" the `<title>` tag using the `'title'` *CSS Selector* and return a list of elements (`[<Element 'title' >]`)."
106 |    ]
107 |   },
108 |   {
109 |    "cell_type": "code",
110 |    "execution_count": null,
111 |    "metadata": {},
112 |    "outputs": [],
113 |    "source": [
114 |     "title =  response.html.find('title')\n",
115 |     "print(title)"
116 |    ]
117 |   },
118 |   {
119 |    "cell_type": "markdown",
120 |    "metadata": {},
121 |    "source": [
122 |     "To print to actual title, we need to use the index with the `text` attribute."
123 |    ]
124 |   },
125 |   {
126 |    "cell_type": "code",
127 |    "execution_count": null,
128 |    "metadata": {},
129 |    "outputs": [],
130 |    "source": [
131 |     "print(title[0].text)"
132 |    ]
133 |   },
134 |   {
135 |    "cell_type": "markdown",
136 |    "metadata": {},
137 |    "source": [
138 |     "This is the same as using the `first` parameter in the `function` in a one-liner."
139 |    ]
140 |   },
141 |   {
142 |    "cell_type": "code",
143 |    "execution_count": null,
144 |    "metadata": {},
145 |    "outputs": [],
146 |    "source": [
147 |     "title =  response.html.find('title', first=True).text\n",
148 |     "print(title)"
149 |    ]
150 |   },
151 |   {
152 |    "cell_type": "markdown",
153 |    "metadata": {},
154 |    "source": [
155 |     "### Extract Meta Description\n",
156 |     "\n",
157 |     "To extract the meta description from a page, we will use the `xpath()` function with the `//meta[@name=\"description\"]/@content` Xpath. "
158 |    ]
159 |   },
160 |   {
161 |    "cell_type": "code",
162 |    "execution_count": null,
163 |    "metadata": {},
164 |    "outputs": [],
165 |    "source": [
166 |     "meta_desc =  response.html.xpath('//meta[@name=\"description\"]/@content')\n",
167 |     "print(meta_desc)"
168 |    ]
169 |   },
170 |   {
171 |    "cell_type": "markdown",
172 |    "metadata": {},
173 |    "source": [
174 |     "### Extract All Links From a Webpage"
175 |    ]
176 |   },
177 |   {
178 |    "cell_type": "code",
179 |    "execution_count": null,
180 |    "metadata": {},
181 |    "outputs": [],
182 |    "source": [
183 |     "links = response.html.absolute_links\n",
184 |     "print(links)"
185 |    ]
186 |   },
187 |   {
188 |    "cell_type": "markdown",
189 |    "metadata": {},
190 |    "source": [
191 |     "### Extract Information Using Class or ID\n",
192 |     "\n",
193 |     "You can extract any specific information from a page using the dot (`.`) notation to select a class, or the pound (`#`) notation to select the ID.\n",
194 |     "\n",
195 |     "Here we are going to extract the author name using the class."
196 |    ]
197 |   },
198 |   {
199 |    "cell_type": "code",
200 |    "execution_count": 7,
201 |    "metadata": {},
202 |    "outputs": [
203 |     {
204 |      "name": "stdout",
205 |      "output_type": "stream",
206 |      "text": [
207 |       "Hamlet Batista\n"
208 |      ]
209 |     }
210 |    ],
211 |    "source": [
212 |     "author = response.html.find('.post-author', first=True).text\n",
213 |     "print(author)"
214 |    ]
215 |   },
216 |   {
217 |    "cell_type": "markdown",
218 |    "metadata": {},
219 |    "source": [
220 |     "### Extract Canonical Link"
221 |    ]
222 |   },
223 |   {
224 |    "cell_type": "code",
225 |    "execution_count": 8,
226 |    "metadata": {},
227 |    "outputs": [
228 |     {
229 |      "name": "stdout",
230 |      "output_type": "stream",
231 |      "text": [
232 |       "['https://www.searchenginejournal.com/introduction-to-python-seo-spreadsheets/342779/']\n"
233 |      ]
234 |     }
235 |    ],
236 |    "source": [
237 |     "canonical = response.html.xpath(\"//link[@rel='canonical']/@href\")\n",
238 |     "print(canonical)"
239 |    ]
240 |   },
241 |   {
242 |    "cell_type": "markdown",
243 |    "metadata": {},
244 |    "source": [
245 |     "### Extract Hreflang"
246 |    ]
247 |   },
248 |   {
249 |    "cell_type": "code",
250 |    "execution_count": 9,
251 |    "metadata": {},
252 |    "outputs": [
253 |     {
254 |      "name": "stdout",
255 |      "output_type": "stream",
256 |      "text": [
257 |       "[]\n"
258 |      ]
259 |     }
260 |    ],
261 |    "source": [
262 |     "hreflang = response.html.xpath(\"//link[@rel='alternate']/@hreflang\")\n",
263 |     "print(hreflang)"
264 |    ]
265 |   },
266 |   {
267 |    "cell_type": "markdown",
268 |    "metadata": {},
269 |    "source": [
270 |     "### Extract Meta Robots"
271 |    ]
272 |   },
273 |   {
274 |    "cell_type": "code",
275 |    "execution_count": 10,
276 |    "metadata": {},
277 |    "outputs": [
278 |     {
279 |      "name": "stdout",
280 |      "output_type": "stream",
281 |      "text": [
282 |       "['NOODP']\n"
283 |      ]
284 |     }
285 |    ],
286 |    "source": [
287 |     "meta_robots = response.html.xpath(\"//meta[@name='ROBOTS']/@content\")\n",
288 |     "print(meta_robots)"
289 |    ]
290 |   },
291 |   {
292 |    "cell_type": "markdown",
293 |    "metadata": {},
294 |    "source": [
295 |     "### Extract Nested Information\n",
296 |     "\n",
297 |     "To extract information within a specific location you can dig down the DOM using CSS Selectors."
298 |    ]
299 |   },
300 |   {
301 |    "cell_type": "code",
302 |    "execution_count": 11,
303 |    "metadata": {},
304 |    "outputs": [],
305 |    "source": [
306 |     "get_nav_links = response.html.find('a.sub-m-cat span')"
307 |    ]
308 |   },
309 |   {
310 |    "cell_type": "markdown",
311 |    "metadata": {},
312 |    "source": [
313 |     "We will build a for loop to loop through all the indices in the `nav_links` list and add the text to another list called `nav_links`."
314 |    ]
315 |   },
316 |   {
317 |    "cell_type": "code",
318 |    "execution_count": 12,
319 |    "metadata": {},
320 |    "outputs": [
321 |     {
322 |      "data": {
323 |       "text/plain": [
324 |        "['SEO', 'PPC', 'CONTENT', 'SOCIAL', 'NEWS', 'ADVERTISE', 'MORE']"
325 |       ]
326 |      },
327 |      "execution_count": 12,
328 |      "metadata": {},
329 |      "output_type": "execute_result"
330 |     }
331 |    ],
332 |    "source": [
333 |     "nav_links = []\n",
334 |     "\n",
335 |     "for i in range(len(get_nav_links)):\n",
336 |     "    x = get_nav_links[i].text\n",
337 |     "    nav_links.append(x)\n",
338 |     "    \n",
339 |     "nav_links"
340 |    ]
341 |   },
342 |   {
343 |    "cell_type": "markdown",
344 |    "metadata": {},
345 |    "source": [
346 |     "### Save a Subsection of a Page in a Variable\n",
347 |     "\n",
348 |     "If the content that you want to extract is always in a specific `<div>`, you can save the path in a variable to call it.\n",
349 |     "\n",
350 |     "Here, I will extract links that are in the actual content of a post by \"saving\" the `post-342779` article in a variable called `article`."
351 |    ]
352 |   },
353 |   {
354 |    "cell_type": "code",
355 |    "execution_count": 13,
356 |    "metadata": {},
357 |    "outputs": [],
358 |    "source": [
359 |     "article = response.html.find('article.cis_post_item_initial.post-342779', first=True)\n",
360 |     "article_links = article.xpath('//a/@href')"
361 |    ]
362 |   },
363 |   {
364 |    "cell_type": "markdown",
365 |    "metadata": {},
366 |    "source": [
367 |     "## Case Study: Extract Broken Links"
368 |    ]
369 |   },
370 |   {
371 |    "cell_type": "code",
372 |    "execution_count": 15,
373 |    "metadata": {},
374 |    "outputs": [],
375 |    "source": [
376 |     "import re\n",
377 |     "import requests\n",
378 |     "from requests_html import HTMLSession\n",
379 |     "from urllib.parse import urlparse\n",
380 |     "\n",
381 |     "# Get Domain Name With urlparse\n",
382 |     "url = \"https://www.jobillico.com/fr/partenaires-corporatifs\"\n",
383 |     "parsed_url = urlparse(url)\n",
384 |     "domain = parsed_url.scheme + \"://\" + parsed_url.netloc\n",
385 |     "\n",
386 |     "# Get URL \n",
387 |     "session = HTMLSession()\n",
388 |     "r = session.get(url)\n",
389 |     "\n",
390 |     "# Extract Links\n",
391 |     "jlinks = r.html.xpath('//a/@href')\n",
392 |     "\n",
393 |     "# Remove bad links and replace relative path for absolute path\n",
394 |     "updated_links = []\n",
395 |     "\n",
396 |     "for link in jlinks:\n",
397 |     "    if re.search(\".*@.*|.*javascript:.*|.*tel:.*\",link):\n",
398 |     "        link = \"\"\n",
399 |     "    elif re.search(\"^(?!http).*\",link):\n",
400 |     "        link = domain + link\n",
401 |     "        updated_links.append(link)\n",
402 |     "    else:\n",
403 |     "        updated_links.append(link)"
404 |    ]
405 |   },
406 |   {
407 |    "cell_type": "code",
408 |    "execution_count": null,
409 |    "metadata": {},
410 |    "outputs": [],
411 |    "source": [
412 |     "print(updated_links)"
413 |    ]
414 |   },
415 |   {
416 |    "cell_type": "code",
417 |    "execution_count": null,
418 |    "metadata": {},
419 |    "outputs": [],
420 |    "source": [
421 |     "broken_links = []\n",
422 |     "\n",
423 |     "for link in updated_links:\n",
424 |     "    print(link)\n",
425 |     "    try: \n",
426 |     "        requests.get(link, timeout=10).status_code\n",
427 |     "        if requests.get(link, timeout=10).status_code != 200:\n",
428 |     "            broken_links.append(link)\n",
429 |     "    except requests.exceptions.RequestException as e:\n",
430 |     "        print(e)\n",
431 |     "\n",
432 |     "broken_links"
433 |    ]
434 |   },
435 |   {
436 |    "cell_type": "markdown",
437 |    "metadata": {},
438 |    "source": [
439 |     "## Full Code"
440 |    ]
441 |   },
442 |   {
443 |    "cell_type": "code",
444 |    "execution_count": null,
445 |    "metadata": {},
446 |    "outputs": [],
447 |    "source": [
448 |     "import pandas as pd\n",
449 |     "import requests\n",
450 |     "from requests_html import HTMLSession\n",
451 |     "\n",
452 |     "\n",
453 |     "url = \"https://www.searchenginejournal.com/introduction-to-python-seo-spreadsheets/342779/\"\n",
454 |     "\n",
455 |     "try:\n",
456 |     "    session = HTMLSession()\n",
457 |     "    response = session.get(url)\n",
458 |     "except HTTPError as error:\n",
459 |     "    print(error)\n",
460 |     "\n",
461 |     "    \n",
462 |     "# Get Title\n",
463 |     "title =  response.html.find('title', first=True).text\n",
464 |     "\n",
465 |     "#Get H1\n",
466 |     "h1 =  response.html.find('h1', first=True).text\n",
467 |     "\n",
468 |     "#Get all Links\n",
469 |     "links = response.html.absolute_links\n",
470 |     "\n",
471 |     "#Get Author using Class\n",
472 |     "author = response.html.find('.post-author', first=True).text\n",
473 |     "\n",
474 |     "#Get Canonical Link\n",
475 |     "canonical = response.html.xpath(\"//link[@rel='canonical']/@href\")\n",
476 |     "\n",
477 |     "#Get Hreflang\n",
478 |     "hreflang = response.html.xpath(\"//link[@rel='alternate']/@hreflang\")\n",
479 |     "\n",
480 |     "#Get Meta Robots\n",
481 |     "meta_robots = response.html.xpath(\"//meta[@name='ROBOTS']/@content\")\n",
482 |     "\n",
483 |     "#Get Navigational links using nested CSS Selector and For Loops\n",
484 |     "get_nav_links = response.html.find('a.sub-m-cat span')\n",
485 |     "\n",
486 |     "nav_links = []\n",
487 |     "\n",
488 |     "for i in range(len(get_nav_links)):\n",
489 |     "    x = get_nav_links[i].text\n",
490 |     "    nav_links.append(x)\n",
491 |     "    \n",
492 |     "nav_links\n",
493 |     "\n",
494 |     "#Create a variable to extract dat from the actual article only.\n",
495 |     "article = response.html.find('article.cis_post_item_initial.post-342779', first=True)\n",
496 |     "article_links = article.xpath('//a/@href')\n"
497 |    ]
498 |   },
499 |   {
500 |    "cell_type": "code",
501 |    "execution_count": null,
502 |    "metadata": {},
503 |    "outputs": [],
504 |    "source": []
505 |   }
506 |  ],
507 |  "metadata": {
508 |   "kernelspec": {
509 |    "display_name": "Python 3",
510 |    "language": "python",
511 |    "name": "python3"
512 |   },
513 |   "language_info": {
514 |    "codemirror_mode": {
515 |     "name": "ipython",
516 |     "version": 3
517 |    },
518 |    "file_extension": ".py",
519 |    "mimetype": "text/x-python",
520 |    "name": "python",
521 |    "nbconvert_exporter": "python",
522 |    "pygments_lexer": "ipython3",
523 |    "version": "3.7.4"
524 |   }
525 |  },
526 |  "nbformat": 4,
527 |  "nbformat_minor": 2
528 | }
529 | 


--------------------------------------------------------------------------------
/Rainbow_lorikeet.jpg:
--------------------------------------------------------------------------------
  1 | 
  2 | <!DOCTYPE html>
  3 | <html class="client-nojs" lang="fr" dir="ltr">
  4 | <head>
  5 | <meta charset="UTF-8"/>
  6 | <title>loriquet arc-en-ciel — Wiktionnaire</title>
  7 | <script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":!1,"wgSeparatorTransformTable":[",\t."," \t,"],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","janvier","février","mars","avril","mai","juin","juillet","août","septembre","octobre","novembre","décembre"],"wgRequestId":"XrTW6gpAAMMAAkAv60gAAAAB","wgCSPNonce":!1,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":!1,"wgNamespaceNumber":0,"wgPageName":"loriquet_arc-en-ciel","wgTitle":"loriquet arc-en-ciel","wgCurRevisionId":25708973,"wgRevisionId":25708973,"wgArticleId":3228278,"wgIsArticle":!0,"wgIsRedirect":!1,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Traductions en conventions internationales","Traductions en allemand","Traductions en anglais","Traductions en catalan","Traductions en danois","Traductions en espagnol","Traductions en italien","Traductions en néerlandais","Traductions en polonais","Traductions en portugais",
  8 | "Traductions en slovaque","Traductions en suédois","Pages liées à Wikipédia en français","français","Locutions nominales en français","Perroquets et perruches en français"],"wgPageContentLanguage":"fr","wgPageContentModel":"wikitext","wgRelevantPageName":"loriquet_arc-en-ciel","wgRelevantArticleId":3228278,"wgIsProbablyEditable":!0,"wgRelevantPageIsProbablyEditable":!0,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgMediaViewerOnClick":!0,"wgMediaViewerEnabledByDefault":!0,"wgVisualEditor":{"pageLanguageCode":"fr","pageLanguageDir":"ltr","pageVariantFallbacks":"fr"},"wgMFDisplayWikibaseDescriptions":{"search":!0,"nearby":!0,"watchlist":!0,"tagline":!1},"wgWMESchemaEditAttemptStepOversample":!1,"wgULSCurrentAutonym":"français","wgNoticeProject":"wiktionary","wgCentralAuthMobileDomain":!1,"wgEditSubmitButtonLabelPublish":!0};RLSTATE={"ext.gadget.StyleWiktionnaire":"ready","ext.globalCssJs.user.styles":"ready","site.styles":"ready","noscript":"ready",
  9 | "user.styles":"ready","ext.globalCssJs.user":"ready","user":"ready","user.options":"loading","mediawiki.toc.styles":"ready","skins.vector.styles.legacy":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready","ext.uls.interlanguage":"ready","ext.wikimediaBadges":"ready"};RLPAGEMODULES=["site","mediawiki.page.startup","skins.vector.js","mediawiki.page.ready","mediawiki.toc","ext.gadget.SommaireDeveloppable","ext.gadget.anagrimes","ext.gadget.ChercheDansSousCategories","ext.gadget.clearer-edit-summary","ext.gadget.translation_editor","ext.gadget.Wiktionnaire","ext.gadget.LiensAncresDansCategories","ext.centralauth.centralautologin","mmv.head","mmv.bootstrap.autostart","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader","ext.eventLogging","ext.wikimediaEvents","ext.navigationTiming","ext.uls.compactlinks","ext.uls.interface","ext.centralNotice.geoIP","ext.centralNotice.startUp"];</script>
 10 | <script>(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@1hzgi",function($,jQuery,require,module){/*@nomin*/mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"});mw.user.options.set({"variant":"fr"});
 11 | });});</script>
 12 | <link rel="stylesheet" href="/w/load.php?lang=fr&amp;modules=ext.uls.interlanguage%7Cext.visualEditor.desktopArticleTarget.noscript%7Cext.wikimediaBadges%7Cmediawiki.toc.styles%7Cskins.vector.styles.legacy&amp;only=styles&amp;skin=vector"/>
 13 | <script async="" src="/w/load.php?lang=fr&amp;modules=startup&amp;only=scripts&amp;raw=1&amp;skin=vector"></script>
 14 | <meta name="ResourceLoaderDynamicStyles" content=""/>
 15 | <link rel="stylesheet" href="/w/load.php?lang=fr&amp;modules=ext.gadget.StyleWiktionnaire&amp;only=styles&amp;skin=vector"/>
 16 | <link rel="stylesheet" href="/w/load.php?lang=fr&amp;modules=site.styles&amp;only=styles&amp;skin=vector"/>
 17 | <meta name="generator" content="MediaWiki 1.35.0-wmf.30"/>
 18 | <meta name="referrer" content="origin"/>
 19 | <meta name="referrer" content="origin-when-crossorigin"/>
 20 | <meta name="referrer" content="origin-when-cross-origin"/>
 21 | <meta property="og:image" content="https://upload.wikimedia.org/wikipedia/commons/c/c2/Rainbow_lorikeet.jpg"/>
 22 | <link rel="alternate" type="application/x-wiki" title="Modifier" href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit"/>
 23 | <link rel="edit" title="Modifier" href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit"/>
 24 | <link rel="apple-touch-icon" href="/static/apple-touch/wiktionary.png"/>
 25 | <link rel="shortcut icon" href="/static/favicon/piece.ico"/>
 26 | <link rel="search" type="application/opensearchdescription+xml" href="/w/opensearch_desc.php" title="Wiktionnaire (fr)"/>
 27 | <link rel="EditURI" type="application/rsd+xml" href="//fr.wiktionary.org/w/api.php?action=rsd"/>
 28 | <link rel="license" href="//creativecommons.org/licenses/by-sa/3.0/"/>
 29 | <link rel="alternate" type="application/atom+xml" title="Flux Atom de Wiktionnaire" href="/w/index.php?title=Sp%C3%A9cial:Modifications_r%C3%A9centes&amp;feed=atom"/>
 30 | <link rel="canonical" href="https://fr.wiktionary.org/wiki/loriquet_arc-en-ciel"/>
 31 | <link rel="dns-prefetch" href="//login.wikimedia.org"/>
 32 | <link rel="dns-prefetch" href="//meta.wikimedia.org" />
 33 | <!--[if lt IE 9]><script src="/w/resources/lib/html5shiv/html5shiv.js"></script><![endif]-->
 34 | </head>
 35 | <body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject mw-editable page-loriquet_arc-en-ciel rootpage-loriquet_arc-en-ciel skin-vector action-view">
 36 | <div id="mw-page-base" class="noprint"></div>
 37 | <div id="mw-head-base" class="noprint"></div>
 38 | <div id="content" class="mw-body" role="main">
 39 | 	<a id="top"></a>
 40 | 		<div id="siteNotice" class="mw-body-content"><!-- CentralNotice --></div>
 41 | 	<div class="mw-indicators mw-body-content">
 42 | </div>
 43 | 
 44 | 	<h1 id="firstHeading" class="firstHeading" lang="fr">loriquet arc-en-ciel</h1>
 45 | 	
 46 | 	<div id="bodyContent" class="mw-body-content">
 47 | 		<div id="siteSub" class="noprint">Définition, traduction, prononciation, anagramme et synonyme sur le dictionnaire libre Wiktionnaire.</div>
 48 | 		<div id="contentSub"></div>
 49 | 		
 50 | 		
 51 | 		<div id="jump-to-nav"></div>
 52 | 		<a class="mw-jump-link" href="#mw-head">Sauter à la navigation</a>
 53 | 		<a class="mw-jump-link" href="#p-search">Sauter à la recherche</a>
 54 | 		<div id="mw-content-text" lang="fr" dir="ltr" class="mw-content-ltr"><div class="mw-parser-output"><div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="fr" dir="ltr"><h2 id="mw-toc-heading">Sommaire</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
 55 | <ul>
 56 | <li class="toclevel-1 tocsection-1"><a href="#Français"><span class="tocnumber">1</span> <span class="toctext"><span>Français</span></span></a>
 57 | <ul>
 58 | <li class="toclevel-2 tocsection-2"><a href="#Étymologie"><span class="tocnumber">1.1</span> <span class="toctext"><span>Étymologie</span></span></a></li>
 59 | <li class="toclevel-2 tocsection-3"><a href="#Locution_nominale"><span class="tocnumber">1.2</span> <span class="toctext"><span>Locution nominale</span><span> </span></span></a>
 60 | <ul>
 61 | <li class="toclevel-3 tocsection-4"><a href="#Notes"><span class="tocnumber">1.2.1</span> <span class="toctext"><span>Notes</span></span></a></li>
 62 | <li class="toclevel-3 tocsection-5"><a href="#Synonymes"><span class="tocnumber">1.2.2</span> <span class="toctext"><span>Synonymes</span></span></a></li>
 63 | <li class="toclevel-3 tocsection-6"><a href="#Hyperonymes"><span class="tocnumber">1.2.3</span> <span class="toctext"><span>Hyperonymes</span></span></a></li>
 64 | <li class="toclevel-3 tocsection-7"><a href="#Traductions"><span class="tocnumber">1.2.4</span> <span class="toctext"><span>Traductions</span></span></a></li>
 65 | </ul>
 66 | </li>
 67 | <li class="toclevel-2 tocsection-8"><a href="#Voir_aussi"><span class="tocnumber">1.3</span> <span class="toctext"><span>Voir aussi</span></span></a></li>
 68 | </ul>
 69 | </li>
 70 | </ul>
 71 | </div>
 72 | 
 73 | <h2><span id="Fran.C3.A7ais"></span><span class="mw-headline" id="Français"><span class="sectionlangue" id="fr"><a href="/wiki/Portail:Fran%C3%A7ais" title="Portail:Français">Français</a></span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=1" title="Modifier la section : Français">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h2>
 74 | <h3><span id=".C3.89tymologie"></span><span class="mw-headline" id="Étymologie"><span class="titreetym" title="">Étymologie</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=2" title="Modifier la section : Étymologie">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h3>
 75 | <dl><dd>De <i><a href="/wiki/loriquet" title="loriquet">loriquet</a></i> et <i><a href="/wiki/arc-en-ciel" title="arc-en-ciel">arc-en-ciel</a></i>.</dd></dl>
 76 | <h3><span class="mw-headline" id="Locution_nominale"><span class="titredef" id="fr-nom-1">Locution nominale</span><span id="fr-nom" style="font-size:0;"> </span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=3" title="Modifier la section : Locution nominale">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h3>
 77 | <table class="flextable flextable-fr-mfsp">
 78 | 
 79 | 
 80 | <tbody><tr>
 81 | <th>Singulier</th>
 82 | <th>Pluriel
 83 | </th></tr>
 84 | 
 85 | <tr>
 86 | <td><span lang="fr" class="lang-fr"><a class="mw-selflink selflink">loriquet arc-en-ciel</a></span>
 87 | </td>
 88 | <td><span lang="fr" class="lang-fr"><a href="/wiki/loriquets_arc-en-ciel#fr" title="loriquets arc-en-ciel">loriquets arc-en-ciel</a></span>
 89 | </td></tr>
 90 | <tr>
 91 | <td colspan="2"><a href="/wiki/Annexe:Prononciation/fran%C3%A7ais" title="Annexe:Prononciation/français"><span class="API" title="Prononciation API">\lɔ.ʁi.kɛ aʁ.kɑ̃.sjɛl\</span></a>
 92 | </td></tr></tbody></table>
 93 | <div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Fichier:Rainbow_lorikeet.jpg" class="image"><img alt="" src="//upload.wikimedia.org/wikipedia/commons/thumb/c/c2/Rainbow_lorikeet.jpg/180px-Rainbow_lorikeet.jpg" decoding="async" width="180" height="270" class="thumbimage" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/c/c2/Rainbow_lorikeet.jpg/270px-Rainbow_lorikeet.jpg 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c2/Rainbow_lorikeet.jpg/360px-Rainbow_lorikeet.jpg 2x" data-file-width="1067" data-file-height="1600" /></a>  <div class="thumbcaption"><div class="magnify"><a href="/wiki/Fichier:Rainbow_lorikeet.jpg" class="internal" title="Agrandir"></a></div>Un <b>loriquet arc-en-ciel</b>.</div></div></div>
 94 | <p><b>loriquet arc-en-ciel</b> <a href="/wiki/Annexe:Prononciation/fran%C3%A7ais" title="Annexe:Prononciation/français"><span class="API" title="Prononciation API">\lɔ.ʁi.kɛ aʁ.kɑ̃.sjɛl\</span></a> <span class="ligne-de-forme"><i>masculin</i></span>
 95 | </p>
 96 | <ol><li><span class="term" id="ornithologie"><i>(<span class="texte">Ornithologie</span>)</i></span> <a href="/wiki/perroquet" title="perroquet">Perroquet</a> de taille modeste, vivant dans l'est de l'<a href="/wiki/Australie" title="Australie">Australie</a>, très proche du <a href="/wiki/loriquet_%C3%A0_t%C3%AAte_bleue" title="loriquet à tête bleue">loriquet à tête bleue</a>, dont il est parfois considéré comme une sous-espèce.
 97 | <ul><li><i>Des <b>loriquets arc-en-ciel</b> ont été introduits à Auckland (<a href="/wiki/Nouvelle-Z%C3%A9lande" title="Nouvelle-Zélande">Nouvelle-Zélande</a>) et à <a href="/wiki/Hong-Kong" title="Hong-Kong">Hong-Kong</a>.</i></li>
 98 | <li><i>La semaine dernière, enfin, trois <b>loriquets arc-en-ciel</b>, l’espèce la plus bruyante et colorée de perroquets, ont réussi à prendre la fuite.</i> <span class="sources"><span class="tiret">—&#160;</span>(<i>Les animaux du zoo de Bristol s’entretuent-ils?</i>, in <i>20 minutes</i>, 9 février 2015)</span></li></ul></li></ol>
 99 | <h4><span class="mw-headline" id="Notes"><span class="" title="">Notes</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=4" title="Modifier la section : Notes">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h4>
100 | <ul><li>Peut être utilisé avec une majuscule <b>(Loriquet arc-en-ciel)</b> pour mettre en avant le fait qu’on donne un caractère générique au mot.</li></ul>
101 | <h4><span class="mw-headline" id="Synonymes"><span class="titresyno" title="">Synonymes</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=5" title="Modifier la section : Synonymes">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h4>
102 | <ul><li><a href="/wiki/loriquet_de_Swainson" title="loriquet de Swainson">loriquet de Swainson</a></li></ul>
103 | <h4><span class="mw-headline" id="Hyperonymes"><span class="titrehyper" title="Mots dont le sens est plus générique que «&#32;loriquet arc-en-ciel&#32;»">Hyperonymes</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=6" title="Modifier la section : Hyperonymes">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h4>
104 | <p>(simplifié)
105 | </p>
106 | <ul><li><a href="/wiki/oiseau" title="oiseau">oiseau</a> (Aves)
107 | <ul><li><a href="/wiki/psittaciforme" title="psittaciforme">psittaciforme</a>
108 | <ul><li><a href="/wiki/psittacid%C3%A9" title="psittacidé">psittacidé</a> (Psittacidae)</li></ul></li></ul></li></ul>
109 | <h4><span class="mw-headline" id="Traductions"><span class="titretrad" title="">Traductions</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=7" title="Modifier la section : Traductions">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h4>
110 | <div class="boite">
111 | <div class="NavFrame">
112 | <div class="NavHead"><b><i></i></b></div>
113 | <div class="NavContent">
114 | <div style="column-width: 26em; -webkit-column-width: 26em; -moz-column-width: 26em; vertical-align: top; text-align: left;"><div class="translations">
115 | <ul><li><span class="trad-conv">Conventions internationales</span>&#160;: <i><a href="/w/index.php?title=Trichoglossus_moluccanus&amp;action=edit&amp;redlink=1" class="new" title="Trichoglossus moluccanus (page inexistante)"><span lang="conv" class="lang-conv">Trichoglossus moluccanus</span></a>&#160;<span class="trad-exposant"><a href="https://species.wikimedia.org/wiki/Trichoglossus_moluccanus" class="extiw" title="wikispecies:Trichoglossus moluccanus"><span class="trad-existe">(wikispecies)</span></a></span></i></li>
116 | <li><span class="trad-de">Allemand</span>&#160;: <a href="/w/index.php?title=Regenbogenlori&amp;action=edit&amp;redlink=1" class="new" title="Regenbogenlori (page inexistante)"><span lang="de" class="lang-de">Regenbogenlori</span></a>&#160;<span class="trad-exposant"><a href="https://de.wiktionary.org/wiki/Regenbogenlori" class="extiw" title="de:Regenbogenlori"><span class="trad-absent">(de)</span></a></span> <i>masculin</i>, <a href="/w/index.php?title=Keilschwanzlori&amp;action=edit&amp;redlink=1" class="new" title="Keilschwanzlori (page inexistante)"><span lang="de" class="lang-de">Keilschwanzlori</span></a>&#160;<span class="trad-exposant"><a href="https://de.wiktionary.org/wiki/Keilschwanzlori" class="extiw" title="de:Keilschwanzlori"><span class="trad-absent">(de)</span></a></span> <i>masculin</i></li>
117 | <li><span class="trad-en">Anglais</span>&#160;: <a href="/w/index.php?title=rainbow_lorikeet&amp;action=edit&amp;redlink=1" class="new" title="rainbow lorikeet (page inexistante)"><span lang="en" class="lang-en">rainbow lorikeet</span></a>&#160;<span class="trad-exposant"><a href="https://en.wiktionary.org/wiki/rainbow_lorikeet" class="extiw" title="en:rainbow lorikeet"><span class="trad-absent">(en)</span></a></span></li>
118 | <li><span class="trad-ca">Catalan</span>&#160;: <a href="/w/index.php?title=lori_arc_iris&amp;action=edit&amp;redlink=1" class="new" title="lori arc iris (page inexistante)"><span lang="ca" class="lang-ca">lori arc iris</span></a>&#160;<span class="trad-exposant"><a href="https://ca.wiktionary.org/wiki/lori_arc_iris" class="extiw" title="ca:lori arc iris"><span class="trad-absent">(ca)</span></a></span> <i>masculin</i></li>
119 | <li><span class="trad-da">Danois</span>&#160;: <a href="/w/index.php?title=regnbuelori&amp;action=edit&amp;redlink=1" class="new" title="regnbuelori (page inexistante)"><span lang="da" class="lang-da">regnbuelori</span></a>&#160;<span class="trad-exposant"><a href="https://da.wiktionary.org/wiki/regnbuelori" class="extiw" title="da:regnbuelori"><span class="trad-absent">(da)</span></a></span> <i>commun</i></li>
120 | <li><span class="trad-es">Espagnol</span>&#160;: <a href="/w/index.php?title=lori_arcoiris&amp;action=edit&amp;redlink=1" class="new" title="lori arcoiris (page inexistante)"><span lang="es" class="lang-es">lori arcoiris</span></a>&#160;<span class="trad-exposant"><a href="https://es.wiktionary.org/wiki/lori_arcoiris" class="extiw" title="es:lori arcoiris"><span class="trad-absent">(es)</span></a></span> <i>masculin</i></li>
121 | <li><span class="trad-it">Italien</span>&#160;: <a href="/w/index.php?title=lorichetto_di_Swainson&amp;action=edit&amp;redlink=1" class="new" title="lorichetto di Swainson (page inexistante)"><span lang="it" class="lang-it">lorichetto di Swainson</span></a>&#160;<span class="trad-exposant"><a href="https://it.wiktionary.org/wiki/lorichetto_di_Swainson" class="extiw" title="it:lorichetto di Swainson"><span class="trad-absent">(it)</span></a></span> <i>masculin</i></li>
122 | <li><span class="trad-nl">Néerlandais</span>&#160;: <a href="/w/index.php?title=lori_van_de_Blauwe_Bergen&amp;action=edit&amp;redlink=1" class="new" title="lori van de Blauwe Bergen (page inexistante)"><span lang="nl" class="lang-nl">lori van de Blauwe Bergen</span></a>&#160;<span class="trad-exposant"><a href="https://nl.wiktionary.org/wiki/lori_van_de_Blauwe_Bergen" class="extiw" title="nl:lori van de Blauwe Bergen"><span class="trad-absent">(nl)</span></a></span></li>
123 | <li><span class="trad-pl">Polonais</span>&#160;: <a href="/w/index.php?title=lorysa_niebieskobrzucha&amp;action=edit&amp;redlink=1" class="new" title="lorysa niebieskobrzucha (page inexistante)"><span lang="pl" class="lang-pl">lorysa niebieskobrzucha</span></a>&#160;<span class="trad-exposant"><a href="https://pl.wiktionary.org/wiki/lorysa_niebieskobrzucha" class="extiw" title="pl:lorysa niebieskobrzucha"><span class="trad-absent">(pl)</span></a></span> <i>féminin</i></li>
124 | <li><span class="trad-pt">Portugais</span>&#160;: <a href="/w/index.php?title=l%C3%B3ris-molucano&amp;action=edit&amp;redlink=1" class="new" title="lóris-molucano (page inexistante)"><span lang="pt" class="lang-pt">lóris-molucano</span></a>&#160;<span class="trad-exposant"><a href="https://pt.wiktionary.org/wiki/l%C3%B3ris-molucano" class="extiw" title="pt:lóris-molucano"><span class="trad-absent">(pt)</span></a></span> <i>masculin</i></li>
125 | <li><span class="trad-sk">Slovaque</span>&#160;: <a href="/w/index.php?title=lori_modrohlav%C3%BD&amp;action=edit&amp;redlink=1" class="new" title="lori modrohlavý (page inexistante)"><span lang="sk" class="lang-sk">lori modrohlavý</span></a>&#160;<span class="trad-exposant"><a href="https://sk.wiktionary.org/wiki/lori_modrohlav%C3%BD" class="extiw" title="sk:lori modrohlavý"><span class="trad-absent">(sk)</span></a></span></li>
126 | <li><span class="trad-sv">Suédois</span>&#160;: <a href="/w/index.php?title=regnb%C3%A5gslorikit&amp;action=edit&amp;redlink=1" class="new" title="regnbågslorikit (page inexistante)"><span lang="sv" class="lang-sv">regnbågslorikit</span></a>&#160;<span class="trad-exposant"><a href="https://sv.wiktionary.org/wiki/regnb%C3%A5gslorikit" class="extiw" title="sv:regnbågslorikit"><span class="trad-absent">(sv)</span></a></span> <i>commun</i></li></ul>
127 | </div></div></div><div style="clear:both"></div></div></div>
128 | <h3><span class="mw-headline" id="Voir_aussi"><span class="titrevoir" title="">Voir aussi</span></span><span class="mw-editsection"><span class="mw-editsection-bracket">[</span><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit&amp;section=8" title="Modifier la section : Voir aussi">modifier le wikicode</a><span class="mw-editsection-bracket">]</span></span></h3>
129 | <ul><li><span lang="fr" class="lang-fr"><a href="https://fr.wikipedia.org/wiki/loriquet_arc-en-ciel" class="extiw" title="w:loriquet arc-en-ciel">loriquet arc-en-ciel</a></span> sur l’encyclopédie Wikipédia <img alt="Wikipedia-logo-v2.svg" src="//upload.wikimedia.org/wikipedia/commons/thumb/8/80/Wikipedia-logo-v2.svg/20px-Wikipedia-logo-v2.svg.png" decoding="async" width="20" height="18" srcset="//upload.wikimedia.org/wikipedia/commons/thumb/8/80/Wikipedia-logo-v2.svg/30px-Wikipedia-logo-v2.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/80/Wikipedia-logo-v2.svg/40px-Wikipedia-logo-v2.svg.png 2x" data-file-width="103" data-file-height="94" /></li></ul>
130 | <!-- 
131 | NewPP limit report
132 | Parsed by mw1325
133 | Cached time: 20200505225155
134 | Cache expiry: 2592000
135 | Dynamic content: false
136 | Complications: []
137 | CPU time usage: 0.212 seconds
138 | Real time usage: 0.292 seconds
139 | Preprocessor visited node count: 873/1000000
140 | Post‐expand include size: 12556/2097152 bytes
141 | Template argument size: 816/2097152 bytes
142 | Highest expansion depth: 12/40
143 | Expensive parser function count: 2/500
144 | Unstrip recursion depth: 0/20
145 | Unstrip post‐expand size: 0/5000000 bytes
146 | Number of Wikibase entities loaded: 0/400
147 | Lua time usage: 0.097/10.000 seconds
148 | Lua memory usage: 6.37 MB/50 MB
149 | -->
150 | <!--
151 | Transclusion expansion time report (%,ms,calls,template)
152 | 100.00%  260.310      1 -total
153 |  22.31%   58.066      1 Modèle:langue
154 |  18.47%   48.089      1 Modèle:fr-rég
155 |  16.07%   41.844      7 Modèle:S
156 |   8.38%   21.802     12 Modèle:T
157 |   7.76%   20.201      3 Modèle:Lang
158 |   7.50%   19.531     12 Modèle:trad-
159 |   6.70%   17.453      1 Modèle:WP
160 |   6.68%   17.399      2 Modèle:pron
161 |   6.58%   17.121      2 Modèle:lien_m
162 | -->
163 | 
164 | <!-- Saved in parser cache with key frwiktionary:pcache:idhash:3228278-0!canonical and timestamp 20200505225155 and revision id 25708973
165 |  -->
166 | </div><noscript><img src="//fr.wiktionary.org/wiki/Special:CentralAutoLogin/start?type=1x1" alt="" title="" width="1" height="1" style="border: none; position: absolute;" /></noscript></div>
167 | 		<div class="printfooter">Récupérée de «&#160;<a dir="ltr" href="https://fr.wiktionary.org/w/index.php?title=loriquet_arc-en-ciel&amp;oldid=25708973">https://fr.wiktionary.org/w/index.php?title=loriquet_arc-en-ciel&amp;oldid=25708973</a>&#160;»</div>
168 | 		<div id="catlinks" class="catlinks" data-mw="interface"><div id="mw-normal-catlinks" class="mw-normal-catlinks"><a href="/wiki/Sp%C3%A9cial:Cat%C3%A9gories" title="Spécial:Catégories">Catégories</a> : <ul><li><a href="/wiki/Cat%C3%A9gorie:fran%C3%A7ais" title="Catégorie:français">français</a></li><li><a href="/wiki/Cat%C3%A9gorie:Locutions_nominales_en_fran%C3%A7ais" title="Catégorie:Locutions nominales en français">Locutions nominales en français</a></li><li><a href="/wiki/Cat%C3%A9gorie:Perroquets_et_perruches_en_fran%C3%A7ais" title="Catégorie:Perroquets et perruches en français">Perroquets et perruches en français</a></li></ul></div><div id="mw-hidden-catlinks" class="mw-hidden-catlinks mw-hidden-cats-hidden">Catégories cachées : <ul><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_conventions_internationales" title="Catégorie:Traductions en conventions internationales">Traductions en conventions internationales</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_allemand" title="Catégorie:Traductions en allemand">Traductions en allemand</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_anglais" title="Catégorie:Traductions en anglais">Traductions en anglais</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_catalan" title="Catégorie:Traductions en catalan">Traductions en catalan</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_danois" title="Catégorie:Traductions en danois">Traductions en danois</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_espagnol" title="Catégorie:Traductions en espagnol">Traductions en espagnol</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_italien" title="Catégorie:Traductions en italien">Traductions en italien</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_n%C3%A9erlandais" title="Catégorie:Traductions en néerlandais">Traductions en néerlandais</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_polonais" title="Catégorie:Traductions en polonais">Traductions en polonais</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_portugais" title="Catégorie:Traductions en portugais">Traductions en portugais</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_slovaque" title="Catégorie:Traductions en slovaque">Traductions en slovaque</a></li><li><a href="/wiki/Cat%C3%A9gorie:Traductions_en_su%C3%A9dois" title="Catégorie:Traductions en suédois">Traductions en suédois</a></li><li><a href="/wiki/Cat%C3%A9gorie:Pages_li%C3%A9es_%C3%A0_Wikip%C3%A9dia_en_fran%C3%A7ais" title="Catégorie:Pages liées à Wikipédia en français">Pages liées à Wikipédia en français</a></li></ul></div></div>
169 | 		<div class="visualClear"></div>
170 | 		
171 | 	</div>
172 | </div>
173 | 
174 | <div id="mw-navigation">
175 | 	<h2>Menu de navigation</h2>
176 | 	<div id="mw-head">
177 | 		
178 | <div id="p-personal" role="navigation" class="" aria-labelledby="p-personal-label">
179 | 	<h3 id="p-personal-label">Outils personnels</h3>
180 | 	<ul >
181 | 		<li id="pt-anonuserpage">Non connecté</li><li id="pt-anontalk"><a href="/wiki/Sp%C3%A9cial:Mes_discussions" title="La page de discussion pour les contributions depuis cette adresse IP [n]" accesskey="n">Discussion</a></li><li id="pt-anoncontribs"><a href="/wiki/Sp%C3%A9cial:Mes_contributions" title="Une liste des modifications effectuées depuis cette adresse IP [y]" accesskey="y">Contributions</a></li><li id="pt-createaccount"><a href="/w/index.php?title=Sp%C3%A9cial:Cr%C3%A9er_un_compte&amp;returnto=loriquet+arc-en-ciel" title="Nous vous encourageons à créer un compte utilisateur et vous connecter ; ce n’est cependant pas obligatoire.">Créer un compte</a></li><li id="pt-login"><a href="/w/index.php?title=Sp%C3%A9cial:Connexion&amp;returnto=loriquet+arc-en-ciel" title="Nous vous encourageons à vous connecter ; ce n’est cependant pas obligatoire. [o]" accesskey="o">Se connecter</a></li>
182 | 	</ul>
183 | </div>
184 | 
185 | 		<div id="left-navigation">
186 | 			<div id="p-namespaces" role="navigation" class="vectorTabs" aria-labelledby="p-namespaces-label">
187 | 	<h3 id="p-namespaces-label">Espaces de noms</h3>
188 | 	<ul >
189 | 		<li id="ca-nstab-main" class="selected"><a href="/wiki/loriquet_arc-en-ciel" title="Voir le contenu de la page [c]" accesskey="c">Article</a></li><li id="ca-talk" class="new"><a href="/w/index.php?title=Discussion:loriquet_arc-en-ciel&amp;action=edit&amp;redlink=1" rel="discussion" title="Discussion au sujet de cette page de contenu (page inexistante) [t]" accesskey="t">Discussion</a></li>
190 | 	</ul>
191 | </div>
192 | 
193 | 			<div id="p-variants" role="navigation" class="emptyPortlet vectorMenu" aria-labelledby="p-variants-label">
194 | 	<input type="checkbox" class="vectorMenuCheckbox" aria-labelledby="p-variants-label" />
195 | 	<h3 id="p-variants-label">
196 | 		<span>Variantes</span>
197 | 	</h3>
198 | 	<ul class="menu" >
199 | 		
200 | 	</ul>
201 | </div>
202 | 
203 | 		</div>
204 | 		<div id="right-navigation">
205 | 			<div id="p-views" role="navigation" class="vectorTabs" aria-labelledby="p-views-label">
206 | 	<h3 id="p-views-label">Affichages</h3>
207 | 	<ul >
208 | 		<li id="ca-view" class="collapsible selected"><a href="/wiki/loriquet_arc-en-ciel">Lire</a></li><li id="ca-edit" class="collapsible"><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=edit" title="Modifier le wikicode [e]" accesskey="e">Modifier le wikicode</a></li><li id="ca-history" class="collapsible"><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=history" title="Historique des versions de cette page [h]" accesskey="h">Voir l’historique</a></li>
209 | 	</ul>
210 | </div>
211 | 
212 | 			<div id="p-cactions" role="navigation" class="emptyPortlet vectorMenu" aria-labelledby="p-cactions-label">
213 | 	<input type="checkbox" class="vectorMenuCheckbox" aria-labelledby="p-cactions-label" />
214 | 	<h3 id="p-cactions-label">
215 | 		<span>Plus</span>
216 | 	</h3>
217 | 	<ul class="menu" >
218 | 		
219 | 	</ul>
220 | </div>
221 | 
222 | 			<div id="p-search" role="search">
223 | 	<h3 >
224 | 		<label for="searchInput">Rechercher</label>
225 | 	</h3>
226 | 	<form action="/w/index.php" id="searchform">
227 | 		<div id="simpleSearch">
228 | 			<input type="search" name="search" placeholder="Rechercher dans le Wiktionnaire" title="Rechercher dans le Wiktionnaire [f]" accesskey="f" id="searchInput"/>
229 | 			<input type="hidden" value="Spécial:Recherche" name="title"/>
230 | 			<input type="submit" name="fulltext" value="Rechercher" title="Rechercher les pages comportant ce texte." id="mw-searchButton" class="searchButton mw-fallbackSearchButton"/>
231 | 			<input type="submit" name="go" value="Continuer" title="Accédez à une page du même nom si elle existe" id="searchButton" class="searchButton"/>
232 | 		</div>
233 | 	</form>
234 | </div>
235 | 
236 | 		</div>
237 | 	</div>
238 | 	
239 | <div id="mw-panel">
240 | 	<div id="p-logo" role="banner">
241 | 		<a  title="Visiter la page d’accueil" class="mw-wiki-logo" href="/wiki/Wiktionnaire:Page_d%E2%80%99accueil"></a>
242 | 	</div>
243 | 	<div class="portal portal-first" role="navigation" id="p-Navigation"  aria-labelledby="p-Navigation-label">
244 | 	<h3  id="p-Navigation-label">
245 | 		Navigation
246 | 	</h3>
247 | 	<div class="body">
248 | 		<ul><li id="n-mainpage-name"><a href="/wiki/Wiktionnaire:Page_d%E2%80%99accueil">Page d’accueil</a></li><li id="n-Recherche-avancée,-anagrammes-et-rimes"><a href="/wiki/Wiktionnaire:Recherche_avanc%C3%A9e">Recherche avancée, anagrammes et rimes</a></li><li id="n-thema"><a href="/wiki/Wiktionnaire:Portails">Portails</a></li><li id="n-randompage"><a href="/wiki/Sp%C3%A9cial:Page_au_hasard" title="Afficher une page au hasard [x]" accesskey="x">Page au hasard</a></li><li id="n-randompagebylang"><a href="/wiki/Wiktionnaire:Page_au_hasard">Page au hasard par langue</a></li><li id="n-Questions-sur-les-mots"><a href="/wiki/Wiktionnaire:Questions_sur_les_mots">Poser une question</a></li></ul>
249 | 		
250 | 	</div>
251 | </div>
252 | 
253 | 
254 | 	<div class="portal" role="navigation" id="p-Contribuer"  aria-labelledby="p-Contribuer-label">
255 | 	<h3  id="p-Contribuer-label">
256 | 		Contribuer
257 | 	</h3>
258 | 	<div class="body">
259 | 		<ul><li id="n-journal"><a href="/wiki/Wiktionnaire:Journal_des_contributeurs">Journal des contributeurs</a></li><li id="n-La-Wikidémie"><a href="/wiki/Wiktionnaire:Wikid%C3%A9mie">La Wikidémie</a></li><li id="n-portal"><a href="/wiki/Wiktionnaire:Accueil_communautaire" title="À propos du projet, ce que vous pouvez faire, où trouver les informations">Communauté</a></li><li id="n-Discuter-sur-Discord"><a href="/wiki/Wiktionnaire:Discord">Discuter sur Discord</a></li><li id="n-recentchanges"><a href="/wiki/Sp%C3%A9cial:Modifications_r%C3%A9centes" title="Liste des modifications récentes sur le wiki [r]" accesskey="r">Modifications récentes</a></li><li id="n-sitesupport"><a href="//donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&amp;utm_medium=sidebar&amp;utm_campaign=C13_fr.wiktionary.org&amp;uselang=fr" title="Soutenez-nous">Faire un don</a></li></ul>
260 | 		
261 | 	</div>
262 | </div>
263 | 
264 | <div class="portal" role="navigation" id="p-Aide"  aria-labelledby="p-Aide-label">
265 | 	<h3  id="p-Aide-label">
266 | 		Aide
267 | 	</h3>
268 | 	<div class="body">
269 | 		<ul><li id="n-Forum-d’entraide"><a href="/wiki/Wiktionnaire:Entraide">Forum d’entraide</a></li><li id="n-help"><a href="/wiki/Aide:Sommaire" title="Accès à l’aide">Aide</a></li><li id="n-Modèles"><a href="/wiki/Wiktionnaire:Mod%C3%A8les" title="Modèles">Modèles</a></li><li id="n-Conventions"><a href="/wiki/Wiktionnaire:Conventions">Conventions</a></li><li id="n-Création-rapide"><a href="/wiki/Aide:Cr%C3%A9ation_rapide">Créer un article</a></li><li id="n-Télécharger"><a href="/wiki/Wiktionnaire:Sites_miroirs#Avec_logiciel_à_télécharger">Télécharger</a></li></ul>
270 | 		
271 | 	</div>
272 | </div>
273 | 
274 | <div class="portal" role="navigation" id="p-coll-print_export"  aria-labelledby="p-coll-print_export-label">
275 | 	<h3  id="p-coll-print_export-label">
276 | 		Imprimer / exporter
277 | 	</h3>
278 | 	<div class="body">
279 | 		<ul><li id="coll-create_a_book"><a href="/w/index.php?title=Sp%C3%A9cial:Livre&amp;bookcmd=book_creator&amp;referer=loriquet+arc-en-ciel">Créer un livre</a></li><li id="coll-download-as-rl"><a href="/w/index.php?title=Sp%C3%A9cial:ElectronPdf&amp;page=loriquet+arc-en-ciel&amp;action=show-download-screen">Télécharger comme PDF</a></li><li id="t-print"><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;printable=yes" title="Version imprimable de cette page [p]" accesskey="p">Version imprimable</a></li></ul>
280 | 		
281 | 	</div>
282 | </div>
283 | 
284 | <div class="portal" role="navigation" id="p-tb"  aria-labelledby="p-tb-label">
285 | 	<h3  id="p-tb-label">
286 | 		Outils
287 | 	</h3>
288 | 	<div class="body">
289 | 		<ul><li id="t-whatlinkshere"><a href="/wiki/Sp%C3%A9cial:Pages_li%C3%A9es/loriquet_arc-en-ciel" title="Liste des pages liées qui pointent sur celle-ci [j]" accesskey="j">Pages liées</a></li><li id="t-recentchangeslinked"><a href="/wiki/Sp%C3%A9cial:Suivi_des_liens/loriquet_arc-en-ciel" rel="nofollow" title="Liste des modifications récentes des pages appelées par celle-ci [k]" accesskey="k">Suivi des pages liées</a></li><li id="t-upload"><a href="//commons.wikimedia.org/wiki/Special:UploadWizard?uselang=fr" title="Téléverser une image ou fichier média sur le serveur [u]" accesskey="u">Téléverser un fichier</a></li><li id="t-specialpages"><a href="/wiki/Sp%C3%A9cial:Pages_sp%C3%A9ciales" title="Liste de toutes les pages spéciales [q]" accesskey="q">Pages spéciales</a></li><li id="t-permalink"><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;oldid=25708973" title="Adresse permanente de cette version de la page">Lien permanent</a></li><li id="t-info"><a href="/w/index.php?title=loriquet_arc-en-ciel&amp;action=info" title="Plus d’informations sur cette page">Informations sur la page</a></li><li id="t-cite"><a href="/w/index.php?title=Sp%C3%A9cial:Citer&amp;page=loriquet_arc-en-ciel&amp;id=25708973&amp;wpFormIdentifier=titleform" title="Informations sur la manière de citer cette page">Citer cette page</a></li></ul>
290 | 		
291 | 	</div>
292 | </div>
293 | 
294 | <div class="portal" role="navigation" id="p-lang"  aria-labelledby="p-lang-label">
295 | 	<h3  id="p-lang-label">
296 | 		Dans d’autres langues
297 | 	</h3>
298 | 	<div class="body">
299 | 		<ul></ul>
300 | 		
301 | 	</div>
302 | </div>
303 | 
304 | 
305 | </div>
306 | 
307 | </div>
308 | 
309 | <div id="footer" class="mw-footer" role="contentinfo" >
310 | 	<ul id="footer-info" class="">
311 | 		<li id="footer-info-lastmod"> La dernière modification de cette page a été faite le 24 octobre 2018 à 05:43.</li>
312 | 		<li id="footer-info-copyright">Les textes sont disponibles sous <a href="//creativecommons.org/licenses/by-sa/3.0/deed.fr">licence Creative Commons attribution partage à l’identique</a> ; d’autres termes peuvent s’appliquer. Voyez les <a href="//wikimediafoundation.org/wiki/Conditions_d'utilisation">termes d’utilisation</a> pour plus de détails.<br/>Pour les illustrations, cliquez sur chaque image ou consultez les <a class="internal" href="//fr.wiktionary.org/wiki/Wiktionnaire:Cr%C3%A9dits_graphiques" title="Droit d'auteur de certaines icônes">crédits graphiques</a>.<br/></li>
313 | 	</ul>
314 | 	<ul id="footer-places" class="">
315 | 		<li id="footer-places-privacy"><a href="https://meta.wikimedia.org/wiki/Privacy_policy/fr" class="extiw" title="m:Privacy policy/fr">Politique de confidentialité</a></li>
316 | 		<li id="footer-places-about"><a href="/wiki/Wiktionnaire:%C3%80_propos_de" class="mw-redirect" title="Wiktionnaire:À propos de">À propos du Wiktionnaire</a></li>
317 | 		<li id="footer-places-disclaimer"><a href="/wiki/Wiktionnaire:Licence" title="Wiktionnaire:Licence">Licence</a></li>
318 | 		<li id="footer-places-developers"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute">Développeurs</a></li>
319 | 		<li id="footer-places-statslink"><a href="https://stats.wikimedia.org/#/fr.wiktionary.org">Statistiques</a></li>
320 | 		<li id="footer-places-cookiestatement"><a href="https://foundation.wikimedia.org/wiki/Cookie_statement">Déclaration sur les témoins (cookies)</a></li>
321 | 		<li id="footer-places-mobileview"><a href="//fr.m.wiktionary.org/w/index.php?title=loriquet_arc-en-ciel&amp;mobileaction=toggle_view_mobile" class="noprint stopMobileRedirectToggle">Version mobile</a></li>
322 | 	</ul>
323 | 	<ul id="footer-icons" class="noprint">
324 | 		<li id="footer-copyrightico"><a href="https://wikimediafoundation.org/"><img src="/static/images/wikimedia-button.png" srcset="/static/images/wikimedia-button-1.5x.png 1.5x, /static/images/wikimedia-button-2x.png 2x" width="88" height="31" alt="Wikimedia Foundation"/></a></li>
325 | 		<li id="footer-poweredbyico"><a href="https://www.mediawiki.org/"><img src="/static/images/poweredby_mediawiki_88x31.png" alt="Powered by MediaWiki" srcset="/static/images/poweredby_mediawiki_132x47.png 1.5x, /static/images/poweredby_mediawiki_176x62.png 2x" width="88" height="31"/></a></li>
326 | 	</ul>
327 | 	<div style="clear: both;"></div>
328 | </div>
329 | 
330 | 
331 | <script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgPageParseReport":{"limitreport":{"cputime":"0.212","walltime":"0.292","ppvisitednodes":{"value":873,"limit":1000000},"postexpandincludesize":{"value":12556,"limit":2097152},"templateargumentsize":{"value":816,"limit":2097152},"expansiondepth":{"value":12,"limit":40},"expensivefunctioncount":{"value":2,"limit":500},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":0,"limit":5000000},"entityaccesscount":{"value":0,"limit":400},"timingprofile":["100.00%  260.310      1 -total"," 22.31%   58.066      1 Modèle:langue"," 18.47%   48.089      1 Modèle:fr-rég"," 16.07%   41.844      7 Modèle:S","  8.38%   21.802     12 Modèle:T","  7.76%   20.201      3 Modèle:Lang","  7.50%   19.531     12 Modèle:trad-","  6.70%   17.453      1 Modèle:WP","  6.68%   17.399      2 Modèle:pron","  6.58%   17.121      2 Modèle:lien_m"]},"scribunto":{"limitreport-timeusage":{"value":"0.097","limit":"10.000"},"limitreport-memusage":{"value":6680453,"limit":52428800}},"cachereport":{"origin":"mw1325","timestamp":"20200505225155","ttl":2592000,"transientcontent":false}}});mw.config.set({"wgBackendResponseTime":104,"wgHostname":"mw1391"});});</script></body></html>
332 | 


--------------------------------------------------------------------------------