├── README.md
└── WebScraping.ipynb


/README.md:
--------------------------------------------------------------------------------
 1 | # WebScraping
 2 | Create a database from scratch by extracting html elements from a webpage
 3 | <br>
 4 | <b>Modules Used:</b> Urllib.request, BeautifulSoup, Regex and Pandas.
 5 | <br>
 6 | <br>
 7 | <b>Step by step walk-through:</b>  
 8 | <br>
 9 | <b>Step 1:</b> pulling HTML out of a webpage.
10 | <br>
11 | <b>Step 2:</b> targeting elements of interest inside the HTML.
12 | <br>
13 | <b>Step 3:</b> fine-tuning targeted elements with Regex (Regular Expressions), string concatenation and slicing.
14 | <br>
15 | <b>Step 4:</b> storing the data inside a DataFrame.
16 | <br>
17 | <b>Step 5:</b> exporting DataFrame into a CSV file.
18 | <br>
19 | <br>
20 | <b>Also available in a video explination:</b> https://youtu.be/ySNSY7iiBDY
21 | <br>
22 | <b>Author:</b> Mariya Sha
23 | <br>
24 | <b>Email:</b> mariyasha888@gmail.com
25 | <br>
26 | <b>LinkedIn:</b> www.linkedin.com/in/mariyasha888/
27 |   
28 | 


--------------------------------------------------------------------------------
/WebScraping.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import urllib.request\n",
 10 |     "from bs4 import BeautifulSoup as bs\n",
 11 |     "import re\n",
 12 |     "import pandas as pd"
 13 |    ]
 14 |   },
 15 |   {
 16 |    "cell_type": "markdown",
 17 |    "metadata": {},
 18 |    "source": [
 19 |     "# Extract and Store Function Names & Usage\n",
 20 |     "<br>\n",
 21 |     "From the Python Documentation - Random Functions webpage: https://docs.python.org/3/library/random.html\n"
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "code",
 26 |    "execution_count": 8,
 27 |    "metadata": {},
 28 |    "outputs": [
 29 |     {
 30 |      "name": "stdout",
 31 |      "output_type": "stream",
 32 |      "text": [
 33 |       "list of function names: ['random.seed', 'random.getstate', 'random.setstate', 'random.getrandbits', 'random.randrange']\n",
 34 |       "\n",
 35 |       "function description: Initialize the random number generator. If a is omitted or None, the current system time is used.  If randomness sources are provided by the operating system, they are used instead of the system time (see the os.urandom() function for details on availability). If a is an int, it is used directly. With version 2 (the default), a str, bytes, or bytearray object gets converted to an int and all of its bits are used. With version 1 (provided for reproducing random sequences from older versions of Python), the algorithm for str and bytes generates a narrower range of seeds.  Changed in version 3.2: Moved to the version 2 scheme which uses all of the bits in a string seed.  \n",
 36 |       "\n",
 37 |       "number of items in function names: 24\n",
 38 |       "number of items in function description: 24\n"
 39 |      ]
 40 |     }
 41 |    ],
 42 |    "source": [
 43 |     "#load html code from a url\n",
 44 |     "page = urllib.request.urlopen(\"https://docs.python.org/3/library/random.html\")\n",
 45 |     "soup = bs(page)\n",
 46 |     "\n",
 47 |     "#find all function names\n",
 48 |     "names = soup.body.findAll('dt')\n",
 49 |     "function_names = re.findall('id=\"random.\\w+', str(names))\n",
 50 |     "function_names = [item[4:] for item in function_names]\n",
 51 |     "\n",
 52 |     "#find all function descriptions\n",
 53 |     "description = soup.body.findAll('dd')\n",
 54 |     "function_usage = []\n",
 55 |     "\n",
 56 |     "for item in description:\n",
 57 |     "  item = item.text\n",
 58 |     "  item = item.replace('\\n', ' ')\n",
 59 |     "  function_usage.append(item)\n",
 60 |     "\n",
 61 |     "print('list of function names:',function_names[:5])\n",
 62 |     "print('\\nfunction description:', function_usage[0])\n",
 63 |     "print('\\nnumber of items in function names:', len(function_names))\n",
 64 |     "print('number of items in function description:', len(function_usage))"
 65 |    ]
 66 |   },
 67 |   {
 68 |    "cell_type": "markdown",
 69 |    "metadata": {},
 70 |    "source": [
 71 |     "# Store Data inside a DataFrame\n",
 72 |     "<br>\n",
 73 |     "After ensuring the lenghts of both lists match"
 74 |    ]
 75 |   },
 76 |   {
 77 |    "cell_type": "code",
 78 |    "execution_count": 4,
 79 |    "metadata": {},
 80 |    "outputs": [
 81 |     {
 82 |      "data": {
 83 |       "text/html": [
 84 |        "<div>\n",
 85 |        "<style scoped>\n",
 86 |        "    .dataframe tbody tr th:only-of-type {\n",
 87 |        "        vertical-align: middle;\n",
 88 |        "    }\n",
 89 |        "\n",
 90 |        "    .dataframe tbody tr th {\n",
 91 |        "        vertical-align: top;\n",
 92 |        "    }\n",
 93 |        "\n",
 94 |        "    .dataframe thead th {\n",
 95 |        "        text-align: right;\n",
 96 |        "    }\n",
 97 |        "</style>\n",
 98 |        "<table border=\"1\" class=\"dataframe\">\n",
 99 |        "  <thead>\n",
100 |        "    <tr style=\"text-align: right;\">\n",
101 |        "      <th></th>\n",
102 |        "      <th>function name</th>\n",
103 |        "      <th>function usage</th>\n",
104 |        "    </tr>\n",
105 |        "  </thead>\n",
106 |        "  <tbody>\n",
107 |        "    <tr>\n",
108 |        "      <th>0</th>\n",
109 |        "      <td>random.seed</td>\n",
110 |        "      <td>Initialize the random number generator. If a i...</td>\n",
111 |        "    </tr>\n",
112 |        "    <tr>\n",
113 |        "      <th>1</th>\n",
114 |        "      <td>random.getstate</td>\n",
115 |        "      <td>Return an object capturing the current interna...</td>\n",
116 |        "    </tr>\n",
117 |        "    <tr>\n",
118 |        "      <th>2</th>\n",
119 |        "      <td>random.setstate</td>\n",
120 |        "      <td>state should have been obtained from a previou...</td>\n",
121 |        "    </tr>\n",
122 |        "    <tr>\n",
123 |        "      <th>3</th>\n",
124 |        "      <td>random.getrandbits</td>\n",
125 |        "      <td>Returns a Python integer with k random bits. T...</td>\n",
126 |        "    </tr>\n",
127 |        "    <tr>\n",
128 |        "      <th>4</th>\n",
129 |        "      <td>random.randrange</td>\n",
130 |        "      <td>Return a randomly selected element from range(...</td>\n",
131 |        "    </tr>\n",
132 |        "  </tbody>\n",
133 |        "</table>\n",
134 |        "</div>"
135 |       ],
136 |       "text/plain": [
137 |        "        function name                                     function usage\n",
138 |        "0         random.seed  Initialize the random number generator. If a i...\n",
139 |        "1     random.getstate  Return an object capturing the current interna...\n",
140 |        "2     random.setstate  state should have been obtained from a previou...\n",
141 |        "3  random.getrandbits  Returns a Python integer with k random bits. T...\n",
142 |        "4    random.randrange  Return a randomly selected element from range(..."
143 |       ]
144 |      },
145 |      "execution_count": 4,
146 |      "metadata": {},
147 |      "output_type": "execute_result"
148 |     }
149 |    ],
150 |    "source": [
151 |     "#create a dataframe\n",
152 |     "data = pd.DataFrame({'function name': function_names, 'function usage': function_usage})\n",
153 |     "data.head()"
154 |    ]
155 |   },
156 |   {
157 |    "cell_type": "markdown",
158 |    "metadata": {},
159 |    "source": [
160 |     "# Exort Data into a csv file\n",
161 |     "<br>\n",
162 |     "The file will be saved in the same directory as the notebook"
163 |    ]
164 |   },
165 |   {
166 |    "cell_type": "code",
167 |    "execution_count": 5,
168 |    "metadata": {},
169 |    "outputs": [],
170 |    "source": [
171 |     "data.to_csv('my_file.csv')"
172 |    ]
173 |   }
174 |  ],
175 |  "metadata": {
176 |   "kernelspec": {
177 |    "display_name": "Python 3",
178 |    "language": "python",
179 |    "name": "python3"
180 |   },
181 |   "language_info": {
182 |    "codemirror_mode": {
183 |     "name": "ipython",
184 |     "version": 3
185 |    },
186 |    "file_extension": ".py",
187 |    "mimetype": "text/x-python",
188 |    "name": "python",
189 |    "nbconvert_exporter": "python",
190 |    "pygments_lexer": "ipython3",
191 |    "version": "3.7.4"
192 |   }
193 |  },
194 |  "nbformat": 4,
195 |  "nbformat_minor": 2
196 | }
197 | 


--------------------------------------------------------------------------------