├── Data
    ├── Brazilian ecommerce EDA .ipynb
    └── brazilian-ecommerce.zip
├── LICENSE
├── README.md
└── airflow
    ├── dags
        └── late_shipments_to_carrier_dag.py
    └── scripts
        ├── s3_download.py
        ├── s3_upload.py
        └── spark_missed_deadline_job.py


/Data/Brazilian ecommerce EDA .ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "markdown",
   5 |    "metadata": {},
   6 |    "source": [
   7 |     "## Super basic EDA\n",
   8 |     "The concern of this super basic EDA is to determine which order packages were delivered passed the shipping limit date. In other words, the idea is to identify orders where the seller did not deliver the package to the carrier prior to the designated limit that Olist placed on them to do so. "
   9 |    ]
  10 |   },
  11 |   {
  12 |    "cell_type": "code",
  13 |    "execution_count": 1,
  14 |    "metadata": {},
  15 |    "outputs": [
  16 |     {
  17 |      "name": "stdout",
  18 |      "output_type": "stream",
  19 |      "text": [
  20 |       "Untitled.ipynb          \u001b[34mbrazilian-ecommerce\u001b[m\u001b[m     brazilian-ecommerce.zip\r\n"
  21 |      ]
  22 |     }
  23 |    ],
  24 |    "source": [
  25 |     "! ls"
  26 |    ]
  27 |   },
  28 |   {
  29 |    "cell_type": "code",
  30 |    "execution_count": 2,
  31 |    "metadata": {},
  32 |    "outputs": [],
  33 |    "source": [
  34 |     "import pandas as pd\n",
  35 |     "import zipfile\n",
  36 |     "\n",
  37 |     "zf = zipfile.ZipFile('brazilian-ecommerce.zip') "
  38 |    ]
  39 |   },
  40 |   {
  41 |    "cell_type": "code",
  42 |    "execution_count": 12,
  43 |    "metadata": {},
  44 |    "outputs": [],
  45 |    "source": [
  46 |     "from zipfile import ZipFile\n",
  47 |     "\n",
  48 |     "# Create a ZipFile Object and load sample.zip in it\n",
  49 |     "with ZipFile('brazilian-ecommerce.zip', 'r') as zipObj:\n",
  50 |     "    # Extract all the contents of zip file in current directory\n",
  51 |     "    zipObj.extractall()"
  52 |    ]
  53 |   },
  54 |   {
  55 |    "cell_type": "code",
  56 |    "execution_count": 13,
  57 |    "metadata": {},
  58 |    "outputs": [
  59 |     {
  60 |      "name": "stdout",
  61 |      "output_type": "stream",
  62 |      "text": [
  63 |       "Untitled.ipynb                        olist_order_reviews_dataset.csv\r\n",
  64 |       "\u001b[34mbrazilian-ecommerce\u001b[m\u001b[m                   olist_orders_dataset.csv\r\n",
  65 |       "brazilian-ecommerce.zip               olist_products_dataset.csv\r\n",
  66 |       "olist_customers_dataset.csv           olist_sellers_dataset.csv\r\n",
  67 |       "olist_geolocation_dataset.csv         product_category_name_translation.csv\r\n",
  68 |       "olist_order_items_dataset.csv         \u001b[34mtemp_csv\u001b[m\u001b[m\r\n",
  69 |       "olist_order_payments_dataset.csv\r\n"
  70 |      ]
  71 |     }
  72 |    ],
  73 |    "source": [
  74 |     "! ls"
  75 |    ]
  76 |   },
  77 |   {
  78 |    "cell_type": "code",
  79 |    "execution_count": 33,
  80 |    "metadata": {},
  81 |    "outputs": [],
  82 |    "source": [
  83 |     "import pandas as pd\n",
  84 |     " \n",
  85 |     "df_items = pd.read_csv('olist_order_items_dataset.csv')\n",
  86 |     "df_orders = pd.read_csv('olist_orders_dataset.csv')\n",
  87 |     "df_products = pd.read_csv('olist_products_dataset.csv')"
  88 |    ]
  89 |   },
  90 |   {
  91 |    "cell_type": "code",
  92 |    "execution_count": 29,
  93 |    "metadata": {},
  94 |    "outputs": [
  95 |     {
  96 |      "data": {
  97 |       "text/html": [
  98 |        "<div>\n",
  99 |        "<style scoped>\n",
 100 |        "    .dataframe tbody tr th:only-of-type {\n",
 101 |        "        vertical-align: middle;\n",
 102 |        "    }\n",
 103 |        "\n",
 104 |        "    .dataframe tbody tr th {\n",
 105 |        "        vertical-align: top;\n",
 106 |        "    }\n",
 107 |        "\n",
 108 |        "    .dataframe thead th {\n",
 109 |        "        text-align: right;\n",
 110 |        "    }\n",
 111 |        "</style>\n",
 112 |        "<table border=\"1\" class=\"dataframe\">\n",
 113 |        "  <thead>\n",
 114 |        "    <tr style=\"text-align: right;\">\n",
 115 |        "      <th></th>\n",
 116 |        "      <th>order_id</th>\n",
 117 |        "      <th>order_item_id</th>\n",
 118 |        "      <th>product_id</th>\n",
 119 |        "      <th>seller_id</th>\n",
 120 |        "      <th>shipping_limit_date</th>\n",
 121 |        "      <th>price</th>\n",
 122 |        "      <th>freight_value</th>\n",
 123 |        "    </tr>\n",
 124 |        "  </thead>\n",
 125 |        "  <tbody>\n",
 126 |        "    <tr>\n",
 127 |        "      <th>0</th>\n",
 128 |        "      <td>00010242fe8c5a6d1ba2dd792cb16214</td>\n",
 129 |        "      <td>1</td>\n",
 130 |        "      <td>4244733e06e7ecb4970a6e2683c13e61</td>\n",
 131 |        "      <td>48436dade18ac8b2bce089ec2a041202</td>\n",
 132 |        "      <td>2017-09-19 09:45:35</td>\n",
 133 |        "      <td>58.90</td>\n",
 134 |        "      <td>13.29</td>\n",
 135 |        "    </tr>\n",
 136 |        "    <tr>\n",
 137 |        "      <th>1</th>\n",
 138 |        "      <td>00018f77f2f0320c557190d7a144bdd3</td>\n",
 139 |        "      <td>1</td>\n",
 140 |        "      <td>e5f2d52b802189ee658865ca93d83a8f</td>\n",
 141 |        "      <td>dd7ddc04e1b6c2c614352b383efe2d36</td>\n",
 142 |        "      <td>2017-05-03 11:05:13</td>\n",
 143 |        "      <td>239.90</td>\n",
 144 |        "      <td>19.93</td>\n",
 145 |        "    </tr>\n",
 146 |        "    <tr>\n",
 147 |        "      <th>2</th>\n",
 148 |        "      <td>000229ec398224ef6ca0657da4fc703e</td>\n",
 149 |        "      <td>1</td>\n",
 150 |        "      <td>c777355d18b72b67abbeef9df44fd0fd</td>\n",
 151 |        "      <td>5b51032eddd242adc84c38acab88f23d</td>\n",
 152 |        "      <td>2018-01-18 14:48:30</td>\n",
 153 |        "      <td>199.00</td>\n",
 154 |        "      <td>17.87</td>\n",
 155 |        "    </tr>\n",
 156 |        "    <tr>\n",
 157 |        "      <th>3</th>\n",
 158 |        "      <td>00024acbcdf0a6daa1e931b038114c75</td>\n",
 159 |        "      <td>1</td>\n",
 160 |        "      <td>7634da152a4610f1595efa32f14722fc</td>\n",
 161 |        "      <td>9d7a1d34a5052409006425275ba1c2b4</td>\n",
 162 |        "      <td>2018-08-15 10:10:18</td>\n",
 163 |        "      <td>12.99</td>\n",
 164 |        "      <td>12.79</td>\n",
 165 |        "    </tr>\n",
 166 |        "    <tr>\n",
 167 |        "      <th>4</th>\n",
 168 |        "      <td>00042b26cf59d7ce69dfabb4e55b4fd9</td>\n",
 169 |        "      <td>1</td>\n",
 170 |        "      <td>ac6c3623068f30de03045865e4e10089</td>\n",
 171 |        "      <td>df560393f3a51e74553ab94004ba5c87</td>\n",
 172 |        "      <td>2017-02-13 13:57:51</td>\n",
 173 |        "      <td>199.90</td>\n",
 174 |        "      <td>18.14</td>\n",
 175 |        "    </tr>\n",
 176 |        "  </tbody>\n",
 177 |        "</table>\n",
 178 |        "</div>"
 179 |       ],
 180 |       "text/plain": [
 181 |        "                           order_id  order_item_id  \\\n",
 182 |        "0  00010242fe8c5a6d1ba2dd792cb16214              1   \n",
 183 |        "1  00018f77f2f0320c557190d7a144bdd3              1   \n",
 184 |        "2  000229ec398224ef6ca0657da4fc703e              1   \n",
 185 |        "3  00024acbcdf0a6daa1e931b038114c75              1   \n",
 186 |        "4  00042b26cf59d7ce69dfabb4e55b4fd9              1   \n",
 187 |        "\n",
 188 |        "                         product_id                         seller_id  \\\n",
 189 |        "0  4244733e06e7ecb4970a6e2683c13e61  48436dade18ac8b2bce089ec2a041202   \n",
 190 |        "1  e5f2d52b802189ee658865ca93d83a8f  dd7ddc04e1b6c2c614352b383efe2d36   \n",
 191 |        "2  c777355d18b72b67abbeef9df44fd0fd  5b51032eddd242adc84c38acab88f23d   \n",
 192 |        "3  7634da152a4610f1595efa32f14722fc  9d7a1d34a5052409006425275ba1c2b4   \n",
 193 |        "4  ac6c3623068f30de03045865e4e10089  df560393f3a51e74553ab94004ba5c87   \n",
 194 |        "\n",
 195 |        "   shipping_limit_date   price  freight_value  \n",
 196 |        "0  2017-09-19 09:45:35   58.90          13.29  \n",
 197 |        "1  2017-05-03 11:05:13  239.90          19.93  \n",
 198 |        "2  2018-01-18 14:48:30  199.00          17.87  \n",
 199 |        "3  2018-08-15 10:10:18   12.99          12.79  \n",
 200 |        "4  2017-02-13 13:57:51  199.90          18.14  "
 201 |       ]
 202 |      },
 203 |      "execution_count": 29,
 204 |      "metadata": {},
 205 |      "output_type": "execute_result"
 206 |     }
 207 |    ],
 208 |    "source": [
 209 |     "df_items.head()"
 210 |    ]
 211 |   },
 212 |   {
 213 |    "cell_type": "code",
 214 |    "execution_count": 30,
 215 |    "metadata": {},
 216 |    "outputs": [
 217 |     {
 218 |      "data": {
 219 |       "text/html": [
 220 |        "<div>\n",
 221 |        "<style scoped>\n",
 222 |        "    .dataframe tbody tr th:only-of-type {\n",
 223 |        "        vertical-align: middle;\n",
 224 |        "    }\n",
 225 |        "\n",
 226 |        "    .dataframe tbody tr th {\n",
 227 |        "        vertical-align: top;\n",
 228 |        "    }\n",
 229 |        "\n",
 230 |        "    .dataframe thead th {\n",
 231 |        "        text-align: right;\n",
 232 |        "    }\n",
 233 |        "</style>\n",
 234 |        "<table border=\"1\" class=\"dataframe\">\n",
 235 |        "  <thead>\n",
 236 |        "    <tr style=\"text-align: right;\">\n",
 237 |        "      <th></th>\n",
 238 |        "      <th>order_id</th>\n",
 239 |        "      <th>customer_id</th>\n",
 240 |        "      <th>order_status</th>\n",
 241 |        "      <th>order_purchase_timestamp</th>\n",
 242 |        "      <th>order_approved_at</th>\n",
 243 |        "      <th>order_delivered_carrier_date</th>\n",
 244 |        "      <th>order_delivered_customer_date</th>\n",
 245 |        "      <th>order_estimated_delivery_date</th>\n",
 246 |        "    </tr>\n",
 247 |        "  </thead>\n",
 248 |        "  <tbody>\n",
 249 |        "    <tr>\n",
 250 |        "      <th>0</th>\n",
 251 |        "      <td>e481f51cbdc54678b7cc49136f2d6af7</td>\n",
 252 |        "      <td>9ef432eb6251297304e76186b10a928d</td>\n",
 253 |        "      <td>delivered</td>\n",
 254 |        "      <td>2017-10-02 10:56:33</td>\n",
 255 |        "      <td>2017-10-02 11:07:15</td>\n",
 256 |        "      <td>2017-10-04 19:55:00</td>\n",
 257 |        "      <td>2017-10-10 21:25:13</td>\n",
 258 |        "      <td>2017-10-18 00:00:00</td>\n",
 259 |        "    </tr>\n",
 260 |        "    <tr>\n",
 261 |        "      <th>1</th>\n",
 262 |        "      <td>53cdb2fc8bc7dce0b6741e2150273451</td>\n",
 263 |        "      <td>b0830fb4747a6c6d20dea0b8c802d7ef</td>\n",
 264 |        "      <td>delivered</td>\n",
 265 |        "      <td>2018-07-24 20:41:37</td>\n",
 266 |        "      <td>2018-07-26 03:24:27</td>\n",
 267 |        "      <td>2018-07-26 14:31:00</td>\n",
 268 |        "      <td>2018-08-07 15:27:45</td>\n",
 269 |        "      <td>2018-08-13 00:00:00</td>\n",
 270 |        "    </tr>\n",
 271 |        "    <tr>\n",
 272 |        "      <th>2</th>\n",
 273 |        "      <td>47770eb9100c2d0c44946d9cf07ec65d</td>\n",
 274 |        "      <td>41ce2a54c0b03bf3443c3d931a367089</td>\n",
 275 |        "      <td>delivered</td>\n",
 276 |        "      <td>2018-08-08 08:38:49</td>\n",
 277 |        "      <td>2018-08-08 08:55:23</td>\n",
 278 |        "      <td>2018-08-08 13:50:00</td>\n",
 279 |        "      <td>2018-08-17 18:06:29</td>\n",
 280 |        "      <td>2018-09-04 00:00:00</td>\n",
 281 |        "    </tr>\n",
 282 |        "    <tr>\n",
 283 |        "      <th>3</th>\n",
 284 |        "      <td>949d5b44dbf5de918fe9c16f97b45f8a</td>\n",
 285 |        "      <td>f88197465ea7920adcdbec7375364d82</td>\n",
 286 |        "      <td>delivered</td>\n",
 287 |        "      <td>2017-11-18 19:28:06</td>\n",
 288 |        "      <td>2017-11-18 19:45:59</td>\n",
 289 |        "      <td>2017-11-22 13:39:59</td>\n",
 290 |        "      <td>2017-12-02 00:28:42</td>\n",
 291 |        "      <td>2017-12-15 00:00:00</td>\n",
 292 |        "    </tr>\n",
 293 |        "    <tr>\n",
 294 |        "      <th>4</th>\n",
 295 |        "      <td>ad21c59c0840e6cb83a9ceb5573f8159</td>\n",
 296 |        "      <td>8ab97904e6daea8866dbdbc4fb7aad2c</td>\n",
 297 |        "      <td>delivered</td>\n",
 298 |        "      <td>2018-02-13 21:18:39</td>\n",
 299 |        "      <td>2018-02-13 22:20:29</td>\n",
 300 |        "      <td>2018-02-14 19:46:34</td>\n",
 301 |        "      <td>2018-02-16 18:17:02</td>\n",
 302 |        "      <td>2018-02-26 00:00:00</td>\n",
 303 |        "    </tr>\n",
 304 |        "  </tbody>\n",
 305 |        "</table>\n",
 306 |        "</div>"
 307 |       ],
 308 |       "text/plain": [
 309 |        "                           order_id                       customer_id  \\\n",
 310 |        "0  e481f51cbdc54678b7cc49136f2d6af7  9ef432eb6251297304e76186b10a928d   \n",
 311 |        "1  53cdb2fc8bc7dce0b6741e2150273451  b0830fb4747a6c6d20dea0b8c802d7ef   \n",
 312 |        "2  47770eb9100c2d0c44946d9cf07ec65d  41ce2a54c0b03bf3443c3d931a367089   \n",
 313 |        "3  949d5b44dbf5de918fe9c16f97b45f8a  f88197465ea7920adcdbec7375364d82   \n",
 314 |        "4  ad21c59c0840e6cb83a9ceb5573f8159  8ab97904e6daea8866dbdbc4fb7aad2c   \n",
 315 |        "\n",
 316 |        "  order_status order_purchase_timestamp    order_approved_at  \\\n",
 317 |        "0    delivered      2017-10-02 10:56:33  2017-10-02 11:07:15   \n",
 318 |        "1    delivered      2018-07-24 20:41:37  2018-07-26 03:24:27   \n",
 319 |        "2    delivered      2018-08-08 08:38:49  2018-08-08 08:55:23   \n",
 320 |        "3    delivered      2017-11-18 19:28:06  2017-11-18 19:45:59   \n",
 321 |        "4    delivered      2018-02-13 21:18:39  2018-02-13 22:20:29   \n",
 322 |        "\n",
 323 |        "  order_delivered_carrier_date order_delivered_customer_date  \\\n",
 324 |        "0          2017-10-04 19:55:00           2017-10-10 21:25:13   \n",
 325 |        "1          2018-07-26 14:31:00           2018-08-07 15:27:45   \n",
 326 |        "2          2018-08-08 13:50:00           2018-08-17 18:06:29   \n",
 327 |        "3          2017-11-22 13:39:59           2017-12-02 00:28:42   \n",
 328 |        "4          2018-02-14 19:46:34           2018-02-16 18:17:02   \n",
 329 |        "\n",
 330 |        "  order_estimated_delivery_date  \n",
 331 |        "0           2017-10-18 00:00:00  \n",
 332 |        "1           2018-08-13 00:00:00  \n",
 333 |        "2           2018-09-04 00:00:00  \n",
 334 |        "3           2017-12-15 00:00:00  \n",
 335 |        "4           2018-02-26 00:00:00  "
 336 |       ]
 337 |      },
 338 |      "execution_count": 30,
 339 |      "metadata": {},
 340 |      "output_type": "execute_result"
 341 |     }
 342 |    ],
 343 |    "source": [
 344 |     "df_orders.head()"
 345 |    ]
 346 |   },
 347 |   {
 348 |    "cell_type": "code",
 349 |    "execution_count": 34,
 350 |    "metadata": {},
 351 |    "outputs": [
 352 |     {
 353 |      "data": {
 354 |       "text/html": [
 355 |        "<div>\n",
 356 |        "<style scoped>\n",
 357 |        "    .dataframe tbody tr th:only-of-type {\n",
 358 |        "        vertical-align: middle;\n",
 359 |        "    }\n",
 360 |        "\n",
 361 |        "    .dataframe tbody tr th {\n",
 362 |        "        vertical-align: top;\n",
 363 |        "    }\n",
 364 |        "\n",
 365 |        "    .dataframe thead th {\n",
 366 |        "        text-align: right;\n",
 367 |        "    }\n",
 368 |        "</style>\n",
 369 |        "<table border=\"1\" class=\"dataframe\">\n",
 370 |        "  <thead>\n",
 371 |        "    <tr style=\"text-align: right;\">\n",
 372 |        "      <th></th>\n",
 373 |        "      <th>product_id</th>\n",
 374 |        "      <th>product_category_name</th>\n",
 375 |        "      <th>product_name_lenght</th>\n",
 376 |        "      <th>product_description_lenght</th>\n",
 377 |        "      <th>product_photos_qty</th>\n",
 378 |        "      <th>product_weight_g</th>\n",
 379 |        "      <th>product_length_cm</th>\n",
 380 |        "      <th>product_height_cm</th>\n",
 381 |        "      <th>product_width_cm</th>\n",
 382 |        "    </tr>\n",
 383 |        "  </thead>\n",
 384 |        "  <tbody>\n",
 385 |        "    <tr>\n",
 386 |        "      <th>0</th>\n",
 387 |        "      <td>1e9e8ef04dbcff4541ed26657ea517e5</td>\n",
 388 |        "      <td>perfumaria</td>\n",
 389 |        "      <td>40.0</td>\n",
 390 |        "      <td>287.0</td>\n",
 391 |        "      <td>1.0</td>\n",
 392 |        "      <td>225.0</td>\n",
 393 |        "      <td>16.0</td>\n",
 394 |        "      <td>10.0</td>\n",
 395 |        "      <td>14.0</td>\n",
 396 |        "    </tr>\n",
 397 |        "    <tr>\n",
 398 |        "      <th>1</th>\n",
 399 |        "      <td>3aa071139cb16b67ca9e5dea641aaa2f</td>\n",
 400 |        "      <td>artes</td>\n",
 401 |        "      <td>44.0</td>\n",
 402 |        "      <td>276.0</td>\n",
 403 |        "      <td>1.0</td>\n",
 404 |        "      <td>1000.0</td>\n",
 405 |        "      <td>30.0</td>\n",
 406 |        "      <td>18.0</td>\n",
 407 |        "      <td>20.0</td>\n",
 408 |        "    </tr>\n",
 409 |        "    <tr>\n",
 410 |        "      <th>2</th>\n",
 411 |        "      <td>96bd76ec8810374ed1b65e291975717f</td>\n",
 412 |        "      <td>esporte_lazer</td>\n",
 413 |        "      <td>46.0</td>\n",
 414 |        "      <td>250.0</td>\n",
 415 |        "      <td>1.0</td>\n",
 416 |        "      <td>154.0</td>\n",
 417 |        "      <td>18.0</td>\n",
 418 |        "      <td>9.0</td>\n",
 419 |        "      <td>15.0</td>\n",
 420 |        "    </tr>\n",
 421 |        "    <tr>\n",
 422 |        "      <th>3</th>\n",
 423 |        "      <td>cef67bcfe19066a932b7673e239eb23d</td>\n",
 424 |        "      <td>bebes</td>\n",
 425 |        "      <td>27.0</td>\n",
 426 |        "      <td>261.0</td>\n",
 427 |        "      <td>1.0</td>\n",
 428 |        "      <td>371.0</td>\n",
 429 |        "      <td>26.0</td>\n",
 430 |        "      <td>4.0</td>\n",
 431 |        "      <td>26.0</td>\n",
 432 |        "    </tr>\n",
 433 |        "    <tr>\n",
 434 |        "      <th>4</th>\n",
 435 |        "      <td>9dc1a7de274444849c219cff195d0b71</td>\n",
 436 |        "      <td>utilidades_domesticas</td>\n",
 437 |        "      <td>37.0</td>\n",
 438 |        "      <td>402.0</td>\n",
 439 |        "      <td>4.0</td>\n",
 440 |        "      <td>625.0</td>\n",
 441 |        "      <td>20.0</td>\n",
 442 |        "      <td>17.0</td>\n",
 443 |        "      <td>13.0</td>\n",
 444 |        "    </tr>\n",
 445 |        "  </tbody>\n",
 446 |        "</table>\n",
 447 |        "</div>"
 448 |       ],
 449 |       "text/plain": [
 450 |        "                         product_id  product_category_name  \\\n",
 451 |        "0  1e9e8ef04dbcff4541ed26657ea517e5             perfumaria   \n",
 452 |        "1  3aa071139cb16b67ca9e5dea641aaa2f                  artes   \n",
 453 |        "2  96bd76ec8810374ed1b65e291975717f          esporte_lazer   \n",
 454 |        "3  cef67bcfe19066a932b7673e239eb23d                  bebes   \n",
 455 |        "4  9dc1a7de274444849c219cff195d0b71  utilidades_domesticas   \n",
 456 |        "\n",
 457 |        "   product_name_lenght  product_description_lenght  product_photos_qty  \\\n",
 458 |        "0                 40.0                       287.0                 1.0   \n",
 459 |        "1                 44.0                       276.0                 1.0   \n",
 460 |        "2                 46.0                       250.0                 1.0   \n",
 461 |        "3                 27.0                       261.0                 1.0   \n",
 462 |        "4                 37.0                       402.0                 4.0   \n",
 463 |        "\n",
 464 |        "   product_weight_g  product_length_cm  product_height_cm  product_width_cm  \n",
 465 |        "0             225.0               16.0               10.0              14.0  \n",
 466 |        "1            1000.0               30.0               18.0              20.0  \n",
 467 |        "2             154.0               18.0                9.0              15.0  \n",
 468 |        "3             371.0               26.0                4.0              26.0  \n",
 469 |        "4             625.0               20.0               17.0              13.0  "
 470 |       ]
 471 |      },
 472 |      "execution_count": 34,
 473 |      "metadata": {},
 474 |      "output_type": "execute_result"
 475 |     }
 476 |    ],
 477 |    "source": [
 478 |     "df_products.head()"
 479 |    ]
 480 |   },
 481 |   {
 482 |    "cell_type": "code",
 483 |    "execution_count": 43,
 484 |    "metadata": {},
 485 |    "outputs": [],
 486 |    "source": [
 487 |     "# Merge the seller id/shipping limit, order information, and product information\n",
 488 |     "df = df_items.merge(df_orders, on='order_id').merge(df_products[['product_id', 'product_category_name']], \n",
 489 |     "                                                    on='product_id')\n",
 490 |     "\n"
 491 |    ]
 492 |   },
 493 |   {
 494 |    "cell_type": "code",
 495 |    "execution_count": 44,
 496 |    "metadata": {},
 497 |    "outputs": [
 498 |     {
 499 |      "data": {
 500 |       "text/plain": [
 501 |        "Index(['order_id', 'order_item_id', 'product_id', 'seller_id',\n",
 502 |        "       'shipping_limit_date', 'price', 'freight_value', 'customer_id',\n",
 503 |        "       'order_status', 'order_purchase_timestamp', 'order_approved_at',\n",
 504 |        "       'order_delivered_carrier_date', 'order_delivered_customer_date',\n",
 505 |        "       'order_estimated_delivery_date', 'product_category_name'],\n",
 506 |        "      dtype='object')"
 507 |       ]
 508 |      },
 509 |      "execution_count": 44,
 510 |      "metadata": {},
 511 |      "output_type": "execute_result"
 512 |     }
 513 |    ],
 514 |    "source": [
 515 |     "df.columns"
 516 |    ]
 517 |   },
 518 |   {
 519 |    "cell_type": "code",
 520 |    "execution_count": 84,
 521 |    "metadata": {},
 522 |    "outputs": [
 523 |     {
 524 |      "data": {
 525 |       "text/html": [
 526 |        "<div>\n",
 527 |        "<style scoped>\n",
 528 |        "    .dataframe tbody tr th:only-of-type {\n",
 529 |        "        vertical-align: middle;\n",
 530 |        "    }\n",
 531 |        "\n",
 532 |        "    .dataframe tbody tr th {\n",
 533 |        "        vertical-align: top;\n",
 534 |        "    }\n",
 535 |        "\n",
 536 |        "    .dataframe thead th {\n",
 537 |        "        text-align: right;\n",
 538 |        "    }\n",
 539 |        "</style>\n",
 540 |        "<table border=\"1\" class=\"dataframe\">\n",
 541 |        "  <thead>\n",
 542 |        "    <tr style=\"text-align: right;\">\n",
 543 |        "      <th></th>\n",
 544 |        "      <th>order_id</th>\n",
 545 |        "      <th>order_item_id</th>\n",
 546 |        "      <th>product_id</th>\n",
 547 |        "      <th>seller_id</th>\n",
 548 |        "      <th>shipping_limit_date</th>\n",
 549 |        "      <th>price</th>\n",
 550 |        "      <th>freight_value</th>\n",
 551 |        "      <th>customer_id</th>\n",
 552 |        "      <th>order_status</th>\n",
 553 |        "      <th>order_purchase_timestamp</th>\n",
 554 |        "      <th>order_approved_at</th>\n",
 555 |        "      <th>order_delivered_carrier_date</th>\n",
 556 |        "      <th>order_delivered_customer_date</th>\n",
 557 |        "      <th>order_estimated_delivery_date</th>\n",
 558 |        "      <th>product_category_name</th>\n",
 559 |        "    </tr>\n",
 560 |        "  </thead>\n",
 561 |        "  <tbody>\n",
 562 |        "    <tr>\n",
 563 |        "      <th>0</th>\n",
 564 |        "      <td>00010242fe8c5a6d1ba2dd792cb16214</td>\n",
 565 |        "      <td>1</td>\n",
 566 |        "      <td>4244733e06e7ecb4970a6e2683c13e61</td>\n",
 567 |        "      <td>48436dade18ac8b2bce089ec2a041202</td>\n",
 568 |        "      <td>2017-09-19 09:45:35</td>\n",
 569 |        "      <td>58.9</td>\n",
 570 |        "      <td>13.29</td>\n",
 571 |        "      <td>3ce436f183e68e07877b285a838db11a</td>\n",
 572 |        "      <td>delivered</td>\n",
 573 |        "      <td>2017-09-13 08:59:02</td>\n",
 574 |        "      <td>2017-09-13 09:45:35</td>\n",
 575 |        "      <td>2017-09-19 18:34:16</td>\n",
 576 |        "      <td>2017-09-20 23:43:48</td>\n",
 577 |        "      <td>2017-09-29 00:00:00</td>\n",
 578 |        "      <td>cool_stuff</td>\n",
 579 |        "    </tr>\n",
 580 |        "    <tr>\n",
 581 |        "      <th>1</th>\n",
 582 |        "      <td>130898c0987d1801452a8ed92a670612</td>\n",
 583 |        "      <td>1</td>\n",
 584 |        "      <td>4244733e06e7ecb4970a6e2683c13e61</td>\n",
 585 |        "      <td>48436dade18ac8b2bce089ec2a041202</td>\n",
 586 |        "      <td>2017-07-05 02:44:11</td>\n",
 587 |        "      <td>55.9</td>\n",
 588 |        "      <td>17.96</td>\n",
 589 |        "      <td>e6eecc5a77de221464d1c4eaff0a9b64</td>\n",
 590 |        "      <td>delivered</td>\n",
 591 |        "      <td>2017-06-28 11:52:20</td>\n",
 592 |        "      <td>2017-06-29 02:44:11</td>\n",
 593 |        "      <td>2017-07-05 12:00:33</td>\n",
 594 |        "      <td>2017-07-13 20:39:29</td>\n",
 595 |        "      <td>2017-07-26 00:00:00</td>\n",
 596 |        "      <td>cool_stuff</td>\n",
 597 |        "    </tr>\n",
 598 |        "    <tr>\n",
 599 |        "      <th>2</th>\n",
 600 |        "      <td>532ed5e14e24ae1f0d735b91524b98b9</td>\n",
 601 |        "      <td>1</td>\n",
 602 |        "      <td>4244733e06e7ecb4970a6e2683c13e61</td>\n",
 603 |        "      <td>48436dade18ac8b2bce089ec2a041202</td>\n",
 604 |        "      <td>2018-05-23 10:56:25</td>\n",
 605 |        "      <td>64.9</td>\n",
 606 |        "      <td>18.33</td>\n",
 607 |        "      <td>4ef55bf80f711b372afebcb7c715344a</td>\n",
 608 |        "      <td>delivered</td>\n",
 609 |        "      <td>2018-05-18 10:25:53</td>\n",
 610 |        "      <td>2018-05-18 12:31:43</td>\n",
 611 |        "      <td>2018-05-23 14:05:00</td>\n",
 612 |        "      <td>2018-06-04 18:34:26</td>\n",
 613 |        "      <td>2018-06-07 00:00:00</td>\n",
 614 |        "      <td>cool_stuff</td>\n",
 615 |        "    </tr>\n",
 616 |        "    <tr>\n",
 617 |        "      <th>9</th>\n",
 618 |        "      <td>00018f77f2f0320c557190d7a144bdd3</td>\n",
 619 |        "      <td>1</td>\n",
 620 |        "      <td>e5f2d52b802189ee658865ca93d83a8f</td>\n",
 621 |        "      <td>dd7ddc04e1b6c2c614352b383efe2d36</td>\n",
 622 |        "      <td>2017-05-03 11:05:13</td>\n",
 623 |        "      <td>239.9</td>\n",
 624 |        "      <td>19.93</td>\n",
 625 |        "      <td>f6dd3ec061db4e3987629fe6b26e5cce</td>\n",
 626 |        "      <td>delivered</td>\n",
 627 |        "      <td>2017-04-26 10:53:06</td>\n",
 628 |        "      <td>2017-04-26 11:05:13</td>\n",
 629 |        "      <td>2017-05-04 14:35:00</td>\n",
 630 |        "      <td>2017-05-12 16:04:24</td>\n",
 631 |        "      <td>2017-05-15 00:00:00</td>\n",
 632 |        "      <td>pet_shop</td>\n",
 633 |        "    </tr>\n",
 634 |        "    <tr>\n",
 635 |        "      <th>15</th>\n",
 636 |        "      <td>00042b26cf59d7ce69dfabb4e55b4fd9</td>\n",
 637 |        "      <td>1</td>\n",
 638 |        "      <td>ac6c3623068f30de03045865e4e10089</td>\n",
 639 |        "      <td>df560393f3a51e74553ab94004ba5c87</td>\n",
 640 |        "      <td>2017-02-13 13:57:51</td>\n",
 641 |        "      <td>199.9</td>\n",
 642 |        "      <td>18.14</td>\n",
 643 |        "      <td>58dbd0b2d70206bf40e62cd34e84d795</td>\n",
 644 |        "      <td>delivered</td>\n",
 645 |        "      <td>2017-02-04 13:57:51</td>\n",
 646 |        "      <td>2017-02-04 14:10:13</td>\n",
 647 |        "      <td>2017-02-16 09:46:09</td>\n",
 648 |        "      <td>2017-03-01 16:42:31</td>\n",
 649 |        "      <td>2017-03-17 00:00:00</td>\n",
 650 |        "      <td>ferramentas_jardim</td>\n",
 651 |        "    </tr>\n",
 652 |        "  </tbody>\n",
 653 |        "</table>\n",
 654 |        "</div>"
 655 |       ],
 656 |       "text/plain": [
 657 |        "                            order_id  order_item_id  \\\n",
 658 |        "0   00010242fe8c5a6d1ba2dd792cb16214              1   \n",
 659 |        "1   130898c0987d1801452a8ed92a670612              1   \n",
 660 |        "2   532ed5e14e24ae1f0d735b91524b98b9              1   \n",
 661 |        "9   00018f77f2f0320c557190d7a144bdd3              1   \n",
 662 |        "15  00042b26cf59d7ce69dfabb4e55b4fd9              1   \n",
 663 |        "\n",
 664 |        "                          product_id                         seller_id  \\\n",
 665 |        "0   4244733e06e7ecb4970a6e2683c13e61  48436dade18ac8b2bce089ec2a041202   \n",
 666 |        "1   4244733e06e7ecb4970a6e2683c13e61  48436dade18ac8b2bce089ec2a041202   \n",
 667 |        "2   4244733e06e7ecb4970a6e2683c13e61  48436dade18ac8b2bce089ec2a041202   \n",
 668 |        "9   e5f2d52b802189ee658865ca93d83a8f  dd7ddc04e1b6c2c614352b383efe2d36   \n",
 669 |        "15  ac6c3623068f30de03045865e4e10089  df560393f3a51e74553ab94004ba5c87   \n",
 670 |        "\n",
 671 |        "    shipping_limit_date  price  freight_value  \\\n",
 672 |        "0   2017-09-19 09:45:35   58.9          13.29   \n",
 673 |        "1   2017-07-05 02:44:11   55.9          17.96   \n",
 674 |        "2   2018-05-23 10:56:25   64.9          18.33   \n",
 675 |        "9   2017-05-03 11:05:13  239.9          19.93   \n",
 676 |        "15  2017-02-13 13:57:51  199.9          18.14   \n",
 677 |        "\n",
 678 |        "                         customer_id order_status order_purchase_timestamp  \\\n",
 679 |        "0   3ce436f183e68e07877b285a838db11a    delivered      2017-09-13 08:59:02   \n",
 680 |        "1   e6eecc5a77de221464d1c4eaff0a9b64    delivered      2017-06-28 11:52:20   \n",
 681 |        "2   4ef55bf80f711b372afebcb7c715344a    delivered      2018-05-18 10:25:53   \n",
 682 |        "9   f6dd3ec061db4e3987629fe6b26e5cce    delivered      2017-04-26 10:53:06   \n",
 683 |        "15  58dbd0b2d70206bf40e62cd34e84d795    delivered      2017-02-04 13:57:51   \n",
 684 |        "\n",
 685 |        "      order_approved_at order_delivered_carrier_date  \\\n",
 686 |        "0   2017-09-13 09:45:35          2017-09-19 18:34:16   \n",
 687 |        "1   2017-06-29 02:44:11          2017-07-05 12:00:33   \n",
 688 |        "2   2018-05-18 12:31:43          2018-05-23 14:05:00   \n",
 689 |        "9   2017-04-26 11:05:13          2017-05-04 14:35:00   \n",
 690 |        "15  2017-02-04 14:10:13          2017-02-16 09:46:09   \n",
 691 |        "\n",
 692 |        "   order_delivered_customer_date order_estimated_delivery_date  \\\n",
 693 |        "0            2017-09-20 23:43:48           2017-09-29 00:00:00   \n",
 694 |        "1            2017-07-13 20:39:29           2017-07-26 00:00:00   \n",
 695 |        "2            2018-06-04 18:34:26           2018-06-07 00:00:00   \n",
 696 |        "9            2017-05-12 16:04:24           2017-05-15 00:00:00   \n",
 697 |        "15           2017-03-01 16:42:31           2017-03-17 00:00:00   \n",
 698 |        "\n",
 699 |        "   product_category_name  \n",
 700 |        "0             cool_stuff  \n",
 701 |        "1             cool_stuff  \n",
 702 |        "2             cool_stuff  \n",
 703 |        "9               pet_shop  \n",
 704 |        "15    ferramentas_jardim  "
 705 |       ]
 706 |      },
 707 |      "execution_count": 84,
 708 |      "metadata": {},
 709 |      "output_type": "execute_result"
 710 |     }
 711 |    ],
 712 |    "source": [
 713 |     "# Orders where the shipping deadline to the carrier was missed\n",
 714 |     "df.loc[df.shipping_limit_date < df.order_delivered_carrier_date].head()\n",
 715 |     "\n"
 716 |    ]
 717 |   },
 718 |   {
 719 |    "cell_type": "code",
 720 |    "execution_count": 48,
 721 |    "metadata": {},
 722 |    "outputs": [
 723 |     {
 724 |      "data": {
 725 |       "text/html": [
 726 |        "<div>\n",
 727 |        "<style scoped>\n",
 728 |        "    .dataframe tbody tr th:only-of-type {\n",
 729 |        "        vertical-align: middle;\n",
 730 |        "    }\n",
 731 |        "\n",
 732 |        "    .dataframe tbody tr th {\n",
 733 |        "        vertical-align: top;\n",
 734 |        "    }\n",
 735 |        "\n",
 736 |        "    .dataframe thead th {\n",
 737 |        "        text-align: right;\n",
 738 |        "    }\n",
 739 |        "</style>\n",
 740 |        "<table border=\"1\" class=\"dataframe\">\n",
 741 |        "  <thead>\n",
 742 |        "    <tr style=\"text-align: right;\">\n",
 743 |        "      <th></th>\n",
 744 |        "      <th>order_id</th>\n",
 745 |        "      <th>order_item_id</th>\n",
 746 |        "      <th>product_id</th>\n",
 747 |        "      <th>seller_id</th>\n",
 748 |        "      <th>shipping_limit_date</th>\n",
 749 |        "      <th>price</th>\n",
 750 |        "      <th>freight_value</th>\n",
 751 |        "      <th>customer_id</th>\n",
 752 |        "      <th>order_purchase_timestamp</th>\n",
 753 |        "      <th>order_approved_at</th>\n",
 754 |        "      <th>order_delivered_carrier_date</th>\n",
 755 |        "      <th>order_delivered_customer_date</th>\n",
 756 |        "      <th>order_estimated_delivery_date</th>\n",
 757 |        "      <th>product_category_name</th>\n",
 758 |        "    </tr>\n",
 759 |        "    <tr>\n",
 760 |        "      <th>order_status</th>\n",
 761 |        "      <th></th>\n",
 762 |        "      <th></th>\n",
 763 |        "      <th></th>\n",
 764 |        "      <th></th>\n",
 765 |        "      <th></th>\n",
 766 |        "      <th></th>\n",
 767 |        "      <th></th>\n",
 768 |        "      <th></th>\n",
 769 |        "      <th></th>\n",
 770 |        "      <th></th>\n",
 771 |        "      <th></th>\n",
 772 |        "      <th></th>\n",
 773 |        "      <th></th>\n",
 774 |        "      <th></th>\n",
 775 |        "    </tr>\n",
 776 |        "  </thead>\n",
 777 |        "  <tbody>\n",
 778 |        "    <tr>\n",
 779 |        "      <th>approved</th>\n",
 780 |        "      <td>3</td>\n",
 781 |        "      <td>3</td>\n",
 782 |        "      <td>3</td>\n",
 783 |        "      <td>3</td>\n",
 784 |        "      <td>3</td>\n",
 785 |        "      <td>3</td>\n",
 786 |        "      <td>3</td>\n",
 787 |        "      <td>3</td>\n",
 788 |        "      <td>3</td>\n",
 789 |        "      <td>3</td>\n",
 790 |        "      <td>0</td>\n",
 791 |        "      <td>0</td>\n",
 792 |        "      <td>3</td>\n",
 793 |        "      <td>3</td>\n",
 794 |        "    </tr>\n",
 795 |        "    <tr>\n",
 796 |        "      <th>canceled</th>\n",
 797 |        "      <td>542</td>\n",
 798 |        "      <td>542</td>\n",
 799 |        "      <td>542</td>\n",
 800 |        "      <td>542</td>\n",
 801 |        "      <td>542</td>\n",
 802 |        "      <td>542</td>\n",
 803 |        "      <td>542</td>\n",
 804 |        "      <td>542</td>\n",
 805 |        "      <td>542</td>\n",
 806 |        "      <td>542</td>\n",
 807 |        "      <td>76</td>\n",
 808 |        "      <td>7</td>\n",
 809 |        "      <td>542</td>\n",
 810 |        "      <td>528</td>\n",
 811 |        "    </tr>\n",
 812 |        "    <tr>\n",
 813 |        "      <th>delivered</th>\n",
 814 |        "      <td>110197</td>\n",
 815 |        "      <td>110197</td>\n",
 816 |        "      <td>110197</td>\n",
 817 |        "      <td>110197</td>\n",
 818 |        "      <td>110197</td>\n",
 819 |        "      <td>110197</td>\n",
 820 |        "      <td>110197</td>\n",
 821 |        "      <td>110197</td>\n",
 822 |        "      <td>110197</td>\n",
 823 |        "      <td>110182</td>\n",
 824 |        "      <td>110195</td>\n",
 825 |        "      <td>110189</td>\n",
 826 |        "      <td>110197</td>\n",
 827 |        "      <td>108660</td>\n",
 828 |        "    </tr>\n",
 829 |        "    <tr>\n",
 830 |        "      <th>invoiced</th>\n",
 831 |        "      <td>359</td>\n",
 832 |        "      <td>359</td>\n",
 833 |        "      <td>359</td>\n",
 834 |        "      <td>359</td>\n",
 835 |        "      <td>359</td>\n",
 836 |        "      <td>359</td>\n",
 837 |        "      <td>359</td>\n",
 838 |        "      <td>359</td>\n",
 839 |        "      <td>359</td>\n",
 840 |        "      <td>359</td>\n",
 841 |        "      <td>0</td>\n",
 842 |        "      <td>0</td>\n",
 843 |        "      <td>359</td>\n",
 844 |        "      <td>347</td>\n",
 845 |        "    </tr>\n",
 846 |        "    <tr>\n",
 847 |        "      <th>processing</th>\n",
 848 |        "      <td>357</td>\n",
 849 |        "      <td>357</td>\n",
 850 |        "      <td>357</td>\n",
 851 |        "      <td>357</td>\n",
 852 |        "      <td>357</td>\n",
 853 |        "      <td>357</td>\n",
 854 |        "      <td>357</td>\n",
 855 |        "      <td>357</td>\n",
 856 |        "      <td>357</td>\n",
 857 |        "      <td>357</td>\n",
 858 |        "      <td>0</td>\n",
 859 |        "      <td>0</td>\n",
 860 |        "      <td>357</td>\n",
 861 |        "      <td>344</td>\n",
 862 |        "    </tr>\n",
 863 |        "    <tr>\n",
 864 |        "      <th>shipped</th>\n",
 865 |        "      <td>1185</td>\n",
 866 |        "      <td>1185</td>\n",
 867 |        "      <td>1185</td>\n",
 868 |        "      <td>1185</td>\n",
 869 |        "      <td>1185</td>\n",
 870 |        "      <td>1185</td>\n",
 871 |        "      <td>1185</td>\n",
 872 |        "      <td>1185</td>\n",
 873 |        "      <td>1185</td>\n",
 874 |        "      <td>1185</td>\n",
 875 |        "      <td>1185</td>\n",
 876 |        "      <td>0</td>\n",
 877 |        "      <td>1185</td>\n",
 878 |        "      <td>1158</td>\n",
 879 |        "    </tr>\n",
 880 |        "    <tr>\n",
 881 |        "      <th>unavailable</th>\n",
 882 |        "      <td>7</td>\n",
 883 |        "      <td>7</td>\n",
 884 |        "      <td>7</td>\n",
 885 |        "      <td>7</td>\n",
 886 |        "      <td>7</td>\n",
 887 |        "      <td>7</td>\n",
 888 |        "      <td>7</td>\n",
 889 |        "      <td>7</td>\n",
 890 |        "      <td>7</td>\n",
 891 |        "      <td>7</td>\n",
 892 |        "      <td>0</td>\n",
 893 |        "      <td>0</td>\n",
 894 |        "      <td>7</td>\n",
 895 |        "      <td>7</td>\n",
 896 |        "    </tr>\n",
 897 |        "  </tbody>\n",
 898 |        "</table>\n",
 899 |        "</div>"
 900 |       ],
 901 |       "text/plain": [
 902 |        "              order_id  order_item_id  product_id  seller_id  \\\n",
 903 |        "order_status                                                   \n",
 904 |        "approved             3              3           3          3   \n",
 905 |        "canceled           542            542         542        542   \n",
 906 |        "delivered       110197         110197      110197     110197   \n",
 907 |        "invoiced           359            359         359        359   \n",
 908 |        "processing         357            357         357        357   \n",
 909 |        "shipped           1185           1185        1185       1185   \n",
 910 |        "unavailable          7              7           7          7   \n",
 911 |        "\n",
 912 |        "              shipping_limit_date   price  freight_value  customer_id  \\\n",
 913 |        "order_status                                                            \n",
 914 |        "approved                        3       3              3            3   \n",
 915 |        "canceled                      542     542            542          542   \n",
 916 |        "delivered                  110197  110197         110197       110197   \n",
 917 |        "invoiced                      359     359            359          359   \n",
 918 |        "processing                    357     357            357          357   \n",
 919 |        "shipped                      1185    1185           1185         1185   \n",
 920 |        "unavailable                     7       7              7            7   \n",
 921 |        "\n",
 922 |        "              order_purchase_timestamp  order_approved_at  \\\n",
 923 |        "order_status                                                \n",
 924 |        "approved                             3                  3   \n",
 925 |        "canceled                           542                542   \n",
 926 |        "delivered                       110197             110182   \n",
 927 |        "invoiced                           359                359   \n",
 928 |        "processing                         357                357   \n",
 929 |        "shipped                           1185               1185   \n",
 930 |        "unavailable                          7                  7   \n",
 931 |        "\n",
 932 |        "              order_delivered_carrier_date  order_delivered_customer_date  \\\n",
 933 |        "order_status                                                                \n",
 934 |        "approved                                 0                              0   \n",
 935 |        "canceled                                76                              7   \n",
 936 |        "delivered                           110195                         110189   \n",
 937 |        "invoiced                                 0                              0   \n",
 938 |        "processing                               0                              0   \n",
 939 |        "shipped                               1185                              0   \n",
 940 |        "unavailable                              0                              0   \n",
 941 |        "\n",
 942 |        "              order_estimated_delivery_date  product_category_name  \n",
 943 |        "order_status                                                        \n",
 944 |        "approved                                  3                      3  \n",
 945 |        "canceled                                542                    528  \n",
 946 |        "delivered                            110197                 108660  \n",
 947 |        "invoiced                                359                    347  \n",
 948 |        "processing                              357                    344  \n",
 949 |        "shipped                                1185                   1158  \n",
 950 |        "unavailable                               7                      7  "
 951 |       ]
 952 |      },
 953 |      "execution_count": 48,
 954 |      "metadata": {},
 955 |      "output_type": "execute_result"
 956 |     }
 957 |    ],
 958 |    "source": [
 959 |     "df.groupby('order_status').count()"
 960 |    ]
 961 |   },
 962 |   {
 963 |    "cell_type": "code",
 964 |    "execution_count": 62,
 965 |    "metadata": {},
 966 |    "outputs": [
 967 |     {
 968 |      "name": "stdout",
 969 |      "output_type": "stream",
 970 |      "text": [
 971 |       "<pyspark.sql.session.SparkSession object at 0x1150fb470>\n"
 972 |      ]
 973 |     }
 974 |    ],
 975 |    "source": [
 976 |     "# Setting up spark\n",
 977 |     "import findspark\n",
 978 |     "findspark.init()\n",
 979 |     "from pyspark import SparkConf, SparkContext\n",
 980 |     "from pyspark.sql import SparkSession\n",
 981 |     "from pyspark.sql import *\n",
 982 |     "from pyspark.sql.functions import *\n",
 983 |     "conf = SparkConf().setMaster(\"local\").setAppName(\"SparkSQL_NLP\")\n",
 984 |     "spark = SparkSession.builder.getOrCreate()\n",
 985 |     "print(spark)"
 986 |    ]
 987 |   },
 988 |   {
 989 |    "cell_type": "code",
 990 |    "execution_count": 77,
 991 |    "metadata": {},
 992 |    "outputs": [
 993 |     {
 994 |      "data": {
 995 |       "text/plain": [
 996 |        "<module 'findspark' from '/Volumes/OWC_Big_Phatty/anaconda/lib/python3.6/site-packages/findspark.py'>"
 997 |       ]
 998 |      },
 999 |      "execution_count": 77,
1000 |      "metadata": {},
1001 |      "output_type": "execute_result"
1002 |     }
1003 |    ],
1004 |    "source": [
1005 |     "findspark"
1006 |    ]
1007 |   },
1008 |   {
1009 |    "cell_type": "code",
1010 |    "execution_count": 64,
1011 |    "metadata": {},
1012 |    "outputs": [],
1013 |    "source": [
1014 |     "# Set sqlContext from the Spark context\n",
1015 |     "from pyspark.sql import SQLContext\n",
1016 |     "sqlContext = SQLContext(spark)"
1017 |    ]
1018 |   },
1019 |   {
1020 |    "cell_type": "code",
1021 |    "execution_count": 65,
1022 |    "metadata": {},
1023 |    "outputs": [],
1024 |    "source": [
1025 |     "# Setup a Spark SQL context and read in the pandas dataframe to a Spark dataframe\n",
1026 |     "spark.conf.set(\"spark.sql.execution.arrow.enabled\", \"true\")"
1027 |    ]
1028 |   },
1029 |   {
1030 |    "cell_type": "code",
1031 |    "execution_count": 66,
1032 |    "metadata": {},
1033 |    "outputs": [],
1034 |    "source": [
1035 |     "spark = SparkSession.builder.getOrCreate()\n",
1036 |     "\n",
1037 |     "# Load in csv file into spark dataframe\n",
1038 |     "df_items = spark.read.format(\"csv\") \\\n",
1039 |     "            .option(\"header\", \"true\") \\\n",
1040 |     "            .option(\"inferSchema\", \"true\") \\\n",
1041 |     "            .load(\"olist_order_items_dataset.csv\")\n",
1042 |     "\n",
1043 |     "df_orders = spark.read.format(\"csv\") \\\n",
1044 |     "             .option(\"header\", \"true\") \\\n",
1045 |     "             .option(\"inferSchema\", \"true\") \\\n",
1046 |     "             .load(\"olist_orders_dataset.csv\")\n",
1047 |     "\n",
1048 |     "df_products = spark.read.format(\"csv\") \\\n",
1049 |     "             .option(\"header\", \"true\") \\\n",
1050 |     "             .option(\"inferSchema\", \"true\") \\\n",
1051 |     "             .load(\"olist_products_dataset.csv\")"
1052 |    ]
1053 |   },
1054 |   {
1055 |    "cell_type": "code",
1056 |    "execution_count": 67,
1057 |    "metadata": {},
1058 |    "outputs": [],
1059 |    "source": [
1060 |     "# Create SQL Tables from dfs\n",
1061 |     "df_items.createOrReplaceTempView('items')\n",
1062 |     "df_orders.createOrReplaceTempView('orders')\n",
1063 |     "df_products.createOrReplaceTempView('products')"
1064 |    ]
1065 |   },
1066 |   {
1067 |    "cell_type": "code",
1068 |    "execution_count": 69,
1069 |    "metadata": {},
1070 |    "outputs": [
1071 |     {
1072 |      "data": {
1073 |       "text/plain": [
1074 |        "['order_id',\n",
1075 |        " 'order_item_id',\n",
1076 |        " 'product_id',\n",
1077 |        " 'seller_id',\n",
1078 |        " 'shipping_limit_date',\n",
1079 |        " 'price',\n",
1080 |        " 'freight_value']"
1081 |       ]
1082 |      },
1083 |      "execution_count": 69,
1084 |      "metadata": {},
1085 |      "output_type": "execute_result"
1086 |     }
1087 |    ],
1088 |    "source": [
1089 |     "# Basic SQL query\n",
1090 |     "spark.sql('SELECT * FROM items LIMIT 0').columns"
1091 |    ]
1092 |   },
1093 |   {
1094 |    "cell_type": "code",
1095 |    "execution_count": 70,
1096 |    "metadata": {},
1097 |    "outputs": [
1098 |     {
1099 |      "data": {
1100 |       "text/plain": [
1101 |        "['product_id',\n",
1102 |        " 'product_category_name',\n",
1103 |        " 'product_name_lenght',\n",
1104 |        " 'product_description_lenght',\n",
1105 |        " 'product_photos_qty',\n",
1106 |        " 'product_weight_g',\n",
1107 |        " 'product_length_cm',\n",
1108 |        " 'product_height_cm',\n",
1109 |        " 'product_width_cm']"
1110 |       ]
1111 |      },
1112 |      "execution_count": 70,
1113 |      "metadata": {},
1114 |      "output_type": "execute_result"
1115 |     }
1116 |    ],
1117 |    "source": [
1118 |     "spark.sql('SELECT * FROM products LIMIT 0').columns"
1119 |    ]
1120 |   },
1121 |   {
1122 |    "cell_type": "code",
1123 |    "execution_count": 71,
1124 |    "metadata": {},
1125 |    "outputs": [
1126 |     {
1127 |      "data": {
1128 |       "text/plain": [
1129 |        "['order_id',\n",
1130 |        " 'customer_id',\n",
1131 |        " 'order_status',\n",
1132 |        " 'order_purchase_timestamp',\n",
1133 |        " 'order_approved_at',\n",
1134 |        " 'order_delivered_carrier_date',\n",
1135 |        " 'order_delivered_customer_date',\n",
1136 |        " 'order_estimated_delivery_date']"
1137 |       ]
1138 |      },
1139 |      "execution_count": 71,
1140 |      "metadata": {},
1141 |      "output_type": "execute_result"
1142 |     }
1143 |    ],
1144 |    "source": [
1145 |     "spark.sql('SELECT * FROM orders LIMIT 0').columns"
1146 |    ]
1147 |   },
1148 |   {
1149 |    "cell_type": "code",
1150 |    "execution_count": 76,
1151 |    "metadata": {},
1152 |    "outputs": [],
1153 |    "source": [
1154 |     "late_carrier_deliveries = spark.sql(\"\"\"\n",
1155 |     "SELECT i.order_id, i.seller_id, i.shipping_limit_date, i.price, i.freight_value,\n",
1156 |     "       p.product_id, p.product_category_name, \n",
1157 |     "       o.customer_id, o.order_status, o.order_purchase_timestamp, o.order_delivered_carrier_date,\n",
1158 |     "       o.order_delivered_customer_date, o.order_estimated_delivery_date\n",
1159 |     "FROM items AS i\n",
1160 |     "JOIN orders AS o\n",
1161 |     "ON i.order_id = o.order_id\n",
1162 |     "JOIN products AS p\n",
1163 |     "ON i.product_id = p.product_id\n",
1164 |     "WHERE i.shipping_limit_date < o.order_delivered_carrier_date\n",
1165 |     "\"\"\")"
1166 |    ]
1167 |   },
1168 |   {
1169 |    "cell_type": "code",
1170 |    "execution_count": 79,
1171 |    "metadata": {},
1172 |    "outputs": [
1173 |     {
1174 |      "name": "stdout",
1175 |      "output_type": "stream",
1176 |      "text": [
1177 |       "+--------------------+--------------------+-------------------+------+-------------+--------------------+---------------------+--------------------+------------+------------------------+----------------------------+-----------------------------+-----------------------------+\n",
1178 |       "|            order_id|           seller_id|shipping_limit_date| price|freight_value|          product_id|product_category_name|         customer_id|order_status|order_purchase_timestamp|order_delivered_carrier_date|order_delivered_customer_date|order_estimated_delivery_date|\n",
1179 |       "+--------------------+--------------------+-------------------+------+-------------+--------------------+---------------------+--------------------+------------+------------------------+----------------------------+-----------------------------+-----------------------------+\n",
1180 |       "|05afef1c185862cab...|53e4c6e0f4312d4d2...|2017-07-31 11:03:10| 27.99|        22.67|ac7e981115ad47f0e...| utilidades_domest...|296de103322e463a1...|   delivered|     2017-07-25 10:47:16|         2017-07-31 15:49:35|          2017-08-21 21:10:53|          2017-09-04 00:00:00|\n",
1181 |       "|05afef1c185862cab...|53e4c6e0f4312d4d2...|2017-07-31 11:03:10| 27.99|        22.67|ac7e981115ad47f0e...| utilidades_domest...|296de103322e463a1...|   delivered|     2017-07-25 10:47:16|         2017-07-31 15:49:35|          2017-08-21 21:10:53|          2017-09-04 00:00:00|\n",
1182 |       "|0b93ff37e8344c601...|b19f3ca2ea4759137...|2018-08-02 11:05:18| 998.9|       222.38|be7af429d53adfab1...|            papelaria|41b40faae6aeea7a8...|   delivered|     2018-07-22 11:13:05|         2018-08-02 16:31:00|          2018-08-07 22:11:10|          2018-08-21 00:00:00|\n",
1183 |       "|110fb617059394261...|9e6229250fedbe058...|2018-08-20 03:24:37| 169.9|        19.49|53a72a0b6feec4a5c...|                bebes|4e99e5342705c1e76...|     shipped|     2018-08-08 16:37:28|         2018-08-20 10:57:00|                         null|          2018-08-28 00:00:00|\n",
1184 |       "|112a61419198a7ce9...|ffdd9f82b9a447f6f...|2018-04-11 17:09:50| 214.0|        70.89|55aec8e90307dc2a7...| utilidades_domest...|dad5e5f178c09543d...|   delivered|     2018-04-05 11:13:05|         2018-04-11 23:33:01|          2018-04-13 23:41:40|          2018-04-24 00:00:00|\n",
1185 |       "|184f17bb701af22b8...|827f8f69dfa529c56...|2017-05-11 18:15:15|  29.9|         14.1|de4a736d3f2df1266...|     fashion_calcados|d6d1c551a52cf82ad...|   delivered|     2017-05-06 18:04:27|         2017-05-15 13:16:14|          2017-05-17 14:44:53|          2017-05-31 00:00:00|\n",
1186 |       "|1c7b6d63329bdc850...|88460e8ebdecbfecb...|2018-04-19 13:31:39|  73.9|         7.87|ee57070aa3b24a06f...| informatica_acess...|2b8a8ed9920523186...|   delivered|     2018-04-13 12:59:58|         2018-04-26 12:59:00|          2018-04-27 19:22:00|          2018-04-26 00:00:00|\n",
1187 |       "|235d0ef0d8ee3f66a...|213b25e6f54661939...|2018-08-03 03:55:19|119.85|        11.86|6f0169f259bb0ff43...|      casa_construcao|fa4bc396185415281...|   delivered|     2018-07-20 13:58:55|         2018-08-03 14:35:00|          2018-08-06 14:56:49|          2018-08-07 00:00:00|\n",
1188 |       "|2b9c1a6934f2dd62c...|88460e8ebdecbfecb...|2018-02-15 20:55:49|  74.9|        12.65|e53e557d5a159f5aa...| informatica_acess...|82c5e6b98e498b415...|   delivered|     2018-02-10 20:46:25|         2018-02-21 10:19:22|          2018-02-28 21:49:02|          2018-03-14 00:00:00|\n",
1189 |       "|3303092810c37e9ab...|cd843d4cf8ef32827...|2017-08-03 10:50:12| 469.0|        90.26|df872c596e00cd016...| moveis_cozinha_ar...|5db727761e58376d9...|   delivered|     2017-07-28 10:36:12|         2017-08-03 17:25:30|          2017-08-15 19:05:13|          2017-08-21 00:00:00|\n",
1190 |       "|35f4b9d2da608b318...|88460e8ebdecbfecb...|2018-01-02 10:06:22| 109.9|        34.57|e53e557d5a159f5aa...| informatica_acess...|04b07985c482f53e0...|   delivered|     2017-12-26 09:49:11|         2018-01-11 23:49:29|          2018-01-24 00:33:55|          2018-01-26 00:00:00|\n",
1191 |       "|38541d08d4eb7d571...|54965bbe3e4f07ae0...|2017-11-30 00:52:38|  69.9|        26.81|9d6c66334a27b9dcf...|      cama_mesa_banho|937682b4176cf634c...|     shipped|     2017-11-24 21:36:30|         2018-01-04 21:07:51|                         null|          2017-12-20 00:00:00|\n",
1192 |       "|3e5d60bbe5a6016db...|0adac9fbd9a2b63cc...|2017-12-11 09:22:02|  10.9|        11.85|293637be1a5ab3bd1...| utilidades_domest...|46ec73876cebebf34...|   delivered|     2017-12-04 22:14:17|         2017-12-12 21:22:34|          2017-12-26 21:36:57|          2017-12-27 00:00:00|\n",
1193 |       "|3f090361c1e3266bf...|c1849d4d32d7a6cec...|2018-08-10 11:50:23| 19.99|         8.89|39251fd407f6dc1db...|           automotivo|c892d1124348cf471...|   delivered|     2018-08-06 11:38:43|         2018-08-10 12:07:00|          2018-08-11 16:12:32|          2018-08-13 00:00:00|\n",
1194 |       "|41537821ce113ccef...|b2ba3715d723d2451...|2017-12-18 04:09:14|  79.9|         9.37|8b8981e920b8a923f...| construcao_ferram...|0312590499e722678...|   delivered|     2017-12-11 13:56:03|         2017-12-18 21:33:03|          2017-12-20 17:29:03|          2017-12-28 00:00:00|\n",
1195 |       "+--------------------+--------------------+-------------------+------+-------------+--------------------+---------------------+--------------------+------------+------------------------+----------------------------+-----------------------------+-----------------------------+\n",
1196 |       "only showing top 15 rows\n",
1197 |       "\n"
1198 |      ]
1199 |     }
1200 |    ],
1201 |    "source": [
1202 |     "late_carrier_deliveries.show(15)"
1203 |    ]
1204 |   },
1205 |   {
1206 |    "cell_type": "code",
1207 |    "execution_count": 83,
1208 |    "metadata": {},
1209 |    "outputs": [],
1210 |    "source": [
1211 |     "# Write the results to a single csv file\n",
1212 |     "# Note the named output is a directory as opposed to the csv filename\n",
1213 |     "# This is because Spark uses HDFS naming conventions for files\n",
1214 |     "late_carrier_deliveries.coalesce(1).write.option(\"header\", \"true\").csv(\"missed_shipping_limit_orders\")"
1215 |    ]
1216 |   },
1217 |   {
1218 |    "cell_type": "code",
1219 |    "execution_count": null,
1220 |    "metadata": {},
1221 |    "outputs": [],
1222 |    "source": []
1223 |   }
1224 |  ],
1225 |  "metadata": {
1226 |   "kernelspec": {
1227 |    "display_name": "Python 3",
1228 |    "language": "python",
1229 |    "name": "python3"
1230 |   },
1231 |   "language_info": {
1232 |    "codemirror_mode": {
1233 |     "name": "ipython",
1234 |     "version": 3
1235 |    },
1236 |    "file_extension": ".py",
1237 |    "mimetype": "text/x-python",
1238 |    "name": "python",
1239 |    "nbconvert_exporter": "python",
1240 |    "pygments_lexer": "ipython3",
1241 |    "version": "3.6.5"
1242 |   }
1243 |  },
1244 |  "nbformat": 4,
1245 |  "nbformat_minor": 2
1246 | }
1247 | 


--------------------------------------------------------------------------------
/Data/brazilian-ecommerce.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ajupton/big-data-engineering-project/c32a6fa746d6053d39898d3881d8a013a6f46689/Data/brazilian-ecommerce.zip


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Andy Upton
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # 'Little' Big Data Engineering Project
 2 | Hey there! Welcome to this repo where I practice building a data engineering pipeline. Here is a basic run down of the project:
 3 | 
 4 | # Problem Statement
 5 | Retailers in the current landscape are adapting to the digital age. Digital retail behemoths have carved out substantial market shares in the online space at the same time that traditional retail stores are broadly in decline. In this time of digital flux, an omni-channel retail approach is necessary to keep pace. This is especially true for retailers that have invested in an extensive brick-and-mortar store portfolio or have strong relationships with brick-and-mortar partners. 
 6 | 
 7 | This data engineering project uses a real world retail dataset to explore delivery performance at scale. The primary concern of data engineering efforts in this project is to create a strong foundation onto which data analytics and modeling may be applied as well as provide summary reports for daily ingestion by decision makers. 
 8 | 
 9 | A series of ETL jobs are programmed as part of this project using python, SQL, Airflow, and Spark to build pipelines that download data from an AWS S3 bucket, apply some manipulations, and then load the cleaned-up data set into another location on the same AWS S3 bucket for higher level analytics. 
10 | 
11 | # Dataset of choice
12 | The dataset of choice for this project is a series of tables [provided by the Brazilian Ecommerce company Olist](https://www.kaggle.com/olistbr/brazilian-ecommerce/home#olist_orders_dataset.csvhttps://www.kaggle.com/olistbr/brazilian-ecommerce/home#olist_orders_dataset.csv). The table schema is provided here:
13 | 
14 | ![](https://i.imgur.com/HRhd2Y0.png)
15 | 
16 | # Methodology:
17 | 1. Construct a mock production data lake in AWS S3 replete with the table schema above
18 | 2. Analyze the tables with an eye toward identifying the delivery performance of Olist orders/sellers 
19 | 3. Write a Spark and Spark SQL job to join together tables answering the question, "Which orders/sellers missed the deadline imposed by Olist for when their packages need to be delivered to a carrier?"
20 | 4. Build an ETL pipeline using Airflow that accomplishes the following:
21 | * Downloads data from an AWS S3 bucket
22 | * Runs a Spark/Spark SQL job on the downloaded data producing a cleaned-up dataset of delivery deadline missing orders
23 | * Upload the cleaned-up dataset back to the same S3 bucket in a folder primed for higher level analytics
24 | 
25 | I'll provide an overview of each of these steps but assume that the reader is already somewhat familiar with python, SQL, Airflow, and Spark. 
26 | 
27 | ## Step 1: Setup and populate an AWS S3 Bucket
28 | Setting up an S3 bucket is pretty straightforward, and the current 5 GB free tier limit for 12 months is a great way to get started with AWS S3 without having to break the bank. Simply create an AWS account, click on "Services" in the upper left hand side, navigate to "Storage" and select S3. From there you'll be able to create a new S3 bucket following the prompts and inputing settings based on your unique needs. I largely followed the default options. Once created, simply select the bucket and in the Overview tab, there are options to upload objects/files, create folders, or set permissions. Alternatively, it's possible to install AWS Cli to your machine in order to transfer files at the command line. 
29 | 
30 | The method for interacting with S3 taken here uses the [python library boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html). boto3 has all sorts of functions for interacting with products across the AWS landscape using python. 
31 | 
32 | We'll use boto3 as part of python scripts to both download the Brazilian ecommerce data to our S3 bucket as well upload the csv file that results from the Spark job back into the S3 bucket. 
33 | 
34 | In reality, a data lake often consists of various SQL tables or log files, which are often billions of rows long. The Brazilian ecommerce data used here comes in a zip file that contains 9 different csv files as seen schema above. This gets sort of close to a data lake format but in a way that's amenable for practicing and learning. 
35 | 
36 | ## Step 2: Analyze the tables with an eye toward identifying the delivery performance of Olist orders/sellers 
37 | One of the strongest skillsets needed by a data engineer is communication. That communication needs to be flexed primarily in conversations with stakeholders that manage data sets as well as stakeholders that will consume the data at the end of the ETL and stream pipelines that data engineers construct. Data engineers often need to rely on the domain expertise of analysts, data scientists, program managers, executives, and others in order to understand what specific data features are needed for various analytics applications. That said, data engineers still need to possess strong EDA (exploratory data analysis) chops in order to deliver data sets that meet the needs of various stakeholders. As a result, it's very often worthwhile for data engineers to get down and dirty with the data in the data lake through data visualization, the computing of summary statistics, and other exploratory methods. 
38 | 
39 | My go-to tool of choice in this regard is a Jupyter notebook using python. A typical workflow involes using SQLAlchemy or the pyspark API to connect to a database and pull data with some pythonic SQL and then use numpy, pandas, altair/seaborn/matplotlib/plotly, and other python packages for visualization, the computing of summary statitistics, maybe running a few statistical models, and other EDA techniques. For data that is truly 'big' and can't fit in memory on a single machine, Spark becomes invaluable. Spark SQL makes pulling and manipulating data a breeze as well for those with SQL chops. Of course, this analytics approach is possible in all sorts of AWS services like Athena and EMR among many others. With massive volumes of data, it can often be worthwhile to take samples of data from the data lake and get some EDA in to get a feel for the tables and schema as well as to understand the quirks of the data. OLTP processes often must follow multiple business rules that make it a major challenge to do things like join disparate datasets, hammer down ground-truth insights, or compare current and historical trends.  
40 | 
41 | Some very basic EDA is applied on the database tables to answer the question "Which orders/sellers missed the deadline imposed by Olist for when their packages need to be delivered to a carrier?". [See the code here or navigate to `Data` -> Brazilian ecommerce.ipynb](https://github.com/ajupton/big-data-engineering-project/blob/master/Data/Brazilian%20ecommerce%20EDA%20.ipynb)
42 | 
43 | ## Step 3: Get hands-on with Airflow 
44 | There are many tools out there to run big data ETL jobs that turn messy data lakes into analytics-ready data warehouses. Here, I'll focus on one of the most popular open source ETL tools currently available - Apache Airflow. [Check out the Airflow docs here](https://airflow.apache.org/index.html). First, install Airflow  with `pip install apache-airflow`. Make sure to include the 'apache' part there, or you'll download an older version of Airflow that will lead to a whole lot of problems down the line. 
45 | 
46 | _Note:_ It can often be worthwhile to run Airflow in a Docker container. There are some major advantages to using a Docker container such as: 
47 | * creating a fully reproducible data analysis in Jupyter notebooks or other dev platforms (something that's less trivial than you might think) 
48 | * having fully documented dependencies (a Dockerfile contains descriptions of all the packages/files/stuff that your container needs to run) 
49 | * having an isolated environment to work in that ensures your tools don't conflict with each other AND recreates the conditions in development, testing, staging, or production environments to ensure your code will run as expected
50 | * not having to start from scratch by taking advantage of DockerHub container 'templates' 
51 | 
52 | Here are a few resources to help you get started with Airflow:
53 | * [Airflow quickstart guide](http://airflow.apache.org/start.html)
54 | * [ETL Best Practices with Airflow](https://gtoonstra.github.io/etl-with-airflow/index.html)
55 | * [Developing Workflows with Apache Airflow](http://michal.karzynski.pl/blog/2017/03/19/developing-workflows-with-apache-airflow/)
56 | * [Getting Started with Airflow Using Docker](https://towardsdatascience.com/getting-started-with-airflow-using-docker-cd8b44dbff98)
57 | * [Basic DAG Configuration](https://adataguru.net/basic-dag-configuration/)
58 | 
59 | Once you have Airflow installed and are able to run through a HelloWorld dag, you're ready for the next step. 
60 | 
61 | Airflow is all about writing python scripts to create data pipelines. There are two primary scripts that need to be written for this purpose. The first are dags and the second are operators. No analytics or processing occurs in dags, which encompass the processes or steps involved in developing a pipeline. The actual analytics live in various scripts that are run as per instructed in the dags. 
62 | 
63 | [The late_shipments_to_carrier.py dag includes the three steps needed to complete our pipeline as defined above.]( https://github.com/ajupton/big-data-engineering-project/blob/master/airflow/dags/late_shipments_to_carrier_dag.py)
64 | 
65 | The first step is downloading the brazilian-ecommerce.zip file from your S3 bucket. [The script to accomplish this task is found here.](https://github.com/ajupton/big-data-engineering-project/blob/master/airflow/scripts/s3_download.py)
66 | 
67 | The next step is to run a Spark SQL job to do a pretty simply join of three relations and filter for orders/sellers that missed the delivery deadline to get their package to a designated carrier for shipment to the consumer. [This script first unzips the dataset, then sets up a Spark session, runs a simple Spark SQL operation, and then writes the results of the Spark SQL operation to a single csv file.](https://github.com/ajupton/big-data-engineering-project/blob/master/airflow/scripts/spark_missed_deadline_job.py)
68 | 
69 | Finally, the dataset identifying orders that missed the carrier delivery deadline is uploaded to the same S3 bucket in a different folder. [This script also screens out non-csv files from being uploaded to keep the folder fairly clean.](https://github.com/ajupton/big-data-engineering-project/blob/master/airflow/scripts/s3_upload.py)
70 | 
71 | To run the job, make sure to first edit the paths of each of the scripts to match the paths where you'd like to run your analysis on your own machine and of course make sure to include the specific details of your S3 bucket. 
72 | 
73 | One thing to note about the scripts is the `.set_upstream()` method applied to the second two operators. This ensures that if, for any reason, the initial file download fails that Airflow will retry the jobs. Another thing to note about the dag is the schedule, which I'm manually triggering using the Airflow UI. There's a lot more depth to job scheduling. 
74 | 
75 | But there you have it! This is a pretty simple pipeline, and shows how powerful Airflow can be in its ability to schedule various jobs using a variety of technologies like python and Spark. This only scratches the surface of what's capable with Airflow. 
76 | 
77 | The next step in this process is incorporating AWS EMR into the pipeline to run the Spark job on a cluster instead of locally on my machine. Stay tuned for updates!
78 | 


--------------------------------------------------------------------------------
/airflow/dags/late_shipments_to_carrier_dag.py:
--------------------------------------------------------------------------------
 1 | from datetime import datetime
 2 | from airflow import DAG
 3 | from airflow.operators.bash_operator import BashOperator
 4 | 
 5 | dag = DAG('late_shipments_to_carrier',
 6 |           description='Returns list of orders where the seller missed the carrier delivery deadline',
 7 |           schedule_interval='0 5 * * *',
 8 |           start_date=datetime(2019, 7, 10), catchup=False)
 9 | 
10 | # Download the data from S3
11 | s3_download_operator = BashOperator(task_id='s3_download',
12 |                                     bash_command='python /path/to/airflow/scripts/s3_download.py', ##<<<< edit path!!
13 |                                     dag=dag)
14 | 
15 | # Run Spark job to return order information where the seller
16 | # missed the deadline to deliver the shipment to the carrier
17 | spark_missed_deadline_operator = BashOperator(task_id='spark_missed_deadline_job',
18 |                                               bash_command='python /path/to/airflow/scripts/spark_missed_deadline_job.py', ##<<<< edit path!!
19 |                                               dag=dag)
20 | 
21 | # Specify that the Spark task above depends on the dataset downloading properly
22 | spark_missed_deadline_operator.set_upstream(s3_download_operator)
23 | 
24 | # Upload cleaned dataset to S3
25 | s3_upload_operator = BashOperator(task_id='s3_upload',
26 |                                   bash_command='python /path/to/airflow/scripts/s3_upload.py', ##<<<< edit path!!
27 |                                   dag=dag)
28 | 
29 | # Specify that the S3 upload task depends on the Spark job running successfully
30 | s3_upload_operator.set_upstream(spark_missed_deadline_operator)
31 | 
32 | s3_download_operator >> spark_missed_deadline_operator >> s3_upload_operator
33 | 


--------------------------------------------------------------------------------
/airflow/scripts/s3_download.py:
--------------------------------------------------------------------------------
 1 | from boto3.s3.transfer import S3Transfer
 2 | import boto3
 3 | 
 4 | ########################################
 5 | ## Edit the keys/paths for your setup ##
 6 | ########################################
 7 | access_key = 'your_access_key_here'
 8 | secret_key = 'your_secret_key_here'
 9 | s3_bucket_name = 'your_s3_bucket_name'
10 | s3_filename = 'brazilian-ecommerce.zip'
11 | download_path = '/path/to/brazilian-ecommerce.zip'
12 | 
13 | client = boto3.client('s3',
14 |                       aws_access_key_id = access_key,
15 |                       aws_secret_access_key = secret_key)
16 | 
17 | print('client')
18 | 
19 | client.download_file(s3_bucket_name, s3_filename, download_path)
20 | 


--------------------------------------------------------------------------------
/airflow/scripts/s3_upload.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from boto3.s3.transfer import S3Transfer
 3 | import boto3
 4 | 
 5 | ########################################
 6 | ## Edit the keys/paths for your setup ##
 7 | ########################################
 8 | access_key = 'your_access_key_here'
 9 | secret_key = 'your_secret_key_here'
10 | s3_bucket_name = 'your_s3_bucket_name'
11 | s3_filename = 'brazilian-ecommerce.zip'
12 | s3_filename = 'missed_shipping_limit_orders.csv'
13 | client = boto3.client('s3',
14 |                       aws_access_key_id = access_key,
15 |                       aws_secret_access_key = secret_key)
16 | 
17 | print('client')
18 | 
19 | transfer = S3Transfer(client)
20 | 
21 | print('transfer - ' + s3_bucket_name)
22 | 
23 | # Define function to scan through the Spark output uploadDirectory,
24 | # identify csv files, and upload them to the S3 bucket
25 | def uploadDirectory(filepath, s3_bucket_name):
26 |     for root, dirs, files in os.walk(filepath):
27 |         for file in files:
28 |             # Transfer only csv files
29 |             if file.endswith('csv'):
30 |                 transfer.upload_file(os.path.join(root, file),
31 |                                      s3_bucket_name,
32 |                                      "Clean_Data/" + file) # File put into Clean-Data folder
33 | 
34 | uploadDirectory(filepath = filepath, s3_bucket_name = s3_bucket_name)
35 | 


--------------------------------------------------------------------------------
/airflow/scripts/spark_missed_deadline_job.py:
--------------------------------------------------------------------------------
 1 | # First, unzip the file with the Olist ecommerce data
 2 | from zipfile import ZipFile
 3 | 
 4 | # Create a ZipFile Object and load brazilian-ecommerce.zip in it
 5 | with ZipFile('/path/to/Brazilian-ecommerce.zip', #<<<<<<<< edit path location where you saved the file!!!
 6 |              'r') as zipObj:
 7 |     # Extract all the contents of zip file in current directory
 8 |     zipObj.extractall()
 9 | 
10 | # Setting up spark
11 | import findspark
12 | findspark.init()
13 | from pyspark import SparkConf, SparkContext
14 | from pyspark.sql import SparkSession
15 | from pyspark.sql import *
16 | from pyspark.sql.functions import *
17 | conf = SparkConf().setMaster("local").setAppName("Missed_Deadlines")
18 | spark = SparkSession.builder.getOrCreate()
19 | print(spark)
20 | 
21 | # Set sqlContext from the Spark context
22 | from pyspark.sql import SQLContext
23 | sqlContext = SQLContext(spark)
24 | 
25 | # Edit Spark SQL context for ease of use with Pandas
26 | spark.conf.set("spark.sql.execution.arrow.enabled", "true")
27 | 
28 | # Build a spark session
29 | spark = SparkSession.builder.getOrCreate()
30 | 
31 | # Load in csv files into spark dataframes
32 | df_items = spark.read.format("csv") \
33 |             .option("header", "true") \
34 |             .option("inferSchema", "true") \
35 |             .load("olist_order_items_dataset.csv")
36 | 
37 | df_orders = spark.read.format("csv") \
38 |              .option("header", "true") \
39 |              .option("inferSchema", "true") \
40 |              .load("olist_orders_dataset.csv")
41 | 
42 | df_products = spark.read.format("csv") \
43 |                 .option("header", "true") \
44 |                 .option("inferSchema", "true") \
45 |                 .load("olist_products_dataset.csv")
46 | 
47 | # Create SQL Table Views from dfs for SQL querying
48 | df_items.createOrReplaceTempView('items')
49 | df_orders.createOrReplaceTempView('orders')
50 | df_products.createOrReplaceTempView('products')
51 | 
52 | # SQL Query to pull order/seller/product info for orders where the
53 | # seller missed the deadline to deliver the shipment to the carrier
54 | late_carrier_deliveries = spark.sql("""
55 | SELECT i.order_id, i.seller_id, i.shipping_limit_date, i.price, i.freight_value,
56 |        p.product_id, p.product_category_name,
57 |        o.customer_id, o.order_status, o.order_purchase_timestamp, o.order_delivered_carrier_date,
58 |        o.order_delivered_customer_date, o.order_estimated_delivery_date
59 | FROM items AS i
60 | JOIN orders AS o
61 | ON i.order_id = o.order_id
62 | JOIN products AS p
63 | ON i.product_id = p.product_id
64 | WHERE i.shipping_limit_date < o.order_delivered_carrier_date
65 | """)
66 | 
67 | # Write the results to a single csv file
68 | # coalesce(1) requires that the file is small enough to fit
69 | # in the heap memory of the master Spark node and is therefore
70 | # only recommended for very small datasets
71 | # Alternatives are converting the Spark df to a Pandas df before
72 | # writing to disk.
73 | # Otherwise, it's best practice to maintain the partitions to
74 | # take advantage of HDFS
75 | late_carrier_deliveries.coalesce(1) \
76 |                        .write \
77 |                        .option("header", "true") \
78 |                        .csv("/path/where/you/want/missed_shipping_limit_orders.csv") # <<<<<<< edit path
79 | 


--------------------------------------------------------------------------------