"
64 | ]
65 | },
66 | "execution_count": 3,
67 | "metadata": {},
68 | "output_type": "execute_result"
69 | }
70 | ],
71 | "source": [
72 | "spark"
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": 4,
78 | "id": "4d3bd081",
79 | "metadata": {},
80 | "outputs": [],
81 | "source": [
82 | "df_pyspark=spark.read.csv('test3.csv',header=True,inferSchema=True)"
83 | ]
84 | },
85 | {
86 | "cell_type": "code",
87 | "execution_count": 5,
88 | "id": "7ed791ed",
89 | "metadata": {},
90 | "outputs": [
91 | {
92 | "name": "stdout",
93 | "output_type": "stream",
94 | "text": [
95 | "+---------+------------+------+\n",
96 | "| Name| Departments|salary|\n",
97 | "+---------+------------+------+\n",
98 | "| Krish|Data Science| 10000|\n",
99 | "| Krish| IOT| 5000|\n",
100 | "| Mahesh| Big Data| 4000|\n",
101 | "| Krish| Big Data| 4000|\n",
102 | "| Mahesh|Data Science| 3000|\n",
103 | "|Sudhanshu|Data Science| 20000|\n",
104 | "|Sudhanshu| IOT| 10000|\n",
105 | "|Sudhanshu| Big Data| 5000|\n",
106 | "| Sunny|Data Science| 10000|\n",
107 | "| Sunny| Big Data| 2000|\n",
108 | "+---------+------------+------+\n",
109 | "\n"
110 | ]
111 | }
112 | ],
113 | "source": [
114 | "df_pyspark.show()"
115 | ]
116 | },
117 | {
118 | "cell_type": "code",
119 | "execution_count": 6,
120 | "id": "d57d24ca",
121 | "metadata": {},
122 | "outputs": [
123 | {
124 | "name": "stdout",
125 | "output_type": "stream",
126 | "text": [
127 | "root\n",
128 | " |-- Name: string (nullable = true)\n",
129 | " |-- Departments: string (nullable = true)\n",
130 | " |-- salary: integer (nullable = true)\n",
131 | "\n"
132 | ]
133 | }
134 | ],
135 | "source": [
136 | "df_pyspark.printSchema()"
137 | ]
138 | },
139 | {
140 | "cell_type": "code",
141 | "execution_count": 9,
142 | "id": "f15f8197",
143 | "metadata": {},
144 | "outputs": [
145 | {
146 | "name": "stdout",
147 | "output_type": "stream",
148 | "text": [
149 | "+---------+-----------+\n",
150 | "| Name|sum(salary)|\n",
151 | "+---------+-----------+\n",
152 | "|Sudhanshu| 35000|\n",
153 | "| Sunny| 12000|\n",
154 | "| Krish| 19000|\n",
155 | "| Mahesh| 7000|\n",
156 | "+---------+-----------+\n",
157 | "\n"
158 | ]
159 | }
160 | ],
161 | "source": [
162 | "## Groupby\n",
163 | "### Grouped to find the maximum salary\n",
164 | "df_pyspark.groupBy('Name').sum().show()"
165 | ]
166 | },
167 | {
168 | "cell_type": "code",
169 | "execution_count": 19,
170 | "id": "fc122ace",
171 | "metadata": {},
172 | "outputs": [
173 | {
174 | "name": "stdout",
175 | "output_type": "stream",
176 | "text": [
177 | "+---------+------------------+\n",
178 | "| Name| avg(salary)|\n",
179 | "+---------+------------------+\n",
180 | "|Sudhanshu|11666.666666666666|\n",
181 | "| Sunny| 6000.0|\n",
182 | "| Krish| 6333.333333333333|\n",
183 | "| Mahesh| 3500.0|\n",
184 | "+---------+------------------+\n",
185 | "\n"
186 | ]
187 | }
188 | ],
189 | "source": [
190 | "df_pyspark.groupBy('Name').avg().show()"
191 | ]
192 | },
193 | {
194 | "cell_type": "code",
195 | "execution_count": 11,
196 | "id": "151d2264",
197 | "metadata": {},
198 | "outputs": [
199 | {
200 | "name": "stdout",
201 | "output_type": "stream",
202 | "text": [
203 | "+------------+-----------+\n",
204 | "| Departments|sum(salary)|\n",
205 | "+------------+-----------+\n",
206 | "| IOT| 15000|\n",
207 | "| Big Data| 15000|\n",
208 | "|Data Science| 43000|\n",
209 | "+------------+-----------+\n",
210 | "\n"
211 | ]
212 | }
213 | ],
214 | "source": [
215 | "### Groupby Departmernts which gives maximum salary\n",
216 | "df_pyspark.groupBy('Departments').sum().show()"
217 | ]
218 | },
219 | {
220 | "cell_type": "code",
221 | "execution_count": 12,
222 | "id": "66fe5552",
223 | "metadata": {},
224 | "outputs": [
225 | {
226 | "name": "stdout",
227 | "output_type": "stream",
228 | "text": [
229 | "+------------+-----------+\n",
230 | "| Departments|avg(salary)|\n",
231 | "+------------+-----------+\n",
232 | "| IOT| 7500.0|\n",
233 | "| Big Data| 3750.0|\n",
234 | "|Data Science| 10750.0|\n",
235 | "+------------+-----------+\n",
236 | "\n"
237 | ]
238 | }
239 | ],
240 | "source": [
241 | "df_pyspark.groupBy('Departments').mean().show()"
242 | ]
243 | },
244 | {
245 | "cell_type": "code",
246 | "execution_count": 14,
247 | "id": "bc7bf192",
248 | "metadata": {},
249 | "outputs": [
250 | {
251 | "name": "stdout",
252 | "output_type": "stream",
253 | "text": [
254 | "+------------+-----+\n",
255 | "| Departments|count|\n",
256 | "+------------+-----+\n",
257 | "| IOT| 2|\n",
258 | "| Big Data| 4|\n",
259 | "|Data Science| 4|\n",
260 | "+------------+-----+\n",
261 | "\n"
262 | ]
263 | }
264 | ],
265 | "source": [
266 | "df_pyspark.groupBy('Departments').count().show()"
267 | ]
268 | },
269 | {
270 | "cell_type": "code",
271 | "execution_count": 15,
272 | "id": "37b26cbe",
273 | "metadata": {},
274 | "outputs": [
275 | {
276 | "name": "stdout",
277 | "output_type": "stream",
278 | "text": [
279 | "+-----------+\n",
280 | "|sum(Salary)|\n",
281 | "+-----------+\n",
282 | "| 73000|\n",
283 | "+-----------+\n",
284 | "\n"
285 | ]
286 | }
287 | ],
288 | "source": [
289 | "df_pyspark.agg({'Salary':'sum'}).show()"
290 | ]
291 | },
292 | {
293 | "cell_type": "code",
294 | "execution_count": null,
295 | "id": "bb21f03f",
296 | "metadata": {},
297 | "outputs": [],
298 | "source": []
299 | },
300 | {
301 | "cell_type": "code",
302 | "execution_count": null,
303 | "id": "1c7d8f83",
304 | "metadata": {},
305 | "outputs": [],
306 | "source": []
307 | },
308 | {
309 | "cell_type": "code",
310 | "execution_count": null,
311 | "id": "9dc7aa65",
312 | "metadata": {},
313 | "outputs": [],
314 | "source": []
315 | },
316 | {
317 | "cell_type": "code",
318 | "execution_count": null,
319 | "id": "fdd3fbac",
320 | "metadata": {},
321 | "outputs": [],
322 | "source": []
323 | },
324 | {
325 | "cell_type": "code",
326 | "execution_count": null,
327 | "id": "375a6fda",
328 | "metadata": {},
329 | "outputs": [],
330 | "source": []
331 | },
332 | {
333 | "cell_type": "code",
334 | "execution_count": null,
335 | "id": "0aa434e6",
336 | "metadata": {},
337 | "outputs": [],
338 | "source": []
339 | },
340 | {
341 | "cell_type": "code",
342 | "execution_count": null,
343 | "id": "c82781ad",
344 | "metadata": {},
345 | "outputs": [],
346 | "source": []
347 | }
348 | ],
349 | "metadata": {
350 | "kernelspec": {
351 | "display_name": "Python 3",
352 | "language": "python",
353 | "name": "python3"
354 | },
355 | "language_info": {
356 | "codemirror_mode": {
357 | "name": "ipython",
358 | "version": 3
359 | },
360 | "file_extension": ".py",
361 | "mimetype": "text/x-python",
362 | "name": "python",
363 | "nbconvert_exporter": "python",
364 | "pygments_lexer": "ipython3",
365 | "version": "3.7.10"
366 | }
367 | },
368 | "nbformat": 4,
369 | "nbformat_minor": 5
370 | }
371 |
--------------------------------------------------------------------------------
/Tutorial 6-Example Of Pyspark ML.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "16da6c54",
6 | "metadata": {},
7 | "source": [
8 | "### Examples Of Pyspark ML"
9 | ]
10 | },
11 | {
12 | "cell_type": "code",
13 | "execution_count": 71,
14 | "id": "0b9da3ad",
15 | "metadata": {},
16 | "outputs": [],
17 | "source": [
18 | "from pyspark.sql import SparkSession\n",
19 | "spark=SparkSession.builder.appName('Missing').getOrCreate()"
20 | ]
21 | },
22 | {
23 | "cell_type": "code",
24 | "execution_count": 72,
25 | "id": "735525da",
26 | "metadata": {},
27 | "outputs": [],
28 | "source": [
29 | "## Read The dataset\n",
30 | "training = spark.read.csv('test1.csv',header=True,inferSchema=True)"
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "execution_count": 73,
36 | "id": "d6e038c9",
37 | "metadata": {},
38 | "outputs": [
39 | {
40 | "name": "stdout",
41 | "output_type": "stream",
42 | "text": [
43 | "+---------+---+----------+------+\n",
44 | "| Name|age|Experience|Salary|\n",
45 | "+---------+---+----------+------+\n",
46 | "| Krish| 31| 10| 30000|\n",
47 | "|Sudhanshu| 30| 8| 25000|\n",
48 | "| Sunny| 29| 4| 20000|\n",
49 | "| Paul| 24| 3| 20000|\n",
50 | "| Harsha| 21| 1| 15000|\n",
51 | "| Shubham| 23| 2| 18000|\n",
52 | "+---------+---+----------+------+\n",
53 | "\n"
54 | ]
55 | }
56 | ],
57 | "source": [
58 | "training.show()\n",
59 | "\n"
60 | ]
61 | },
62 | {
63 | "cell_type": "code",
64 | "execution_count": 74,
65 | "id": "6b3dd5ff",
66 | "metadata": {},
67 | "outputs": [
68 | {
69 | "name": "stdout",
70 | "output_type": "stream",
71 | "text": [
72 | "root\n",
73 | " |-- Name: string (nullable = true)\n",
74 | " |-- age: integer (nullable = true)\n",
75 | " |-- Experience: integer (nullable = true)\n",
76 | " |-- Salary: integer (nullable = true)\n",
77 | "\n"
78 | ]
79 | }
80 | ],
81 | "source": [
82 | "training.printSchema()"
83 | ]
84 | },
85 | {
86 | "cell_type": "code",
87 | "execution_count": 75,
88 | "id": "5d3227e6",
89 | "metadata": {},
90 | "outputs": [
91 | {
92 | "data": {
93 | "text/plain": [
94 | "['Name', 'age', 'Experience', 'Salary']"
95 | ]
96 | },
97 | "execution_count": 75,
98 | "metadata": {},
99 | "output_type": "execute_result"
100 | }
101 | ],
102 | "source": [
103 | "training.columns"
104 | ]
105 | },
106 | {
107 | "cell_type": "code",
108 | "execution_count": null,
109 | "id": "cffef5b9",
110 | "metadata": {},
111 | "outputs": [],
112 | "source": [
113 | "[Age,Experience]----> new feature--->independent feature"
114 | ]
115 | },
116 | {
117 | "cell_type": "code",
118 | "execution_count": 76,
119 | "id": "e6273555",
120 | "metadata": {},
121 | "outputs": [],
122 | "source": [
123 | "from pyspark.ml.feature import VectorAssembler\n",
124 | "featureassembler=VectorAssembler(inputCols=[\"age\",\"Experience\"],outputCol=\"Independent Features\")"
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": 77,
130 | "id": "0b69744c",
131 | "metadata": {},
132 | "outputs": [],
133 | "source": [
134 | "output=featureassembler.transform(training)"
135 | ]
136 | },
137 | {
138 | "cell_type": "code",
139 | "execution_count": 78,
140 | "id": "60961194",
141 | "metadata": {},
142 | "outputs": [
143 | {
144 | "name": "stdout",
145 | "output_type": "stream",
146 | "text": [
147 | "+---------+---+----------+------+--------------------+\n",
148 | "| Name|age|Experience|Salary|Independent Features|\n",
149 | "+---------+---+----------+------+--------------------+\n",
150 | "| Krish| 31| 10| 30000| [31.0,10.0]|\n",
151 | "|Sudhanshu| 30| 8| 25000| [30.0,8.0]|\n",
152 | "| Sunny| 29| 4| 20000| [29.0,4.0]|\n",
153 | "| Paul| 24| 3| 20000| [24.0,3.0]|\n",
154 | "| Harsha| 21| 1| 15000| [21.0,1.0]|\n",
155 | "| Shubham| 23| 2| 18000| [23.0,2.0]|\n",
156 | "+---------+---+----------+------+--------------------+\n",
157 | "\n"
158 | ]
159 | }
160 | ],
161 | "source": [
162 | "output.show()"
163 | ]
164 | },
165 | {
166 | "cell_type": "code",
167 | "execution_count": 79,
168 | "id": "2c27434a",
169 | "metadata": {},
170 | "outputs": [
171 | {
172 | "data": {
173 | "text/plain": [
174 | "['Name', 'age', 'Experience', 'Salary', 'Independent Features']"
175 | ]
176 | },
177 | "execution_count": 79,
178 | "metadata": {},
179 | "output_type": "execute_result"
180 | }
181 | ],
182 | "source": [
183 | "output.columns"
184 | ]
185 | },
186 | {
187 | "cell_type": "code",
188 | "execution_count": 80,
189 | "id": "54a0ccab",
190 | "metadata": {},
191 | "outputs": [],
192 | "source": [
193 | "finalized_data=output.select(\"Independent Features\",\"Salary\")"
194 | ]
195 | },
196 | {
197 | "cell_type": "code",
198 | "execution_count": 81,
199 | "id": "f7a73845",
200 | "metadata": {},
201 | "outputs": [
202 | {
203 | "name": "stdout",
204 | "output_type": "stream",
205 | "text": [
206 | "+--------------------+------+\n",
207 | "|Independent Features|Salary|\n",
208 | "+--------------------+------+\n",
209 | "| [31.0,10.0]| 30000|\n",
210 | "| [30.0,8.0]| 25000|\n",
211 | "| [29.0,4.0]| 20000|\n",
212 | "| [24.0,3.0]| 20000|\n",
213 | "| [21.0,1.0]| 15000|\n",
214 | "| [23.0,2.0]| 18000|\n",
215 | "+--------------------+------+\n",
216 | "\n"
217 | ]
218 | }
219 | ],
220 | "source": [
221 | "finalized_data.show()"
222 | ]
223 | },
224 | {
225 | "cell_type": "code",
226 | "execution_count": 82,
227 | "id": "0b11192b",
228 | "metadata": {},
229 | "outputs": [],
230 | "source": [
231 | "from pyspark.ml.regression import LinearRegression\n",
232 | "##train test split\n",
233 | "train_data,test_data=finalized_data.randomSplit([0.75,0.25])\n",
234 | "regressor=LinearRegression(featuresCol='Independent Features', labelCol='Salary')\n",
235 | "regressor=regressor.fit(train_data)"
236 | ]
237 | },
238 | {
239 | "cell_type": "code",
240 | "execution_count": 83,
241 | "id": "fa4ec997",
242 | "metadata": {},
243 | "outputs": [
244 | {
245 | "data": {
246 | "text/plain": [
247 | "DenseVector([-5000.0, 7000.0])"
248 | ]
249 | },
250 | "execution_count": 83,
251 | "metadata": {},
252 | "output_type": "execute_result"
253 | }
254 | ],
255 | "source": [
256 | "### Coefficients\n",
257 | "regressor.coefficients"
258 | ]
259 | },
260 | {
261 | "cell_type": "code",
262 | "execution_count": 84,
263 | "id": "eba911b6",
264 | "metadata": {},
265 | "outputs": [
266 | {
267 | "data": {
268 | "text/plain": [
269 | "118999.99999893687"
270 | ]
271 | },
272 | "execution_count": 84,
273 | "metadata": {},
274 | "output_type": "execute_result"
275 | }
276 | ],
277 | "source": [
278 | "### Intercepts\n",
279 | "regressor.intercept"
280 | ]
281 | },
282 | {
283 | "cell_type": "code",
284 | "execution_count": 85,
285 | "id": "2ba2bc70",
286 | "metadata": {},
287 | "outputs": [],
288 | "source": [
289 | "### Prediction\n",
290 | "pred_results=regressor.evaluate(test_data)"
291 | ]
292 | },
293 | {
294 | "cell_type": "code",
295 | "execution_count": 86,
296 | "id": "489d6392",
297 | "metadata": {},
298 | "outputs": [
299 | {
300 | "name": "stdout",
301 | "output_type": "stream",
302 | "text": [
303 | "+--------------------+------+-----------------+\n",
304 | "|Independent Features|Salary| prediction|\n",
305 | "+--------------------+------+-----------------+\n",
306 | "| [21.0,1.0]| 15000|20999.99999996154|\n",
307 | "| [29.0,4.0]| 20000|2000.000000192551|\n",
308 | "| [31.0,10.0]| 30000|33999.99999993094|\n",
309 | "+--------------------+------+-----------------+\n",
310 | "\n"
311 | ]
312 | }
313 | ],
314 | "source": [
315 | "pred_results.predictions.show()"
316 | ]
317 | },
318 | {
319 | "cell_type": "code",
320 | "execution_count": 87,
321 | "id": "0534e854",
322 | "metadata": {},
323 | "outputs": [
324 | {
325 | "data": {
326 | "text/plain": [
327 | "(9333.333333233308, 125333333.3306847)"
328 | ]
329 | },
330 | "execution_count": 87,
331 | "metadata": {},
332 | "output_type": "execute_result"
333 | }
334 | ],
335 | "source": [
336 | "pred_results.meanAbsoluteError,pred_results.meanSquaredError"
337 | ]
338 | },
339 | {
340 | "cell_type": "code",
341 | "execution_count": null,
342 | "id": "70de559b",
343 | "metadata": {},
344 | "outputs": [],
345 | "source": []
346 | }
347 | ],
348 | "metadata": {
349 | "kernelspec": {
350 | "display_name": "Python 3",
351 | "language": "python",
352 | "name": "python3"
353 | },
354 | "language_info": {
355 | "codemirror_mode": {
356 | "name": "ipython",
357 | "version": 3
358 | },
359 | "file_extension": ".py",
360 | "mimetype": "text/x-python",
361 | "name": "python",
362 | "nbconvert_exporter": "python",
363 | "pygments_lexer": "ipython3",
364 | "version": "3.7.10"
365 | }
366 | },
367 | "nbformat": 4,
368 | "nbformat_minor": 5
369 | }
370 |
--------------------------------------------------------------------------------
/Tutorial 8-Linear Regression With Pyspark.ipynb:
--------------------------------------------------------------------------------
1 | {"cells":[{"cell_type":"markdown","source":["## Overview\n\nThis notebook will show you how to create and query a table or DataFrame that you uploaded to DBFS. [DBFS](https://docs.databricks.com/user-guide/dbfs-databricks-file-system.html) is a Databricks File System that allows you to store data for querying inside of Databricks. This notebook assumes that you have a file already inside of DBFS that you would like to read from.\n\nThis notebook is written in **Python** so the default cell type is Python. However, you can use different languages by using the `%LANGUAGE` syntax. Python, Scala, SQL, and R are all supported."],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"96816ed7-b08a-4ca3-abb9-f99880c3535d"}}},{"cell_type":"code","source":["# File location and type\nfile_location = \"/FileStore/tables/tips.csv\"\nfile_type = \"csv\"\n\n# The applied options are for CSV files. For other file types, these will be ignored.\ndf =spark.read.csv(file_location,header=True,inferSchema=True)\ndf.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6482be4c-f067-47c9-b0ac-35c938b94601"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+\n|total_bill| tip| sex|smoker|day| time|size|\n+----------+----+------+------+---+------+----+\n| 16.99|1.01|Female| No|Sun|Dinner| 2|\n| 10.34|1.66| Male| No|Sun|Dinner| 3|\n| 21.01| 3.5| Male| No|Sun|Dinner| 3|\n| 23.68|3.31| Male| No|Sun|Dinner| 2|\n| 24.59|3.61|Female| No|Sun|Dinner| 4|\n| 25.29|4.71| Male| No|Sun|Dinner| 4|\n| 8.77| 2.0| Male| No|Sun|Dinner| 2|\n| 26.88|3.12| Male| No|Sun|Dinner| 4|\n| 15.04|1.96| Male| No|Sun|Dinner| 2|\n| 14.78|3.23| Male| No|Sun|Dinner| 2|\n| 10.27|1.71| Male| No|Sun|Dinner| 2|\n| 35.26| 5.0|Female| No|Sun|Dinner| 4|\n| 15.42|1.57| Male| No|Sun|Dinner| 2|\n| 18.43| 3.0| Male| No|Sun|Dinner| 4|\n| 14.83|3.02|Female| No|Sun|Dinner| 2|\n| 21.58|3.92| Male| No|Sun|Dinner| 2|\n| 10.33|1.67|Female| No|Sun|Dinner| 3|\n| 16.29|3.71| Male| No|Sun|Dinner| 3|\n| 16.97| 3.5|Female| No|Sun|Dinner| 3|\n| 20.65|3.35| Male| No|Sat|Dinner| 3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+----------+----+------+------+---+------+----+\ntotal_bill| tip| sex|smoker|day| time|size|\n+----------+----+------+------+---+------+----+\n 16.99|1.01|Female| No|Sun|Dinner| 2|\n 10.34|1.66| Male| No|Sun|Dinner| 3|\n 21.01| 3.5| Male| No|Sun|Dinner| 3|\n 23.68|3.31| Male| No|Sun|Dinner| 2|\n 24.59|3.61|Female| No|Sun|Dinner| 4|\n 25.29|4.71| Male| No|Sun|Dinner| 4|\n 8.77| 2.0| Male| No|Sun|Dinner| 2|\n 26.88|3.12| Male| No|Sun|Dinner| 4|\n 15.04|1.96| Male| No|Sun|Dinner| 2|\n 14.78|3.23| Male| No|Sun|Dinner| 2|\n 10.27|1.71| Male| No|Sun|Dinner| 2|\n 35.26| 5.0|Female| No|Sun|Dinner| 4|\n 15.42|1.57| Male| No|Sun|Dinner| 2|\n 18.43| 3.0| Male| No|Sun|Dinner| 4|\n 14.83|3.02|Female| No|Sun|Dinner| 2|\n 21.58|3.92| Male| No|Sun|Dinner| 2|\n 10.33|1.67|Female| No|Sun|Dinner| 3|\n 16.29|3.71| Male| No|Sun|Dinner| 3|\n 16.97| 3.5|Female| No|Sun|Dinner| 3|\n 20.65|3.35| Male| No|Sat|Dinner| 3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["df.printSchema()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5e5b80f2-3426-44e1-b86e-171314f4827e"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- total_bill: double (nullable = true)\n |-- tip: double (nullable = true)\n |-- sex: string (nullable = true)\n |-- smoker: string (nullable = true)\n |-- day: string (nullable = true)\n |-- time: string (nullable = true)\n |-- size: integer (nullable = true)\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nroot\n-- total_bill: double (nullable = true)\n-- tip: double (nullable = true)\n-- sex: string (nullable = true)\n-- smoker: string (nullable = true)\n-- day: string (nullable = true)\n-- time: string (nullable = true)\n-- size: integer (nullable = true)\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["df.columns"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0432b71c-b266-417d-b0d5-1c17afa0f090"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[3]: ['total_bill', 'tip', 'sex', 'smoker', 'day', 'time', 'size']
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nOut[3]: ['total_bill', 'tip', 'sex', 'smoker', 'day', 'time', 'size']
"]}}],"execution_count":0},{"cell_type":"code","source":["### Handling Categorical Features\nfrom pyspark.ml.feature import StringIndexer"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0ae62ac1-81a6-4b1d-92b9-f85ec9cc93ff"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n"]}}],"execution_count":0},{"cell_type":"code","source":["df.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"faa6f9b0-6f8b-4dbd-a5a2-dc074181f2e3"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+\n|total_bill| tip| sex|smoker|day| time|size|\n+----------+----+------+------+---+------+----+\n| 16.99|1.01|Female| No|Sun|Dinner| 2|\n| 10.34|1.66| Male| No|Sun|Dinner| 3|\n| 21.01| 3.5| Male| No|Sun|Dinner| 3|\n| 23.68|3.31| Male| No|Sun|Dinner| 2|\n| 24.59|3.61|Female| No|Sun|Dinner| 4|\n| 25.29|4.71| Male| No|Sun|Dinner| 4|\n| 8.77| 2.0| Male| No|Sun|Dinner| 2|\n| 26.88|3.12| Male| No|Sun|Dinner| 4|\n| 15.04|1.96| Male| No|Sun|Dinner| 2|\n| 14.78|3.23| Male| No|Sun|Dinner| 2|\n| 10.27|1.71| Male| No|Sun|Dinner| 2|\n| 35.26| 5.0|Female| No|Sun|Dinner| 4|\n| 15.42|1.57| Male| No|Sun|Dinner| 2|\n| 18.43| 3.0| Male| No|Sun|Dinner| 4|\n| 14.83|3.02|Female| No|Sun|Dinner| 2|\n| 21.58|3.92| Male| No|Sun|Dinner| 2|\n| 10.33|1.67|Female| No|Sun|Dinner| 3|\n| 16.29|3.71| Male| No|Sun|Dinner| 3|\n| 16.97| 3.5|Female| No|Sun|Dinner| 3|\n| 20.65|3.35| Male| No|Sat|Dinner| 3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+----------+----+------+------+---+------+----+\ntotal_bill| tip| sex|smoker|day| time|size|\n+----------+----+------+------+---+------+----+\n 16.99|1.01|Female| No|Sun|Dinner| 2|\n 10.34|1.66| Male| No|Sun|Dinner| 3|\n 21.01| 3.5| Male| No|Sun|Dinner| 3|\n 23.68|3.31| Male| No|Sun|Dinner| 2|\n 24.59|3.61|Female| No|Sun|Dinner| 4|\n 25.29|4.71| Male| No|Sun|Dinner| 4|\n 8.77| 2.0| Male| No|Sun|Dinner| 2|\n 26.88|3.12| Male| No|Sun|Dinner| 4|\n 15.04|1.96| Male| No|Sun|Dinner| 2|\n 14.78|3.23| Male| No|Sun|Dinner| 2|\n 10.27|1.71| Male| No|Sun|Dinner| 2|\n 35.26| 5.0|Female| No|Sun|Dinner| 4|\n 15.42|1.57| Male| No|Sun|Dinner| 2|\n 18.43| 3.0| Male| No|Sun|Dinner| 4|\n 14.83|3.02|Female| No|Sun|Dinner| 2|\n 21.58|3.92| Male| No|Sun|Dinner| 2|\n 10.33|1.67|Female| No|Sun|Dinner| 3|\n 16.29|3.71| Male| No|Sun|Dinner| 3|\n 16.97| 3.5|Female| No|Sun|Dinner| 3|\n 20.65|3.35| Male| No|Sat|Dinner| 3|\n+----------+----+------+------+---+------+----+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["indexer=StringIndexer(inputCol=\"sex\",outputCol=\"sex_indexed\")\ndf_r=indexer.fit(df).transform(df)\ndf_r.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"2ee7ab64-9804-4afb-852c-ee02eb5d3a20"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+\n|total_bill| tip| sex|smoker|day| time|size|sex_indexed|\n+----------+----+------+------+---+------+----+-----------+\n| 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0|\n| 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0|\n| 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0|\n| 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0|\n| 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0|\n| 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0|\n| 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0|\n| 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0|\n| 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0|\n| 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0|\n| 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0|\n| 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0|\n| 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0|\n| 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0|\n| 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0|\n| 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0|\n| 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0|\n| 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0|\n| 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0|\n| 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0|\n+----------+----+------+------+---+------+----+-----------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+----------+----+------+------+---+------+----+-----------+\ntotal_bill| tip| sex|smoker|day| time|size|sex_indexed|\n+----------+----+------+------+---+------+----+-----------+\n 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0|\n 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0|\n 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0|\n 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0|\n 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0|\n 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0|\n 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0|\n 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0|\n 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0|\n 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0|\n 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0|\n 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0|\n 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0|\n 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0|\n 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0|\n 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0|\n 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0|\n 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0|\n 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0|\n 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0|\n+----------+----+------+------+---+------+----+-----------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["indexer=StringIndexer(inputCols=[\"smoker\",\"day\",\"time\"],outputCols=[\"smoker_indexed\",\"day_indexed\",\n \"time_index\"])\ndf_r=indexer.fit(df_r).transform(df_r)\ndf_r.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6b95d734-4c80-4762-bd9b-92b6a107dced"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\n|total_bill| tip| sex|smoker|day| time|size|sex_indexed|smoker_indexed|day_indexed|time_index|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\n| 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|\n| 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n| 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n| 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|\n| 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n| 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n| 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|\n| 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n| 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|\n| 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n| 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|\n| 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n| 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|\n| 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0| 0.0| 0.0| 0.0|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\ntotal_bill| tip| sex|smoker|day| time|size|sex_indexed|smoker_indexed|day_indexed|time_index|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\n 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|\n 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|\n 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|\n 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|\n 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|\n 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|\n 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|\n 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|\n 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|\n 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0| 0.0| 0.0| 0.0|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["df_r.columns"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"a9909b0b-caee-4838-b477-47c3701dbfd4"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[9]: ['total_bill',\n 'tip',\n 'sex',\n 'smoker',\n 'day',\n 'time',\n 'size',\n 'sex_indexed',\n 'smoker_indexed',\n 'day_indexed',\n 'time_index']
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nOut[9]: ['total_bill',\n 'tip',\n 'sex',\n 'smoker',\n 'day',\n 'time',\n 'size',\n 'sex_indexed',\n 'smoker_indexed',\n 'day_indexed',\n 'time_index']
"]}}],"execution_count":0},{"cell_type":"code","source":["from pyspark.ml.feature import VectorAssembler\nfeatureassembler=VectorAssembler(inputCols=['tip','size','sex_indexed','smoker_indexed','day_indexed',\n 'time_index'],outputCol=\"Independent Features\")\noutput=featureassembler.transform(df_r)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"61d875e5-71fa-4dc4-ae90-54924b00a632"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n"]}}],"execution_count":0},{"cell_type":"code","source":["output.select('Independent Features').show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d33d1178-95a2-468f-a94a-e0eebc67be86"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+\n|Independent Features|\n+--------------------+\n|[1.01,2.0,1.0,0.0...|\n|[1.66,3.0,0.0,0.0...|\n|[3.5,3.0,0.0,0.0,...|\n|[3.31,2.0,0.0,0.0...|\n|[3.61,4.0,1.0,0.0...|\n|[4.71,4.0,0.0,0.0...|\n|[2.0,2.0,0.0,0.0,...|\n|[3.12,4.0,0.0,0.0...|\n|[1.96,2.0,0.0,0.0...|\n|[3.23,2.0,0.0,0.0...|\n|[1.71,2.0,0.0,0.0...|\n|[5.0,4.0,1.0,0.0,...|\n|[1.57,2.0,0.0,0.0...|\n|[3.0,4.0,0.0,0.0,...|\n|[3.02,2.0,1.0,0.0...|\n|[3.92,2.0,0.0,0.0...|\n|[1.67,3.0,1.0,0.0...|\n|[3.71,3.0,0.0,0.0...|\n|[3.5,3.0,1.0,0.0,...|\n|(6,[0,1],[3.35,3.0])|\n+--------------------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+--------------------+\nIndependent Features|\n+--------------------+\n[1.01,2.0,1.0,0.0...|\n[1.66,3.0,0.0,0.0...|\n[3.5,3.0,0.0,0.0,...|\n[3.31,2.0,0.0,0.0...|\n[3.61,4.0,1.0,0.0...|\n[4.71,4.0,0.0,0.0...|\n[2.0,2.0,0.0,0.0,...|\n[3.12,4.0,0.0,0.0...|\n[1.96,2.0,0.0,0.0...|\n[3.23,2.0,0.0,0.0...|\n[1.71,2.0,0.0,0.0...|\n[5.0,4.0,1.0,0.0,...|\n[1.57,2.0,0.0,0.0...|\n[3.0,4.0,0.0,0.0,...|\n[3.02,2.0,1.0,0.0...|\n[3.92,2.0,0.0,0.0...|\n[1.67,3.0,1.0,0.0...|\n[3.71,3.0,0.0,0.0...|\n[3.5,3.0,1.0,0.0,...|\n(6,[0,1],[3.35,3.0])|\n+--------------------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["output.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f2646b66-7710-4297-a6e1-156a37e6582d"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\n|total_bill| tip| sex|smoker|day| time|size|sex_indexed|smoker_indexed|day_indexed|time_index|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\n| 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|[1.01,2.0,1.0,0.0...|\n| 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[1.66,3.0,0.0,0.0...|\n| 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[3.5,3.0,0.0,0.0,...|\n| 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.31,2.0,0.0,0.0...|\n| 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|[3.61,4.0,1.0,0.0...|\n| 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[4.71,4.0,0.0,0.0...|\n| 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[2.0,2.0,0.0,0.0,...|\n| 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[3.12,4.0,0.0,0.0...|\n| 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.96,2.0,0.0,0.0...|\n| 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.23,2.0,0.0,0.0...|\n| 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.71,2.0,0.0,0.0...|\n| 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|[5.0,4.0,1.0,0.0,...|\n| 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.57,2.0,0.0,0.0...|\n| 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[3.0,4.0,0.0,0.0,...|\n| 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|[3.02,2.0,1.0,0.0...|\n| 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.92,2.0,0.0,0.0...|\n| 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|[1.67,3.0,1.0,0.0...|\n| 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[3.71,3.0,0.0,0.0...|\n| 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|[3.5,3.0,1.0,0.0,...|\n| 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0| 0.0| 0.0| 0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\ntotal_bill| tip| sex|smoker|day| time|size|sex_indexed|smoker_indexed|day_indexed|time_index|Independent Features|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\n 16.99|1.01|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|[1.01,2.0,1.0,0.0...|\n 10.34|1.66| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[1.66,3.0,0.0,0.0...|\n 21.01| 3.5| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[3.5,3.0,0.0,0.0,...|\n 23.68|3.31| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.31,2.0,0.0,0.0...|\n 24.59|3.61|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|[3.61,4.0,1.0,0.0...|\n 25.29|4.71| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[4.71,4.0,0.0,0.0...|\n 8.77| 2.0| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[2.0,2.0,0.0,0.0,...|\n 26.88|3.12| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[3.12,4.0,0.0,0.0...|\n 15.04|1.96| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.96,2.0,0.0,0.0...|\n 14.78|3.23| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.23,2.0,0.0,0.0...|\n 10.27|1.71| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.71,2.0,0.0,0.0...|\n 35.26| 5.0|Female| No|Sun|Dinner| 4| 1.0| 0.0| 1.0| 0.0|[5.0,4.0,1.0,0.0,...|\n 15.42|1.57| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[1.57,2.0,0.0,0.0...|\n 18.43| 3.0| Male| No|Sun|Dinner| 4| 0.0| 0.0| 1.0| 0.0|[3.0,4.0,0.0,0.0,...|\n 14.83|3.02|Female| No|Sun|Dinner| 2| 1.0| 0.0| 1.0| 0.0|[3.02,2.0,1.0,0.0...|\n 21.58|3.92| Male| No|Sun|Dinner| 2| 0.0| 0.0| 1.0| 0.0|[3.92,2.0,0.0,0.0...|\n 10.33|1.67|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|[1.67,3.0,1.0,0.0...|\n 16.29|3.71| Male| No|Sun|Dinner| 3| 0.0| 0.0| 1.0| 0.0|[3.71,3.0,0.0,0.0...|\n 16.97| 3.5|Female| No|Sun|Dinner| 3| 1.0| 0.0| 1.0| 0.0|[3.5,3.0,1.0,0.0,...|\n 20.65|3.35| Male| No|Sat|Dinner| 3| 0.0| 0.0| 0.0| 0.0|(6,[0,1],[3.35,3.0])|\n+----------+----+------+------+---+------+----+-----------+--------------+-----------+----------+--------------------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["finalized_data=output.select(\"Independent Features\",\"total_bill\")"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d1c1fa4c-c78a-441a-bed9-3bcfcc5af966"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n"]}}],"execution_count":0},{"cell_type":"code","source":["finalized_data.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"3d14fe7b-bc59-4376-8139-142283af09b0"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+----------+\n|Independent Features|total_bill|\n+--------------------+----------+\n|[1.01,2.0,1.0,0.0...| 16.99|\n|[1.66,3.0,0.0,0.0...| 10.34|\n|[3.5,3.0,0.0,0.0,...| 21.01|\n|[3.31,2.0,0.0,0.0...| 23.68|\n|[3.61,4.0,1.0,0.0...| 24.59|\n|[4.71,4.0,0.0,0.0...| 25.29|\n|[2.0,2.0,0.0,0.0,...| 8.77|\n|[3.12,4.0,0.0,0.0...| 26.88|\n|[1.96,2.0,0.0,0.0...| 15.04|\n|[3.23,2.0,0.0,0.0...| 14.78|\n|[1.71,2.0,0.0,0.0...| 10.27|\n|[5.0,4.0,1.0,0.0,...| 35.26|\n|[1.57,2.0,0.0,0.0...| 15.42|\n|[3.0,4.0,0.0,0.0,...| 18.43|\n|[3.02,2.0,1.0,0.0...| 14.83|\n|[3.92,2.0,0.0,0.0...| 21.58|\n|[1.67,3.0,1.0,0.0...| 10.33|\n|[3.71,3.0,0.0,0.0...| 16.29|\n|[3.5,3.0,1.0,0.0,...| 16.97|\n|(6,[0,1],[3.35,3.0])| 20.65|\n+--------------------+----------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+--------------------+----------+\nIndependent Features|total_bill|\n+--------------------+----------+\n[1.01,2.0,1.0,0.0...| 16.99|\n[1.66,3.0,0.0,0.0...| 10.34|\n[3.5,3.0,0.0,0.0,...| 21.01|\n[3.31,2.0,0.0,0.0...| 23.68|\n[3.61,4.0,1.0,0.0...| 24.59|\n[4.71,4.0,0.0,0.0...| 25.29|\n[2.0,2.0,0.0,0.0,...| 8.77|\n[3.12,4.0,0.0,0.0...| 26.88|\n[1.96,2.0,0.0,0.0...| 15.04|\n[3.23,2.0,0.0,0.0...| 14.78|\n[1.71,2.0,0.0,0.0...| 10.27|\n[5.0,4.0,1.0,0.0,...| 35.26|\n[1.57,2.0,0.0,0.0...| 15.42|\n[3.0,4.0,0.0,0.0,...| 18.43|\n[3.02,2.0,1.0,0.0...| 14.83|\n[3.92,2.0,0.0,0.0...| 21.58|\n[1.67,3.0,1.0,0.0...| 10.33|\n[3.71,3.0,0.0,0.0...| 16.29|\n[3.5,3.0,1.0,0.0,...| 16.97|\n(6,[0,1],[3.35,3.0])| 20.65|\n+--------------------+----------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["from pyspark.ml.regression import LinearRegression\n##train test split\ntrain_data,test_data=finalized_data.randomSplit([0.75,0.25])\nregressor=LinearRegression(featuresCol='Independent Features', labelCol='total_bill')\nregressor=regressor.fit(train_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"dbe03a38-e728-40f9-8a53-0b7968b8dc87"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n"]}}],"execution_count":0},{"cell_type":"code","source":["regressor.coefficients"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0fdc835a-96fb-4ab3-89be-6cfbc57c7ac6"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[17]: DenseVector([3.3598, 3.3861, -0.6641, 2.5847, -0.1423, -1.3377])
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nOut[17]: DenseVector([3.3598, 3.3861, -0.6641, 2.5847, -0.1423, -1.3377])
"]}}],"execution_count":0},{"cell_type":"code","source":["regressor.intercept"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fd1642d4-bb73-4fc0-a410-ada61d0f3410"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[18]: 0.9231025978363154
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nOut[18]: 0.9231025978363154
"]}}],"execution_count":0},{"cell_type":"code","source":["### Predictions\npred_results=regressor.evaluate(test_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"2e45a3d8-af1c-408b-b64f-fe466c3401bd"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n"]}}],"execution_count":0},{"cell_type":"code","source":["## Final comparison\npred_results.predictions.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"01d128d2-1a71-44d0-a14c-b0377693547b"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+----------+------------------+\n|Independent Features|total_bill| prediction|\n+--------------------+----------+------------------+\n| (6,[0,1],[2.0,2.0])| 12.69|14.414877568922382|\n|(6,[0,1],[3.35,3.0])| 20.65|22.336705086951124|\n|[1.0,1.0,1.0,0.0,...| 7.25| 7.004851678101628|\n|[1.17,2.0,0.0,1.0...| 32.83|14.210940490994291|\n|[1.36,3.0,1.0,0.0...| 18.64|13.364280305420156|\n|[1.5,2.0,0.0,1.0,...| 11.59|15.319683950195104|\n|[1.58,2.0,0.0,1.0...| 13.42| 13.82395853728497|\n|[1.66,3.0,0.0,0.0...| 10.34|16.516310272733463|\n|[1.73,2.0,0.0,0.0...| 9.78| 11.88549649517034|\n|[2.0,2.0,0.0,0.0,...| 13.81| 14.27259319727858|\n|[2.0,2.0,0.0,0.0,...| 13.03| 12.79265023451646|\n|[2.0,2.0,1.0,0.0,...| 14.15|12.128511829238738|\n|[2.0,2.0,1.0,0.0,...| 14.52|12.128511829238738|\n|[2.0,2.0,1.0,1.0,...| 10.63|16.335459877039824|\n|[2.0,2.0,1.0,1.0,...| 27.18|16.335459877039824|\n|[2.0,3.0,1.0,0.0,...| 16.21|16.994513613299002|\n|[2.23,2.0,1.0,1.0...| 12.76| 17.10822046981615|\n|[2.24,2.0,0.0,0.0...| 20.76|15.078952076697353|\n|[2.31,2.0,0.0,0.0...| 11.69|13.834197120432375|\n|[2.5,2.0,0.0,0.0,...| 14.07|15.952507529401023|\n+--------------------+----------+------------------+\nonly showing top 20 rows\n\n
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\n+--------------------+----------+------------------+\nIndependent Features|total_bill| prediction|\n+--------------------+----------+------------------+\n (6,[0,1],[2.0,2.0])| 12.69|14.414877568922382|\n(6,[0,1],[3.35,3.0])| 20.65|22.336705086951124|\n[1.0,1.0,1.0,0.0,...| 7.25| 7.004851678101628|\n[1.17,2.0,0.0,1.0...| 32.83|14.210940490994291|\n[1.36,3.0,1.0,0.0...| 18.64|13.364280305420156|\n[1.5,2.0,0.0,1.0,...| 11.59|15.319683950195104|\n[1.58,2.0,0.0,1.0...| 13.42| 13.82395853728497|\n[1.66,3.0,0.0,0.0...| 10.34|16.516310272733463|\n[1.73,2.0,0.0,0.0...| 9.78| 11.88549649517034|\n[2.0,2.0,0.0,0.0,...| 13.81| 14.27259319727858|\n[2.0,2.0,0.0,0.0,...| 13.03| 12.79265023451646|\n[2.0,2.0,1.0,0.0,...| 14.15|12.128511829238738|\n[2.0,2.0,1.0,0.0,...| 14.52|12.128511829238738|\n[2.0,2.0,1.0,1.0,...| 10.63|16.335459877039824|\n[2.0,2.0,1.0,1.0,...| 27.18|16.335459877039824|\n[2.0,3.0,1.0,0.0,...| 16.21|16.994513613299002|\n[2.23,2.0,1.0,1.0...| 12.76| 17.10822046981615|\n[2.24,2.0,0.0,0.0...| 20.76|15.078952076697353|\n[2.31,2.0,0.0,0.0...| 11.69|13.834197120432375|\n[2.5,2.0,0.0,0.0,...| 14.07|15.952507529401023|\n+--------------------+----------+------------------+\nonly showing top 20 rows\n\n
"]}}],"execution_count":0},{"cell_type":"code","source":["### PErformance Metrics\npred_results.r2,pred_results.meanAbsoluteError,pred_results.meanSquaredError"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"75e3e5b1-0bb4-4dbe-a1ca-08e5a31ee173"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[25]: (0.40050077944613716, 4.809771114444798, 40.934088106916576)
","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"html","arguments":{}}},"output_type":"display_data","data":{"text/html":["\nOut[25]: (0.40050077944613716, 4.809771114444798, 40.934088106916576)
"]}}],"execution_count":0},{"cell_type":"code","source":[""],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ce0398a7-7ebd-4f2c-b12e-2ef701925124"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"data":"","errorSummary":"","metadata":{},"type":"ipynbError","arguments":{}}},"output_type":"display_data","data":{"text/html":[""]}}],"execution_count":0}],"metadata":{"application/vnd.databricks.v1+notebook":{"notebookName":"Tutorial 8-Linear Regression With Pyspark","dashboards":[],"notebookMetadata":{"pythonIndentUnit":2},"language":"python","widgets":{},"notebookOrigID":523045182520803}},"nbformat":4,"nbformat_minor":0}
2 |
--------------------------------------------------------------------------------
/pyspark basic introduction.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "8ff41f5d",
6 | "metadata": {},
7 | "source": [
8 | "#### Pyspark Basic Introduction"
9 | ]
10 | },
11 | {
12 | "cell_type": "code",
13 | "execution_count": 1,
14 | "id": "4abcaad4",
15 | "metadata": {},
16 | "outputs": [
17 | {
18 | "name": "stdout",
19 | "output_type": "stream",
20 | "text": [
21 | "Requirement already satisfied: pyspark in c:\\users\\win10\\anaconda3\\envs\\myenv\\lib\\site-packages (3.1.1)\n",
22 | "Requirement already satisfied: py4j==0.10.9 in c:\\users\\win10\\anaconda3\\envs\\myenv\\lib\\site-packages (from pyspark) (0.10.9)\n"
23 | ]
24 | }
25 | ],
26 | "source": [
27 | "!pip install pyspark"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": 2,
33 | "id": "08b29f6a",
34 | "metadata": {},
35 | "outputs": [],
36 | "source": [
37 | "import pyspark"
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "execution_count": 15,
43 | "id": "e1de79e4",
44 | "metadata": {},
45 | "outputs": [
46 | {
47 | "data": {
48 | "text/plain": [
49 | "pandas.core.frame.DataFrame"
50 | ]
51 | },
52 | "execution_count": 15,
53 | "metadata": {},
54 | "output_type": "execute_result"
55 | }
56 | ],
57 | "source": [
58 | "import pandas as pd\n",
59 | "type(pd.read_csv('test1.csv'))"
60 | ]
61 | },
62 | {
63 | "cell_type": "code",
64 | "execution_count": 4,
65 | "id": "37a82e23",
66 | "metadata": {},
67 | "outputs": [],
68 | "source": [
69 | "from pyspark.sql import SparkSession"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 5,
75 | "id": "c334b45e",
76 | "metadata": {},
77 | "outputs": [],
78 | "source": [
79 | "spark=SparkSession.builder.appName('Practise').getOrCreate()"
80 | ]
81 | },
82 | {
83 | "cell_type": "code",
84 | "execution_count": 6,
85 | "id": "558caca5",
86 | "metadata": {},
87 | "outputs": [
88 | {
89 | "data": {
90 | "text/html": [
91 | "\n",
92 | " \n",
93 | "
SparkSession - in-memory
\n",
94 | " \n",
95 | "
\n",
96 | "
SparkContext
\n",
97 | "\n",
98 | "
Spark UI
\n",
99 | "\n",
100 | "
\n",
101 | " - Version
\n",
102 | " v3.1.1- Master
\n",
104 | " local[*]- AppName
\n",
106 | " Practise
\n",
108 | "
\n",
109 | " \n",
110 | "