├── .gitignore
├── LICENSE
├── README.md
├── activator
├── activator-launch-1.3.6.jar
├── activator.bat
├── activator.properties
├── build.sbt
├── project
    └── build.properties
├── spark-tutorial-logo.png
├── src
    ├── main
    │   ├── resources
    │   │   ├── Cricket_Edges.csv
    │   │   ├── Cricket_Edges1.csv
    │   │   ├── Cricket_Edges_Follower.csv
    │   │   ├── Cricket_Node.csv
    │   │   ├── Cricket_Node_Rklick_Test.csv
    │   │   ├── Person
    │   │   ├── animals.json
    │   │   ├── cars.csv
    │   │   ├── cars.json
    │   │   ├── cars1.json
    │   │   ├── cars_price.json
    │   │   ├── employee.json
    │   │   ├── employee.txt
    │   │   ├── employee1.json
    │   │   ├── employees.json
    │   │   ├── facebook_edges.csv
    │   │   ├── facebook_nodes.csv
    │   │   ├── followers.txt
    │   │   ├── followers1.txt
    │   │   ├── fruits.json
    │   │   ├── fruits.txt
    │   │   ├── graph1_edges.csv
    │   │   ├── graph1_vertex.csv
    │   │   ├── graph_data.csv
    │   │   ├── people.txt
    │   │   ├── poc_1.json
    │   │   ├── relationedge.csv
    │   │   ├── relationnode.csv
    │   │   ├── relationship.csv
    │   │   ├── rita2014jan.csv
    │   │   ├── sales.json
    │   │   ├── test_file.txt
    │   │   ├── twitter_followers.txt
    │   │   ├── twitter_users.txt
    │   │   ├── users.txt
    │   │   └── users1.txt
    │   └── scala
    │   │   └── com
    │   │       └── tutorial
    │   │           ├── graphX
    │   │               ├── Basic.scala
    │   │               ├── CreateGraph.scala
    │   │               ├── CreateRddEdges.scala
    │   │               ├── CreateRddVertices.scala
    │   │               ├── GraphAlgorithms
    │   │               │   ├── ConnectedGraph.scala
    │   │               │   ├── GraphExample.scala
    │   │               │   ├── PageRank.scala
    │   │               │   ├── PageRank1.scala
    │   │               │   └── TriangleCounting.scala
    │   │               ├── GraphxExamples
    │   │               │   └── FlightApp.scala
    │   │               ├── GraphxSql.scala
    │   │               ├── Operations.scala
    │   │               ├── Operators.scala
    │   │               ├── Page1.scala
    │   │               ├── Properties.scala
    │   │               ├── Relation.scala
    │   │               ├── Relationship.scala
    │   │               ├── Rklick
    │   │               │   ├── Connected_RelationSpecialization.scala
    │   │               │   ├── Cricket_Relation.scala
    │   │               │   ├── Social_Relation.scala
    │   │               │   └── Test.scala
    │   │               ├── Rklick_Implement
    │   │               │   ├── GraphConnection.scala
    │   │               │   ├── GraphXImplement.scala
    │   │               │   ├── GraphXMain.scala
    │   │               │   └── GraphXMainImplementation.scala
    │   │               ├── Rklick_Test
    │   │               │   └── Test1.scala
    │   │               └── examlples.scala
    │   │           ├── sparkcore
    │   │               ├── CreateRDD.scala
    │   │               ├── RDDAction.scala
    │   │               ├── RDDOperations.scala
    │   │               └── RDDTransformations.scala
    │   │           ├── sparksql
    │   │               ├── BasicQueryExample.scala
    │   │               ├── CreatingDataFarmes.scala
    │   │               ├── CreatingDatasets.scala
    │   │               ├── DataFrameAPI.scala
    │   │               ├── DataFrameOperations.scala
    │   │               ├── DataSources
    │   │               │   ├── CsvFile.scala
    │   │               │   ├── JSONFile.scala
    │   │               │   └── TextFile.scala
    │   │               ├── InferringTheSchema.scala
    │   │               ├── JsonSQLContext.scala
    │   │               └── ProgrammaticallySchema.scala
    │   │           └── utils
    │   │               └── SparkCommon.scala
    └── test
    │   └── scala
    │       └── HelloSpec.scala
└── tutorial
    └── index.html


/.gitignore:
--------------------------------------------------------------------------------
 1 | .idea/
 2 | *.class
 3 | *.log
 4 | 
 5 | # sbt specific
 6 | .cache
 7 | .history
 8 | .lib/
 9 | dist/*
10 | target/
11 | /target/
12 | /logs/
13 | /RUNNING_PID
14 | lib_managed/
15 | src_managed/
16 | project/boot/
17 | /project/*-shim.sbt
18 | /project/project/
19 | /project/target/
20 | project/plugins/project/
21 | 
22 | # Scala-IDE specific
23 | .scala_dependencies
24 | .worksheet
25 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "{}"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright {yyyy} {name of copyright owner}
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 
204 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ![alt Spark Tutorial](spark-tutorial-logo.png)
 2 | 
 3 | >
 4 | > This tutorial provides a quick introduction to using Spark. It demonstrates the basic functionality of RDD and DataFrame API
 5 | >
 6 | > This is the start of using Spark with Scala, from next week onwards we would be working on this tutorial to make it grow.
 7 | > We would look at how we can add more functionality into it , then we would be adding more modules to it together. If you have any
 8 | >changes then feel free to send in pull requests and we would do the merges :) Stay tuned.
 9 | 
10 | 
11 | ####[Spark Core](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#spark-core)
12 |   * [Introduction ] (https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#introduction-to-apache-spark)
13 |  * [Features ](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#features-of-apache-spark)
14 |  * [Initializing ](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#initializing-spark)
15 |  * [RDDs] (https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#rdds)
16 |   * [Create] (https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#create-rdds)
17 |   * [Operations](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#rdds-operations)
18 |     * [Transformations] (https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#1transformations)
19 |     * [Actions] (https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#2actions)
20 | 
21 | #### [Spark SQL](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#spark-sql)
22 |  * [Introduction](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#Introduction)
23 |   * [Create SQL Context](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#create-sql-context)
24 |   * [Basic Query](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-query)
25 |  *  [DataFrames](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframes)
26 |   * [Creating DataFrames](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#creating-dataframes)
27 |   * [DataFarme API Functionality](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#datafarme-api-example-using-different-types-of-functionality)
28 |     * [Action](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#action)
29 |     * [Basic DataFrame functions](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-dataframe-functions)
30 |     * [DataFrame Operations](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#data-frame-operations)
31 |  * [Interoperating with RDDs](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#interoperating-with-rdds)
32 |   * [Inferring the Schema using Reflection](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#1-inferring-the-schema-using-reflection)
33 |   * [Programmatically Specifying the Schema](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#2-programmatically-specifying-the-schema)
34 |  * [Data Sources](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#data-sources)
35 |   * [DataFrame Operations in JSON file](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframe-operations-in-json-file)
36 |   * [DataFrame Operations in Text file](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframe-operations-in-text-file)
37 |   * [DataFrame Operations in CSV file](https://github.com/rklick-solutions/sparhttps://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#initializing-sparkk-tutorial/wiki/Spark-SQL#dataframe-operations-in-csv-file)
38 |  * [Datasets](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataset)
39 |   * [Creating Datasets](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#creating-datasets)
40 |   * [Basic Opeartion](https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-opeartion)
41 | 
42 | 
43 | 


--------------------------------------------------------------------------------
/activator:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | 
  3 | ###  ------------------------------- ###
  4 | ###  Helper methods for BASH scripts ###
  5 | ###  ------------------------------- ###
  6 | 
  7 | realpath () {
  8 | (
  9 |   TARGET_FILE="$1"
 10 | 
 11 |   cd "$(dirname "$TARGET_FILE")"
 12 |   TARGET_FILE=$(basename "$TARGET_FILE")
 13 | 
 14 |   COUNT=0
 15 |   while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
 16 |   do
 17 |       TARGET_FILE=$(readlink "$TARGET_FILE")
 18 |       cd "$(dirname "$TARGET_FILE")"
 19 |       TARGET_FILE=$(basename "$TARGET_FILE")
 20 |       COUNT=$(($COUNT + 1))
 21 |   done
 22 | 
 23 |   if [ "$TARGET_FILE" == "." -o "$TARGET_FILE" == ".." ]; then
 24 |     cd "$TARGET_FILE"
 25 |     TARGET_FILEPATH=
 26 |   else
 27 |     TARGET_FILEPATH=/$TARGET_FILE
 28 |   fi
 29 | 
 30 |   # make sure we grab the actual windows path, instead of cygwin's path.
 31 |   if ! is_cygwin; then
 32 |     echo "$(pwd -P)/$TARGET_FILE"
 33 |   else
 34 |     echo $(cygwinpath "$(pwd -P)/$TARGET_FILE")
 35 |   fi
 36 | )
 37 | }
 38 | 
 39 | # TODO - Do we need to detect msys?
 40 | 
 41 | # Uses uname to detect if we're in the odd cygwin environment.
 42 | is_cygwin() {
 43 |   local os=$(uname -s)
 44 |   case "$os" in
 45 |     CYGWIN*) return 0 ;;
 46 |     *)  return 1 ;;
 47 |   esac
 48 | }
 49 | 
 50 | # This can fix cygwin style /cygdrive paths so we get the
 51 | # windows style paths.
 52 | cygwinpath() {
 53 |   local file="$1"
 54 |   if is_cygwin; then
 55 |     echo $(cygpath -w $file)
 56 |   else
 57 |     echo $file
 58 |   fi
 59 | }
 60 | 
 61 | # Make something URI friendly
 62 | make_url() {
 63 |   url="$1"
 64 |   local nospaces=${url// /%20}
 65 |   if is_cygwin; then
 66 |     echo "/${nospaces//\\//}"
 67 |   else
 68 |     echo "$nospaces"
 69 |   fi
 70 | }
 71 | 
 72 | # Detect if we should use JAVA_HOME or just try PATH.
 73 | get_java_cmd() {
 74 |   if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]];  then
 75 |     echo "$JAVA_HOME/bin/java"
 76 |   else
 77 |     echo "java"
 78 |   fi
 79 | }
 80 | 
 81 | echoerr () {
 82 |   echo 1>&2 "$@"
 83 | }
 84 | vlog () {
 85 |   [[ $verbose || $debug ]] && echoerr "$@"
 86 | }
 87 | dlog () {
 88 |   [[ $debug ]] && echoerr "$@"
 89 | }
 90 | execRunner () {
 91 |   # print the arguments one to a line, quoting any containing spaces
 92 |   [[ $verbose || $debug ]] && echo "# Executing command line:" && {
 93 |     for arg; do
 94 |       if printf "%s\n" "$arg" | grep -q ' '; then
 95 |         printf "\"%s\"\n" "$arg"
 96 |       else
 97 |         printf "%s\n" "$arg"
 98 |       fi
 99 |     done
100 |     echo ""
101 |   }
102 | 
103 |   exec "$@"
104 | }
105 | addJava () {
106 |   dlog "[addJava] arg = '$1'"
107 |   java_args=( "${java_args[@]}" "$1" )
108 | }
109 | addApp () {
110 |   dlog "[addApp] arg = '$1'"
111 |   sbt_commands=( "${app_commands[@]}" "$1" )
112 | }
113 | addResidual () {
114 |   dlog "[residual] arg = '$1'"
115 |   residual_args=( "${residual_args[@]}" "$1" )
116 | }
117 | addDebugger () {
118 |   addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
119 | }
120 | addConfigOpts () {
121 |   dlog "[addConfigOpts] arg = '$*'"
122 |   for item in $*
123 |   do
124 |     addJava "$item"
125 |   done
126 | }
127 | # a ham-fisted attempt to move some memory settings in concert
128 | # so they need not be messed around with individually.
129 | get_mem_opts () {
130 |   local mem=${1:-1024}
131 |   local meta=$(( $mem / 4 ))
132 |   (( $meta > 256 )) || meta=256
133 |   (( $meta < 1024 )) || meta=1024
134 | 
135 |   # default is to set memory options but this can be overridden by code section below
136 |   memopts="-Xms${mem}m -Xmx${mem}m"
137 |   if [[ "${java_version}" > "1.8" ]]; then
138 |     extmemopts="-XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=${meta}m"
139 |   else
140 |     extmemopts="-XX:PermSize=64m -XX:MaxPermSize=${meta}m"
141 |   fi
142 | 
143 |   if [[ "${java_opts}" == *-Xmx* ]] || [[ "${java_opts}" == *-Xms* ]] || [[ "${java_opts}" == *-XX:MaxPermSize* ]] || [[ "${java_opts}" == *-XX:ReservedCodeCacheSize* ]] || [[ "${java_opts}" == *-XX:MaxMetaspaceSize* ]]; then
144 |     # if we detect any of these settings in ${java_opts} we need to NOT output our settings.
145 |     # The reason is the Xms/Xmx, if they don't line up, cause errors.
146 |     memopts=""
147 |     extmemopts=""
148 |   fi
149 | 
150 |   echo "${memopts} ${extmemopts}"
151 | }
152 | require_arg () {
153 |   local type="$1"
154 |   local opt="$2"
155 |   local arg="$3"
156 |   if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
157 |     die "$opt requires <$type> argument"
158 |   fi
159 | }
160 | is_function_defined() {
161 |   declare -f "$1" > /dev/null
162 | }
163 | 
164 | # If we're *not* running in a terminal, and we don't have any arguments, then we need to add the 'ui' parameter
165 | detect_terminal_for_ui() {
166 |   [[ ! -t 0 ]] && [[ "${#residual_args}" == "0" ]] && {
167 |     addResidual "ui"
168 |   }
169 |   # SPECIAL TEST FOR MAC
170 |   [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]] && [[ "${#residual_args}" == "0" ]] && {
171 |     echo "Detected MAC OSX launched script...."
172 |     echo "Swapping to UI"
173 |     addResidual "ui"
174 |   }
175 | }
176 | 
177 | # Processes incoming arguments and places them in appropriate global variables.  called by the run method.
178 | process_args () {
179 |   while [[ $# -gt 0 ]]; do
180 |     case "$1" in
181 |        -h|-help) usage; exit 1 ;;
182 |     -v|-verbose) verbose=1 && shift ;;
183 |       -d|-debug) debug=1 && shift ;;
184 |            -mem) require_arg integer "$1" "$2" && app_mem="$2" && shift 2 ;;
185 |      -jvm-debug) 
186 |         if echo "$2" | grep -E ^[0-9]+$ > /dev/null; then 
187 |             addDebugger "$2" && shift 
188 |         else
189 |             addDebugger 9999
190 |         fi 
191 |         shift ;;
192 |      -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
193 |             -D*) addJava "$1" && shift ;;
194 |             -J*) addJava "${1:2}" && shift ;;
195 |               *) addResidual "$1" && shift ;;
196 |     esac
197 |   done
198 | 
199 |   is_function_defined process_my_args && {
200 |     myargs=("${residual_args[@]}")
201 |     residual_args=()
202 |     process_my_args "${myargs[@]}"
203 |   }
204 | }
205 | 
206 | # Actually runs the script.
207 | run() {
208 |   # TODO - check for sane environment
209 | 
210 |   # process the combined args, then reset "$@" to the residuals
211 |   process_args "$@"
212 |   detect_terminal_for_ui
213 |   set -- "${residual_args[@]}"
214 |   argumentCount=$#
215 | 
216 |   #check for jline terminal fixes on cygwin
217 |   if is_cygwin; then
218 |     stty -icanon min 1 -echo > /dev/null 2>&1
219 |     addJava "-Djline.terminal=jline.UnixTerminal"
220 |     addJava "-Dsbt.cygwin=true"
221 |   fi
222 | 
223 |   # run sbt
224 |   execRunner "$java_cmd" \
225 |     "-Dactivator.home=$(make_url "$activator_home")" \
226 |     $(get_mem_opts $app_mem) \
227 |     ${java_opts[@]} \
228 |     ${java_args[@]} \
229 |     -jar "$app_launcher" \
230 |     "${app_commands[@]}" \
231 |     "${residual_args[@]}"
232 |     
233 |   local exit_code=$?
234 |   if is_cygwin; then
235 |     stty icanon echo > /dev/null 2>&1
236 |   fi
237 |   exit $exit_code
238 | }
239 | 
240 | # Loads a configuration file full of default command line options for this script.
241 | loadConfigFile() {
242 |   cat "$1" | sed '/^\#/d'
243 | }
244 | 
245 | ###  ------------------------------- ###
246 | ###  Start of customized settings    ###
247 | ###  ------------------------------- ###
248 | usage() {
249 |  cat <<EOM
250 | Usage: $script_name <command> [options]
251 | 
252 |   Command:
253 |   ui                 Start the Activator UI
254 |   new [name] [template-id]  Create a new project with [name] using template [template-id]
255 |   list-templates     Print all available template names
256 |   -h | -help         Print this message
257 | 
258 |   Options:
259 |   -v | -verbose      Make this runner chattier
260 |   -d | -debug        Set sbt log level to debug
261 |   -mem <integer>     Set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
262 |   -jvm-debug <port>  Turn on JVM debugging, open at the given port.
263 | 
264 |   # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
265 |   -java-home <path>  Alternate JAVA_HOME
266 | 
267 |   # jvm options and output control
268 |   -Dkey=val          Pass -Dkey=val directly to the java runtime
269 |   -J-X               Pass option -X directly to the java runtime
270 |                      (-J is stripped)
271 | 
272 |   # environment variables (read from context)
273 |   JAVA_OPTS          Environment variable, if unset uses ""
274 |   SBT_OPTS           Environment variable, if unset uses ""
275 |   ACTIVATOR_OPTS     Environment variable, if unset uses ""
276 | 
277 | In the case of duplicated or conflicting options, the order above
278 | shows precedence: environment variables lowest, command line options highest.
279 | EOM
280 | }
281 | 
282 | ###  ------------------------------- ###
283 | ###  Main script                     ###
284 | ###  ------------------------------- ###
285 | 
286 | declare -a residual_args
287 | declare -a java_args
288 | declare -a app_commands
289 | declare -r real_script_path="$(realpath "$0")"
290 | declare -r activator_home="$(realpath "$(dirname "$real_script_path")")"
291 | declare -r app_version="1.3.6"
292 | 
293 | declare -r app_launcher="${activator_home}/activator-launch-${app_version}.jar"
294 | declare -r script_name=activator
295 | java_cmd=$(get_java_cmd)
296 | declare -r java_opts=( "${ACTIVATOR_OPTS[@]}" "${SBT_OPTS[@]}" "${JAVA_OPTS[@]}" "${java_opts[@]}" )
297 | userhome="$HOME"
298 | if is_cygwin; then
299 |   # cygwin sets home to something f-d up, set to real windows homedir
300 |   userhome="$USERPROFILE"
301 | fi
302 | declare -r activator_user_home_dir="${userhome}/.activator"
303 | declare -r java_opts_config_home="${activator_user_home_dir}/activatorconfig.txt"
304 | declare -r java_opts_config_version="${activator_user_home_dir}/${app_version}/activatorconfig.txt"
305 | 
306 | # Now check to see if it's a good enough version
307 | declare -r java_version=$("$java_cmd" -version 2>&1 | awk -F '"' '/version/ {print $2}')
308 | if [[ "$java_version" == "" ]]; then
309 |   echo
310 |   echo No java installations was detected.
311 |   echo Please go to http://www.java.com/getjava/ and download
312 |   echo
313 |   exit 1
314 | elif [[ ! "$java_version" > "1.6" ]]; then
315 |   echo
316 |   echo The java installation you have is not up to date
317 |   echo Activator requires at least version 1.6+, you have
318 |   echo version $java_version
319 |   echo
320 |   echo Please go to http://www.java.com/getjava/ and download
321 |   echo a valid Java Runtime and install before running Activator.
322 |   echo
323 |   exit 1
324 | fi
325 | 
326 | # if configuration files exist, prepend their contents to the java args so it can be processed by this runner
327 | # a "versioned" config trumps one on the top level
328 | if [[ -f "$java_opts_config_version" ]]; then
329 |   addConfigOpts $(loadConfigFile "$java_opts_config_version")
330 | elif [[ -f "$java_opts_config_home" ]]; then
331 |   addConfigOpts $(loadConfigFile "$java_opts_config_home")
332 | fi
333 | 
334 | run "$@"
335 | 


--------------------------------------------------------------------------------
/activator-launch-1.3.6.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rklick-solutions/spark-tutorial/a6851379302d6205cce5a9ebdf5c42dbec809882/activator-launch-1.3.6.jar


--------------------------------------------------------------------------------
/activator.bat:
--------------------------------------------------------------------------------
  1 | @REM activator launcher script
  2 | @REM
  3 | @REM Environment:
  4 | @REM In order for Activator to work you must have Java available on the classpath
  5 | @REM JAVA_HOME - location of a JDK home dir (optional if java on path)
  6 | @REM CFG_OPTS  - JVM options (optional)
  7 | @REM Configuration:
  8 | @REM activatorconfig.txt found in the ACTIVATOR_HOME or ACTIVATOR_HOME/ACTIVATOR_VERSION
  9 | @setlocal enabledelayedexpansion
 10 | 
 11 | @echo off
 12 | 
 13 | set "var1=%~1"
 14 | if defined var1 (
 15 |   if "%var1%"=="help" (
 16 |     echo.
 17 |     echo Usage activator [options] [command]
 18 |     echo.
 19 |     echo Commands:
 20 |     echo ui                 Start the Activator UI
 21 |     echo new [name] [template-id]  Create a new project with [name] using template [template-id]
 22 |     echo list-templates     Print all available template names
 23 |     echo help               Print this message
 24 |     echo.
 25 |     echo Options:
 26 |     echo -jvm-debug [port]  Turn on JVM debugging, open at the given port.  Defaults to 9999 if no port given.
 27 |     echo.
 28 |     echo Environment variables ^(read from context^):
 29 |     echo JAVA_OPTS          Environment variable, if unset uses ""
 30 |     echo SBT_OPTS           Environment variable, if unset uses ""
 31 |     echo ACTIVATOR_OPTS     Environment variable, if unset uses ""
 32 |     echo.
 33 |     echo Please note that in order for Activator to work you must have Java available on the classpath
 34 |     echo.
 35 |     goto :end
 36 |   )
 37 | )
 38 | 
 39 | if "%ACTIVATOR_HOME%"=="" (
 40 | 	set "ACTIVATOR_HOME=%~dp0"
 41 | 	@REM remove trailing "\" from path
 42 | 	set ACTIVATOR_HOME=!ACTIVATOR_HOME:~0,-1!
 43 | )
 44 | 
 45 | set ERROR_CODE=0
 46 | set APP_VERSION=1.3.6
 47 | set ACTIVATOR_LAUNCH_JAR=activator-launch-%APP_VERSION%.jar
 48 | 
 49 | rem Detect if we were double clicked, although theoretically A user could
 50 | rem manually run cmd /c
 51 | for %%x in (%cmdcmdline%) do if %%~x==/c set DOUBLECLICKED=1
 52 | 
 53 | rem FIRST we load a config file of extra options (if there is one)
 54 | set "CFG_FILE_HOME=%UserProfile%\.activator\activatorconfig.txt"
 55 | set "CFG_FILE_VERSION=%UserProfile%\.activator\%APP_VERSION%\activatorconfig.txt"
 56 | set CFG_OPTS=
 57 | if exist %CFG_FILE_VERSION% (
 58 |   FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_VERSION%") DO (
 59 |     set DO_NOT_REUSE_ME=%%i
 60 |     rem ZOMG (Part #2) WE use !! here to delay the expansion of
 61 |     rem CFG_OPTS, otherwise it remains "" for this loop.
 62 |     set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME!
 63 |   )
 64 | )
 65 | if "%CFG_OPTS%"=="" (
 66 |   if exist %CFG_FILE_HOME% (
 67 |     FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_HOME%") DO (
 68 |       set DO_NOT_REUSE_ME=%%i
 69 |       rem ZOMG (Part #2) WE use !! here to delay the expansion of
 70 |       rem CFG_OPTS, otherwise it remains "" for this loop.
 71 |       set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME!
 72 |     )
 73 |   )
 74 | )
 75 | 
 76 | rem We use the value of the JAVACMD environment variable if defined
 77 | set _JAVACMD=%JAVACMD%
 78 | 
 79 | if "%_JAVACMD%"=="" (
 80 |   if not "%JAVA_HOME%"=="" (
 81 |     if exist "%JAVA_HOME%\bin\java.exe" set "_JAVACMD=%JAVA_HOME%\bin\java.exe"
 82 | 
 83 |     rem if there is a java home set we make sure it is the first picked up when invoking 'java'
 84 |     SET "PATH=%JAVA_HOME%\bin;%PATH%"
 85 |   )
 86 | )
 87 | 
 88 | if "%_JAVACMD%"=="" set _JAVACMD=java
 89 | 
 90 | rem Detect if this java is ok to use.
 91 | for /F %%j in ('"%_JAVACMD%" -version  2^>^&1') do (
 92 |   if %%~j==java set JAVAINSTALLED=1
 93 |   if %%~j==openjdk set JAVAINSTALLED=1
 94 | )
 95 | 
 96 | rem Detect the same thing about javac
 97 | if "%_JAVACCMD%"=="" (
 98 |   if not "%JAVA_HOME%"=="" (
 99 |     if exist "%JAVA_HOME%\bin\javac.exe" set "_JAVACCMD=%JAVA_HOME%\bin\javac.exe"
100 |   )
101 | )
102 | if "%_JAVACCMD%"=="" set _JAVACCMD=javac
103 | for /F %%j in ('"%_JAVACCMD%" -version 2^>^&1') do (
104 |   if %%~j==javac set JAVACINSTALLED=1
105 | )
106 | 
107 | rem BAT has no logical or, so we do it OLD SCHOOL! Oppan Redmond Style
108 | set JAVAOK=true
109 | if not defined JAVAINSTALLED set JAVAOK=false
110 | if not defined JAVACINSTALLED set JAVAOK=false
111 | 
112 | if "%JAVAOK%"=="false" (
113 |   echo.
114 |   echo A Java JDK is not installed or can't be found.
115 |   if not "%JAVA_HOME%"=="" (
116 |     echo JAVA_HOME = "%JAVA_HOME%"
117 |   )
118 |   echo.
119 |   echo Please go to
120 |   echo   http://www.oracle.com/technetwork/java/javase/downloads/index.html
121 |   echo and download a valid Java JDK and install before running Activator.
122 |   echo.
123 |   echo If you think this message is in error, please check
124 |   echo your environment variables to see if "java.exe" and "javac.exe" are
125 |   echo available via JAVA_HOME or PATH.
126 |   echo.
127 |   if defined DOUBLECLICKED pause
128 |   exit /B 1
129 | )
130 | 
131 | rem Check what Java version is being used to determine what memory options to use
132 | for /f "tokens=3" %%g in ('java -version 2^>^&1 ^| findstr /i "version"') do (
133 |     set JAVA_VERSION=%%g
134 | )
135 | 
136 | rem Strips away the " characters
137 | set JAVA_VERSION=%JAVA_VERSION:"=%
138 | 
139 | rem TODO Check if there are existing mem settings in JAVA_OPTS/CFG_OPTS and use those instead of the below
140 | for /f "delims=. tokens=1-3" %%v in ("%JAVA_VERSION%") do (
141 |     set MAJOR=%%v
142 |     set MINOR=%%w
143 |     set BUILD=%%x
144 | 
145 |     set META_SIZE=-XX:MetaspaceSize=64M -XX:MaxMetaspaceSize=256M
146 |     if "!MINOR!" LSS "8" (
147 |       set META_SIZE=-XX:PermSize=64M -XX:MaxPermSize=256M
148 |     )
149 | 
150 |     set MEM_OPTS=!META_SIZE!
151 |  )
152 | 
153 | rem We use the value of the JAVA_OPTS environment variable if defined, rather than the config.
154 | set _JAVA_OPTS=%JAVA_OPTS%
155 | if "%_JAVA_OPTS%"=="" set _JAVA_OPTS=%CFG_OPTS%
156 | 
157 | set DEBUG_OPTS=
158 | 
159 | rem Loop through the arguments, building remaining args in args variable
160 | set args=
161 | :argsloop
162 | if not "%~1"=="" (
163 |   rem Checks if the argument contains "-D" and if true, adds argument 1 with 2 and puts an equal sign between them.
164 |   rem This is done since batch considers "=" to be a delimiter so we need to circumvent this behavior with a small hack.
165 |   set arg1=%~1
166 |   if "!arg1:~0,2!"=="-D" (
167 |    	set "args=%args% "%~1"="%~2""
168 |     shift
169 |     shift
170 |     goto argsloop
171 |   )
172 | 
173 |   if "%~1"=="-jvm-debug" (
174 |     if not "%~2"=="" (
175 |       rem This piece of magic somehow checks that an argument is a number
176 |       for /F "delims=0123456789" %%i in ("%~2") do (
177 |         set var="%%i"
178 |       )
179 |       if defined var (
180 |         rem Not a number, assume no argument given and default to 9999
181 |         set JPDA_PORT=9999
182 |       ) else (
183 |         rem Port was given, shift arguments
184 |         set JPDA_PORT=%~2
185 |         shift
186 |       )
187 |     ) else (
188 |       set JPDA_PORT=9999
189 |     )
190 |     shift
191 | 
192 |     set DEBUG_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=!JPDA_PORT!
193 |     goto argsloop
194 |   )
195 |   rem else
196 |   set "args=%args% "%~1""
197 |   shift
198 |   goto argsloop
199 | )
200 | 
201 | :run
202 | 
203 | if "!args!"=="" (
204 |   if defined DOUBLECLICKED (
205 |     set CMDS="ui"
206 |   ) else set CMDS=!args!
207 | ) else set CMDS=!args!
208 | 
209 | rem We add a / in front, so we get file:///C: instead of file://C:
210 | rem Java considers the later a UNC path.
211 | rem We also attempt a solid effort at making it URI friendly.
212 | rem We don't even bother with UNC paths.
213 | set JAVA_FRIENDLY_HOME_1=/!ACTIVATOR_HOME:\=/!
214 | set JAVA_FRIENDLY_HOME=/!JAVA_FRIENDLY_HOME_1: =%%20!
215 | 
216 | rem Checks if the command contains spaces to know if it should be wrapped in quotes or not
217 | set NON_SPACED_CMD=%_JAVACMD: =%
218 | if "%_JAVACMD%"=="%NON_SPACED_CMD%" %_JAVACMD% %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS%
219 | if NOT "%_JAVACMD%"=="%NON_SPACED_CMD%" "%_JAVACMD%" %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS%
220 | 
221 | if ERRORLEVEL 1 goto error
222 | goto end
223 | 
224 | :error
225 | set ERROR_CODE=1
226 | 
227 | :end
228 | 
229 | @endlocal
230 | 
231 | exit /B %ERROR_CODE%
232 | 


--------------------------------------------------------------------------------
/activator.properties:
--------------------------------------------------------------------------------
 1 | name=spark-tutorial
 2 | title=Spark Tutorial provides a quick introduction to using Spark
 3 | description=This tutorial provides a quick introduction to using Spark. It demonstrates the basic functionality of RDD and DataFrame API
 4 | tags=Spark,Scala,RDD,SparkSQL,DataFrame,Dataset,SBT,Activator
 5 | authorName=Rklick Solutions
 6 | authorLink=https://github.com/rklick-solutions
 7 | sourceLink=https://github.com/rklick-solutions/spark-tutorial
 8 | authorLogo=http://www.rklick.com/images/logo.png
 9 | authorBio=Rklick Solutions is mature and dynamic solution oriented IT full service product development and customized consulting services company
10 | authorTwitter=RklickSolutions
11 | 


--------------------------------------------------------------------------------
/build.sbt:
--------------------------------------------------------------------------------
 1 | name := """spark-tutorial"""
 2 | 
 3 | version := "1.0"
 4 | 
 5 | scalaVersion := "2.11.7"
 6 | 
 7 | lazy val spark = "1.6.0"
 8 | 
 9 | 
10 | // Spark related dependencies
11 | 
12 | libraryDependencies ++= Seq(
13 |   "org.apache.spark" %% "spark-core" % spark,
14 |   "org.apache.spark" %% "spark-sql" % spark,
15 |   "org.apache.spark" %% "spark-streaming" % spark,
16 |   "org.apache.spark" %% "spark-graphx" % "1.6.0",
17 |   "com.databricks" %% "spark-csv" % "1.3.0"
18 | 
19 | )
20 | 
21 | // Change this to another test framework if you prefer
22 | libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.4" % "test"
23 | 
24 | 
25 | // Uncomment to use Akka
26 | //libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.3.11"
27 | 
28 | 


--------------------------------------------------------------------------------
/project/build.properties:
--------------------------------------------------------------------------------
1 | #Activator-generated Properties
2 | #Sat Nov 21 16:03:05 IST 2015
3 | template.uuid=e17acfbb-1ff5-41f5-b8cf-2c40be6a8340
4 | sbt.version=0.13.8
5 | 


--------------------------------------------------------------------------------
/spark-tutorial-logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rklick-solutions/spark-tutorial/a6851379302d6205cce5a9ebdf5c42dbec809882/spark-tutorial-logo.png


--------------------------------------------------------------------------------
/src/main/resources/Cricket_Edges.csv:
--------------------------------------------------------------------------------
 1 | id,id1,location
 2 | 2,16,delhi
 3 | 3,4,mumbai
 4 | 4,8,mumbai
 5 | 8,9,mumbai
 6 | 14,15,mumbai
 7 | 5,10,chennai
 8 | 6,7,up
 9 | 11,12,up
10 | 13,11,up
11 | 1,1,ranchi
12 | 
13 | 
14 | 
15 | 
16 | 


--------------------------------------------------------------------------------
/src/main/resources/Cricket_Edges1.csv:
--------------------------------------------------------------------------------
 1 | id,id1,specialization
 2 | 1,6,all rounder
 3 | 7,9,all rounder
 4 | 9,10,all rounder
 5 | 2,3,batting
 6 | 4,8,batting
 7 | 8,15,batting
 8 | 5,7,bowler
 9 | 8,15,batting
10 | 11,12,bowler
11 | 13,14,bowler
12 | 14,16,bowler
13 | 
14 | 
15 | 
16 | 
17 | 
18 | 
19 | 
20 | 


--------------------------------------------------------------------------------
/src/main/resources/Cricket_Edges_Follower.csv:
--------------------------------------------------------------------------------
 1 | id,id1,follow
 2 | 1,3,friend
 3 | 2,5,friend
 4 | 3,6,best Friend
 5 | 4,2,compititor
 6 | 5,6,friend
 7 | 6,8,good friend
 8 | 7,9,friend
 9 | 8,1,friend
10 | 9,4,friend
11 | 10,16,good friend
12 | 11,14,good friend
13 | 12,13,best friend
14 | 13,10,follow
15 | 14,15,follow
16 | 15,11,compititor
17 | 16,12,friend
18 | 


--------------------------------------------------------------------------------
/src/main/resources/Cricket_Node.csv:
--------------------------------------------------------------------------------
 1 | id,name,age,location,specialization
 2 | 1,Mahendra Singh Dhoni ,38,ranchi,all rounder
 3 | 2,Virat Kohli,25,delhi,batting
 4 | 3,Shikhar Dhawan,25,mumbai,batting
 5 | 4,Rohit Sharma,33,mumbai,batting
 6 | 5,Stuart Binny,22,chennai,bowler
 7 | 6,Suresh Raina,32,up,all rounder
 8 | 7,Ravindra Jadeja,28,up,all rounder
 9 | 8,Ambati Rayudu,21,mumbai,batting
10 | 9,Axar Patel,22,mumbai,all rounder
11 | 10,Ravichandran Ashwin,33,chennai,all rounder
12 | 11,Bhuvneshwar Kumar,26,up,bowler
13 | 12,Mohammed Shami,25,up,bowler
14 | 13,Umesh Yadav,30,up,bowler
15 | 14,Ishant Sharma,33,mumbai,bowler
16 | 15,Sachin,40,mumbai,batting
17 | 16,Shehwag,38,delhi,bowler
18 | 
19 | 


--------------------------------------------------------------------------------
/src/main/resources/Cricket_Node_Rklick_Test.csv:
--------------------------------------------------------------------------------
 1 | id,fid,lid,follow,name,age,location,specialization,marital,twitter,fb,movie
 2 | 1,3,1, friend,Mahendra Singh Dhoni ,38,ranchi,all rounder,Married,y,y,Bollywood
 3 | 2,5,16,friend,Virat Kohli,25,delhi,batting,Unmarried,y,y,Hollywood
 4 | 3,6,4,best friend,Shikhar Dhawan,25,mumbai,batting,Unmarried,y,y,Hollywood
 5 | 4,2,3,compititor,Rohit Sharma,33,mumbai,batting,Unmarried,y,y,Hollywood
 6 | 5,6,10, friend,Stuart Binny,22,chennai,bowler,Unmarried,y,n,Hollywood
 7 | 6,8,7,good friend,Suresh Raina,32,up,all rounder,Married,n,n,Hollywood
 8 | 7,9,6, friend,Ravindra Jadeja,28,up,all rounder,Unmarried,n,n,Hollywood
 9 | 8,1,9, friend,Ambati Rayudu,21,mumbai,batting,Unmarried,n,n,Hollywood
10 | 9,4,8, friend,Axar Patel,22,mumbai,all rounder,Unmarried,n,n,Bollywood
11 | 10,16,5,good friend,Ravichandran Ashwin,33,chennai,all rounder,Married,n,y,Bollywood
12 | 11,14,12,good friend,Bhuvneshwar Kumar,26,up,bowler,Unmarried,y,y,Bollywood
13 | 12,13,11, best friend,Mohammed Shami,25,up,bowler,Unmarried,y,y,Bollywood
14 | 13,10,12,follow,Umesh Yadav,30,up,bowler,Married,y,n,Bollywood
15 | 14,15,15,follow,Ishant Sharma,33,mumbai,bowler,Unmarried,n,n,Bollywood
16 | 15,11,14,compititor,Sachin,40,mumbai,batting,Married,y,y,Bollywood
17 | 16,12,2,friend,Shehwag,38,delhi,bowler,Married,y,n,Bollywood
18 | 


--------------------------------------------------------------------------------
/src/main/resources/Person:
--------------------------------------------------------------------------------
 1 | THEFT,1206745
 2 | BATTERY,1066110
 3 | CRIMINAL DAMAGE,672782
 4 | NARCOTICS,662257
 5 | OTHER OFFENSE,360824
 6 | ASSAULT,354583
 7 | BURGLARY,343443
 8 | MOTOR VEHICLE THEFT,278014
 9 | ROBBERY,218190
10 | DECEPTIVE PRACTICE,197477
11 | CRIMINAL TRESPASS,171363
12 | PROSTITUTION,65660
13 | WEAPONS VIOLATION,56218
14 | PUBLIC PEACE VIOLATION,42446


--------------------------------------------------------------------------------
/src/main/resources/animals.json:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rklick-solutions/spark-tutorial/a6851379302d6205cce5a9ebdf5c42dbec809882/src/main/resources/animals.json


--------------------------------------------------------------------------------
/src/main/resources/cars.csv:
--------------------------------------------------------------------------------
1 | year,make,model,comment,blank
2 | "2012","Tesla","S","No comment",
3 | 
4 | 1997,Ford,E350,"Go get one now they are going fast",
5 | 2015,Chevy,Volt


--------------------------------------------------------------------------------
/src/main/resources/cars.json:
--------------------------------------------------------------------------------
1 | [{"itemNo" : 1, "name" : "Ferrari", "speed" : 259 , "weight": 800},  {"itemNo" : 2, "name" : "Jaguar", "speed" : 274 , "weight":998},  {"itemNo" : 3, "name" : "Mercedes", "speed" : 340 , "weight": 1800},  {"itemNo" : 4, "name" : "Audi", "speed" : 345 , "weight": 875},  {"itemNo" : 5, "name" : "Lamborghini", "speed" : 355 , "weight": 1490}]
2 | 
3 | 
4 | 
5 | 


--------------------------------------------------------------------------------
/src/main/resources/cars1.json:
--------------------------------------------------------------------------------
1 | [{"itemNo" : 1, "name" : "ferrari", "speed" : 259 , "weight": 800},  {"itemNo" : 2, "name" : "jaguar", "speed" : 274 , "weight":998},  {"itemNo" : 3, "name" : "mercedes", "speed" : 340 , "weight": 1800},  {"itemNo" : 4, "name" : "audi", "speed" : 345 , "weight": 875},  {"itemNo" : 5, "name" : "lamborghini", "speed" : 355 , "weight": 1490},{"itemNo" : 6, "name" : "chevrolet", "speed" : 260 , "weight": 900},  {"itemNo" : 7, "name" : "ford", "speed" : 250 , "weight": 1061},  {"itemNo" : 8, "name" : "porche", "speed" : 320 , "weight": 1490},  {"itemNo" : 9, "name" : "bmw", "speed" : 325 , "weight": 1190},  {"itemNo" : 10, "name" : "mercedes-benz", "speed" : 312 , "weight": 1567}]
2 | 


--------------------------------------------------------------------------------
/src/main/resources/cars_price.json:
--------------------------------------------------------------------------------
1 | [{"itemNo" : 1, "name" : "Ferrari", "price" : 52000000 , "kph": 6.1},  {"itemNo" : 2, "name" : "Jaguar", "price" : 15000000 , "kph": 3.4},  {"itemNo" : 3, "name" : "Mercedes", "price" : 10000000, "kph": 3}, {"itemNo" : 4, "name" : "Audi", "price" : 5000000 , "kph": 3.6}, {"itemNo" : 5, "name" : "Lamborghini", "price" : 5000000 , "kph": 2.9}]


--------------------------------------------------------------------------------
/src/main/resources/employee.json:
--------------------------------------------------------------------------------
1 | [{"userId" : "1201", "name" : "ved", "age" : "25" , "employeeCode": "E1"},{"userId" : "1202", "name" : "krishna", "age" : "28" , "employeeCode": "E2"},{"userId" : "1203", "name" : "amit", "age" : "39" , "employeeCode": "E3"},{"userId" : "1204", "name" : "sandeep", "age" : "23" , "employeeCode": "E4"},{"userId" : "1205", "name" : "puneet", "age" : "23" , "employeeCode": "E5"}]


--------------------------------------------------------------------------------
/src/main/resources/employee.txt:
--------------------------------------------------------------------------------
1 | 1201, sandeep, 25
2 | 1202, krishna, 28
3 | 1203, amit, 39
4 | 1204, ved, 23
5 | 1205, kapil, 23


--------------------------------------------------------------------------------
/src/main/resources/employee1.json:
--------------------------------------------------------------------------------
1 | [{"id" : "1201", "firstName" : "jaykay", "age" : "25" , "employeeCode": "E1"},{"id" : "1202", "firstName" : "amit", "age" : "28" , "employeeCode": "E2"},{"id" : "1203", "firstName" : "mayank", "age" : "39" , "employeeCode": "E3"},{"id" : "1204", "firstName" : "mukesh", "age" : "23" , "employeeCode": "E4"},{"id" : "1205", "firstName" : "chiran", "age" : "23" , "employeeCode": "E5"}]


--------------------------------------------------------------------------------
/src/main/resources/employees.json:
--------------------------------------------------------------------------------
1 | [{"age":26,"userId":123,"jobTitleName":"Developer","salary":100000,"firstName":"Romin","lastName":"Irani","preferredFullName":"Romin Irani","employeeCode":"E1","region":"CA","phoneNumber":"408-1234567","emailAddress":"romin.k.irani@gmail.com"},{"age":36,"userId":456,"jobTitleName":"Developer","salary":200000,"firstName":"Neil","lastName":"Irani","preferredFullName":"Neil Irani","employeeCode":"E2","region":"CA","phoneNumber":"408-1111111","emailAddress":"neilrirani@gmail.com"},{"age":30,"userId":543,"jobTitleName":"Program Directory","salary":300000,"firstName":"Tom","lastName":"Hanks","preferredFullName":"Tom Hanks","employeeCode":"E3","region":"CA","phoneNumber":"408-2222222","emailAddress":"tomhanks@gmail.com"}]


--------------------------------------------------------------------------------
/src/main/resources/facebook_edges.csv:
--------------------------------------------------------------------------------
1 | 1,2,follows
2 | 2,3,follows
3 | 4,5,follows
4 | 5,6,follows
5 | 6,7,follows
6 | 7,8,follows
7 | 8,9,follows
8 | 9,10,follows


--------------------------------------------------------------------------------
/src/main/resources/facebook_nodes.csv:
--------------------------------------------------------------------------------
 1 | 1,A
 2 | 2,B
 3 | 3,C
 4 | 4,D
 5 | 5,E
 6 | 6,F
 7 | 7,G
 8 | 8,H
 9 | 9,I
10 | 10,J


--------------------------------------------------------------------------------
/src/main/resources/followers.txt:
--------------------------------------------------------------------------------
1 | 2 1
2 | 4 1
3 | 5 2
4 | 6 3
5 | 7 3
6 | 7 6
7 | 6 7
8 | 3 7


--------------------------------------------------------------------------------
/src/main/resources/followers1.txt:
--------------------------------------------------------------------------------
1 | 2 1
2 | 4 1
3 | 5 2
4 | 6 3
5 | 7 3
6 | 7 6
7 | 6 7
8 | 3 7


--------------------------------------------------------------------------------
/src/main/resources/fruits.json:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rklick-solutions/spark-tutorial/a6851379302d6205cce5a9ebdf5c42dbec809882/src/main/resources/fruits.json


--------------------------------------------------------------------------------
/src/main/resources/fruits.txt:
--------------------------------------------------------------------------------
1 | 1, Grapes, 25
2 | 2, Guava, 28
3 | 3, Gooseberry, 39
4 | 4,  Raisins, 23
5 | 5, Naseberry, 23


--------------------------------------------------------------------------------
/src/main/resources/graph1_edges.csv:
--------------------------------------------------------------------------------
 1 | 6,1,Sister
 2 | 1,2,Husband
 3 | 2,1,Wife
 4 | 5,1,Daughter
 5 | 5,2,Daughter
 6 | 3,1,Son
 7 | 3,2,Son
 8 | 4,1,Friend
 9 | 1,5,Father
10 | 1,3,Father
11 | 2,5,Mother
12 | 2,3,Mother


--------------------------------------------------------------------------------
/src/main/resources/graph1_vertex.csv:
--------------------------------------------------------------------------------
1 | 1,Mike,48
2 | 2,Sarah,45
3 | 3,John,25
4 | 4,Jim,53
5 | 5,Kate,22
6 | 6,Flo,52


--------------------------------------------------------------------------------
/src/main/resources/graph_data.csv:
--------------------------------------------------------------------------------
 1 | ID,RelationId,User,Relationship
 2 | 2000,2001,Zim,Like
 3 | 2001,2002,John,friend
 4 | 2002,2000,Alex,follow
 5 | 2003,2004,Charlie,Like
 6 | 2004,2005,Xin,follower
 7 | 2005,2003,Supriya,Like
 8 | 2006,2007,Jelly,follow
 9 | 2007,2008,Allen,friend
10 | 2008,2009,Pepi,friend
11 | 2009,2010,Sandy,friend
12 | 2010,2011,San,friend
13 | 2011,2012,Bob,Like
14 | 2012,2013,Alice,Like
15 | 2013,2014,Eliza,Like
16 | 2014,2015,Annie,Like
17 | 


--------------------------------------------------------------------------------
/src/main/resources/people.txt:
--------------------------------------------------------------------------------
1 | 1201, satish, 25
2 | 1202, krishna, 28
3 | 1203, amith, 39
4 | 1204, javed, 23
5 | 1205, prudvi, 23


--------------------------------------------------------------------------------
/src/main/resources/poc_1.json:
--------------------------------------------------------------------------------
 1 | {"_id":{"$oid":"5688c985e4b016b22569beee"},"id":"8ed82241-fcf4-4d28-9d74-c279ebc73d56","track_id":"8ed82241-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"One","lastname_lc":"one","fullname_lc":"test one","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"EMAIL","isdefault":true},"direction":{"id":1,"direction":"Right","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 30 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 30 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 30 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 30 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 30 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 30 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"contactmembers":[{"id":"8ed82241-fcf4-4d28-9d74-c279ebc73d56"}]}
 2 | {"_id":{"$oid":"5688caa6e4b016b22569bef1"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
 3 | {"_id":{"$oid":"5688cb3be4b016b22569bef7"},"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","track_id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"three","lastname_lc":"three","fullname_lc":"test three","phone":"2222222222","oktocall":false,"medium":{"id":1,"medium":"EMAIL","isdefault":false},"direction":{"id":1,"direction":"Left","isdefault":false},"washangup":false,"endcall":false,"starttime":"Wed Dec 15 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 15 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56"}]}
 4 | {"_id":{"$oid":"5688cb7fe4b016b22569befa"},"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","track_id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"three","lastname_lc":"three","fullname_lc":"test three","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"EMAIL","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 15 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 15 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56"}]}
 5 | {"_id":{"$oid":"5688d7a8e4b016b22569bf2c"},"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","track_id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"three","lastname_lc":"three","fullname_lc":"test three","phone":"2222222222","oktocall":false,"medium":{"id":1,"medium":"EMAIL","isdefault":false},"direction":{"id":1,"direction":"Left","isdefault":false},"washangup":false,"endcall":false,"starttime":"Wed Dec 15 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 15 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 15 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"2569bef1-fcf4-4d28-9d74-c279ebc73d56"}]}
 6 | {"_id":{"$oid":"5688d7ece4b016b22569bf2d"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
 7 | {"_id":{"$oid":"5688d7f8e4b016b22569bf2e"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
 8 | {"_id":{"$oid":"5688d804e4b016b22569bf2f"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
 9 | {"_id":{"$oid":"5688d82ee4b016b22569bf32"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
10 | {"_id":{"$oid":"5688d83ce4b016b22569bf33"},"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","track_id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56","firstname":"Test","firstname_lc":"test","lastname":"Two","lastname_lc":"two","fullname_lc":"test two","phone":"2222222222","oktocall":true,"medium":{"id":1,"medium":"SMS","isdefault":true},"direction":{"id":1,"direction":"Left","isdefault":true},"washangup":true,"endcall":true,"starttime":"Wed Dec 22 2015 20:19:00 GMT+00:00","endtime":"Wed Dec 22 2015 20:19:00 GMT+00:00","created":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"anand","name":"Anand Singh","hostip":"192.168.1.1","hostname":"localhost"},"modified":{"datestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","datetimestamp":"Wed Dec 22 2015 20:19:00 GMT+00:00","user":"raj","name":"Raj Singh","hostip":"192.168.1.2","hostname":"localhost"},"contactmembers":[{"id":"6dd23341-fcf4-4d28-9d74-c279ebc73d56"}]}
11 | 


--------------------------------------------------------------------------------
/src/main/resources/relationedge.csv:
--------------------------------------------------------------------------------
1 | 1,2,follows
2 | 2,3,follows
3 | 4,5,follows
4 | 5,6,follows
5 | 6,7,follows
6 | 7,8,follows
7 | 8,9,follows
8 | 9,10,follows


--------------------------------------------------------------------------------
/src/main/resources/relationnode.csv:
--------------------------------------------------------------------------------
 1 | 1,A,HW,M,H
 2 | 2,B,HW,M,H
 3 | 3,C,BW,M,H
 4 | 4,D,BW,M,H
 5 | 5,E,BW,EV,E
 6 | 6,F,HW,EV,E
 7 | 7,G,TW,EV,E
 8 | 8,H,TW,EV,E
 9 | 9,I,HW,EV,E
10 | 10,J,HW,EV,H


--------------------------------------------------------------------------------
/src/main/resources/relationship.csv:
--------------------------------------------------------------------------------
 1 | First Name,Last Name,E-mail Address,hollywood,bollywood,chat time,Other Country,Company,Department,Job Title,Assistant's Phone,Business Fax,Business Phone,Business Phone 2,Callback,Car Phone,Company Main Phone,Home Fax,Home Phone,Home Phone 2,ISDN,Mobile Phone,Other Fax,Other Phone,Pager,Primary Phone,Radio Phone,TTY/TDD Phone,Telex,Assistant's Name,Birthday,Manager's Name,Notes,Other Address PO Box,Spouse,Web Page,Personal Web Page
 2 | Squad 2 Serve,Placement,s2splacement@gmail.com,y,n,m,india,Squad 2 Serve,,HR Manager,,,,,,,,,,,,,,,,,,,,,,,,,,,
 3 | kapil,arora,kapilarora602@rediffmail.com,y,n,m,india,Webner Solutions Private Limited,,Software Developer,,,,,,,,,,,,,,,,,,,,,,,,,,,
 4 | Kuldeep Singh,Bisht,bisht.kul009@hotmail.com,y,n,m,india,New Delhi,,Software Devloper Trannie,,,,,,,,,,,,,,,,,,,,,,,,,,,
 5 | Lokesh,Rawte,lokesh.rawte22@gmail.com,n,n,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 6 | Vijay,Rai,raivijayrai@gmail.com,n,n,m,us,Your Merchandise,,Founder,,,,,,,,,,,,,,,,,,,,,,,,,,,
 7 | Md,Imteyaz,imteyazjeh2@gmail.com,n,n,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 8 | Kuldeep,Bisht,bisht.kul09@gmail.com,n,y,e,india,New Delhi,,Software Trainee,,,,,,,,,,,,,,,,,,,,,,,,,,,
 9 | Gurdeep Singh,Sohi,gurdeep.sohi@yahoo.com,y,y,e,us,HCL Technologies,,System Analyst,,,,,,,,,,,,,,,,,,,,,,,,,,,
10 | MOHMMAD,SAJID,sajidmohmmad86@gmail.com,y,n,e,us,Fresher,,Student,,,,,,,,,,,,,,,,,,,,,,,,,,,
11 | AAKIF,SHAIKH,salman.80@rediffmail.com,y,y,e,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
12 | Deepak,Raturi,deepak_r30@yahoo.com,y,y,e,us,Himalayan Institute of Technology,,pursuing BCA..,,,,,,,,,,,,,,,,,,,,,,,,,,,
13 | Satyapal,Singh,satyapal06@gmail.com,y,y,e,uk,Sapient Consulting Pvt Ltd,,Senior Software Engineer,,,,,,,,,,,,,,,,,,,,,,,,,,,
14 | Anuj,Kaparwan,Anuj.shakapruwan@gmail.com,n,y,e,uk,somoro innovation Pvt. Ltd.,,Software Developer,,,,,,,,,,,,,,,,,,,,,,,,,,,
15 | Mayank,Siddhu,m.siddhu14@gmail.com,n,y,e,india,ANR Software Pvt. Ltd.,,Oracle Database Administrator,,,,,,,,,,,,,,,,,,,,,,,,,,,
16 | Anand,Singh,imanandksingh@gmail.com,n,y,e,us,Rklick Solutions LLC,,Lead Software Engineer,,,,,,,,,,,,,,,,,,,,,,,,,,,
17 | tarun,chauhan,doittarun@gmail.com,n,y,m,uk,Shiksha Infotech Pvt. Ltd.,,Software Engineer,,,,,,,,,,,,,,,,,,,,,,,,,,,
18 | Vibhanshu,Gupta,vgshinchan@gmail.com,n,y,m,can,Adixsoft Technologies Pvt Ltd,,Business Development Manager,,,,,,,,,,,,,,,,,,,,,,,,,,,
19 | GIRIJESH,CHAUBEY,girijeshchaubey786@gmail.com,n,y,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
20 | Sandeep,Kumar,sandeep7020@outlook.com,n,y,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
21 | Ahmad,Salihu Ben-Musa,ahmadbinsaleh@yahoo.co.uk,n,y,e,can,MEDIWAT School of Computer Science,,Engineer,,,,,,,,,,,,,,,,,,,,,,,,,,,
22 | Ripundeep,Gill,ripungill001@gmail.com,n,y,e,uk,Drish Infotech Ltd.,,Programmer Analyst,,,,,,,,,,,,,,,,,,,,,,,,,,,
23 | Suraj,Kamya,kamyasuraj@yahoo.com,n,y,e,uk,IIMT Group of  colleges,,Assistant Professor,,,,,,,,,,,,,,,,,,,,,,,,,,,
24 | Gour Sundar,Mitra Thakur,cse.gour@gmail.com,y,y,e,pk,Dr. B. C. Roy Engineering College,,Asst. Professor (CSE),,,,,,,,,,,,,,,,,,,,,,,,,,,
25 | Cubic,Web Solutions,shilpi@cubicwebsolutions.com,y,n,m,pk,Cubic Web Solutions,,HR Manager,,,,,,,,,,,,,,,,,,,,,,,,,,,
26 | Shivani,Aggarwal,shivani.aggarwal.s@gmail.com,y,n,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
27 | Sujeet,Painuly,sujeetpainuly@gmail.com,y,n,m,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
28 | deepak,sachan,deepak.sachan97@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
29 | Neha,Thakur,neha.kss@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
30 | Shivam kumar,Shrivastava,saurabh688403@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
31 | Kapil,Arora,kapilaora786@facebook.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
32 | simranpreet,kaur,simmyaulakh@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
33 | Krishna Prasad,Mukherjee,krishna.mtechcse@gmail.com,,,,,"Honeywell Technology Solutions, Inc.",,IT Specialist,,,,,,,,,,,,,,,,,,,,,,,,,,,
34 | Swati,(Hiring Interns),hr@enterprisemonkey.com.au,,,,,Enterprise Monkey,,HR Manager,,,,,,,,,,,,,,,,,,,,,,,,,,,
35 | Ashutosh,Bhardwaj,chd.ashutosh@gmail.com,,,,,CCI Wireless,,Technical Support Specialist,,,,,,,,,,,,,,,,,,,,,,,,,,,
36 | mrinal,kashyap,mrinalkashyap89@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
37 | ijcsmc,Journal,editor.ijcsmc@gmail.com,,,,,IJCSMC,,Journalism,,,,,,,,,,,,,,,,,,,,,,,,,,,
38 | Ravi,Nandan Pandey,pandey_ravinandan@yahoo.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
39 | Sharad,Panwar,sharad_loves@yahoo.com,,,,,SWARAJ ENGINES LIMITED,,HR Officer,,,,,,,,,,,,,,,,,,,,,,,,,,,
40 | Jeevan,Thakur,jeevan@visionlaryngoscope.com,,,,,Vision Medical Devices (I) Pvt Ltd,,Director,,,,,,,,,,,,,,,,,,,,,,,,,,,
41 | Nikhil,Rai,nikhilanshu92@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
42 | Vibhu,Thakur,thakurvibhu@yahoo.com,,,,,Union Bank of India,,Information Technology Officer,,,,,,,,,,,,,,,,,,,,,,,,,,,
43 | Chiran Jeebi,Chhetri,cjccc89@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
44 | Bhagat,Singh,dtdcsec18noida@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
45 | San,Singh,ce.san.singh394@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
46 | Amit,Vikram,vikramamit89@gmail.com,,,,,SQE Labs Inc.,,Software Test Engineer,,,,,,,,,,,,,,,,,,,,,,,,,,,
47 | soumya,balachandran,somu.balan89@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
48 | Akhilesh,§|-|µk£ä●┼๑۩۩,akhi.shukla2@gmail.com,,,,,Wipro- GDC,,IVRS Developer,,,,,,,,,,,,,,,,,,,,,,,,,,,
49 | ROBIN,MATHUR,mathur.robin@gmail.com,,,,,Lovely Professional University,,Asst.Professor - CSE/IT & Coordinator (Co-Curriculum),,,,,,,,,,,,,,,,,,,,,,,,,,,
50 | santosh,kumar,santosh_1988_kumar@yahoo.co.in,,,,,Translink Express,,production manager,,,,,,,,,,,,,,,,,,,,,,,,,,,
51 | Ravi,Srivastav,raviprakash9160@gmail.com,,,,,Watanmal Group,,Accounts Executive,,,,,,,,,,,,,,,,,,,,,,,,,,,
52 | Ashutosh,Kumar,eb.ashutosh@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
53 | Jitendra,Kumar,jitendra.nke@gmail.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
54 | YASHWANT SINGH,CHAUHAN,yadusc1588@gmail.com,,,,,"Information, Public Relations and Cultural Affairs, Haryana",,Senior Programmer,,,,,,,,,,,,,,,,,,,,,,,,,,,
55 | 


--------------------------------------------------------------------------------
/src/main/resources/rita2014jan.csv:
--------------------------------------------------------------------------------
 1 | 1,3,AA,N338AA,1,12478,JFK,12892,LAX,900,914,14,1225,1238,13,385,2475
 2 | 2,4,AA,N338AA,1,12478,JFK,12892,LAX,900,857,0,1225,1226,1,385,2475
 3 | 4,6,AA,N327AA,1,12478,JFK,12892,LAX,900,1005,65,1225,1324,59,385,2475
 4 | 5,7,AA,N323AA,1,12478,JFK,12892,LAX,900,1050,110,1225,1415,110,385,2475
 5 | 6,1,AA,N319AA,1,12478,JFK,12892,LAX,900,917,17,1225,1217,0,385,2475
 6 | 7,2,AA,N328AA,1,12478,JFK,12892,LAX,900,910,10,1225,1212,0,385,2475
 7 | 8,3,AA,N323AA,1,12478,JFK,12892,LAX,900,923,23,1225,1215,0,385,2475
 8 | 9,4,AA,N339AA,1,12478,JFK,12892,LAX,900,859,0,1225,1204,0,385,2475
 9 | 10,5,AA,N319AA,1,12478,JFK,12892,LAX,900,929,29,1225,1245,20,385,2475
10 | 11,6,AA,N328AA,1,12478,JFK,12892,LAX,900,915,15,1225,1344,79,385,2475
11 | 12,7,AA,N338AA,1,12478,JFK,12892,LAX,900,854,0,1225,1208,0,385,2475
12 | 13,1,AA,N332AA,1,12478,JFK,12892,LAX,900,855,0,1225,1152,0,385,2475
13 | 14,2,AA,N339AA,1,12478,JFK,12892,LAX,900,853,0,1225,1144,0,385,2475
14 | 15,3,AA,N786AA,1,12478,JFK,12892,LAX,900,851,0,1225,1201,0,385,2475
15 | 16,4,AA,N783AA,1,12478,JFK,12892,LAX,900,856,0,1225,1216,0,385,2475
16 | 17,5,AA,N783AA,1,12478,JFK,12892,LAX,900,855,0,1225,1154,0,385,2475
17 | 18,6,AA,N783AA,1,12478,JFK,12892,LAX,900,854,0,1225,1156,0,385,2475
18 | 19,7,AA,N786AA,1,12478,JFK,12892,LAX,900,858,0,1225,1152,0,385,2475
19 | 20,1,AA,N783AA,1,12478,JFK,12892,LAX,900,855,0,1225,1209,0,385,2475
20 | 21,2,AA,N783AA,1,12478,JFK,12892,LAX,900,858,0,1225,1229,4,385,2475
21 | 23,4,AA,N787AA,1,12478,JFK,12892,LAX,900,1136,156,1225,1433,128,385,2475
22 | 24,5,AA,N785AA,1,12478,JFK,12892,LAX,900,856,0,1225,1158,0,385,2475
23 | 25,6,AA,N785AA,1,12478,JFK,12892,LAX,900,856,0,1225,1217,0,385,2475
24 | 26,7,AA,N784AA,1,12478,JFK,12892,LAX,900,852,0,1225,1226,1,385,2475
25 | 27,1,AA,N783AA,1,12478,JFK,12892,LAX,900,853,0,1225,1228,3,385,2475
26 | 28,2,AA,N783AA,1,12478,JFK,12892,LAX,900,903,3,1225,1237,12,385,2475
27 | 29,3,AA,N785AA,1,12478,JFK,12892,LAX,900,904,4,1225,1230,5,385,2475
28 | 30,4,AA,N783AA,1,12478,JFK,12892,LAX,900,907,7,1225,1247,22,385,2475
29 | 31,5,AA,N785AA,1,12478,JFK,12892,LAX,900,855,0,1225,1225,0,385,2475
30 | 


--------------------------------------------------------------------------------
/src/main/resources/sales.json:
--------------------------------------------------------------------------------
1 | [{"transactionId":111,"customerId":1,"itemId": 1,"amountPaid": 100.0},{"transactionId":112,"customerId":2,"itemId": 2,"amountPaid": 505.0},{"transactionId":113,"customerId":3,"itemId": 3,"amountPaid": 510.0},{"transactionId":114,"customerId":4,"itemId": 4,"amountPaid": 600.0},{"transactionId":115,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":116,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":117,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":118,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":119,"customerId":2,"itemId": 3,"amountPaid": 500.0},{"transactionId":120,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":121,"customerId":1,"itemId": 4,"amountPaid": 500.0},{"transactionId":122,"customerId":1,"itemId": 2,"amountPaid": 500.0},{"transactionId":123,"customerId":1,"itemId": 4,"amountPaid": 500.0},{"transactionId":124,"customerId":1,"itemId": 2,"amountPaid": 500.0}]


--------------------------------------------------------------------------------
/src/main/resources/test_file.txt:
--------------------------------------------------------------------------------
1 | JSON is a popular semistructured data format. The simplest way to load JSON data is
2 | by loading the data as a text file and then mapping over the values with a JSON
3 | parser. Likewise, we can use our preferred JSON serialization library to write out the
4 | values to strings, which we can then write out.
5 |  In Java and Scala we can also work
6 | with JSON data using a custom Hadoop format.
7 |  “JSON” on page 172 also shows how to
8 | load JSON data with Spark SQL.
9 | 


--------------------------------------------------------------------------------
/src/main/resources/twitter_followers.txt:
--------------------------------------------------------------------------------
 1 | 1 2
 2 | 2 3
 3 | 3 4
 4 | 4 5
 5 | 5 6
 6 | 6 7
 7 | 7 8
 8 | 8 9
 9 | 9 10
10 | 10 1
11 | 2 7
12 | 4 7
13 | 
14 | 


--------------------------------------------------------------------------------
/src/main/resources/twitter_users.txt:
--------------------------------------------------------------------------------
 1 | 1 Zuckerberg
 2 | 2 KATY PERRY
 3 | 3 Narendra Modi
 4 | 4 Bill Gates
 5 | 5 Jennifer Lopez
 6 | 6 Pitbull
 7 | 7 Deepika Padukone
 8 | 8 Barack Obama
 9 | 9 Cristiano Ronaldo
10 | 10 Sachin Tendulkar


--------------------------------------------------------------------------------
/src/main/resources/users.txt:
--------------------------------------------------------------------------------
1 | 1,Modi,Narender Modi
2 | 2,Kohali,Virat Kohali
3 | 3,Ved,Ved Prakash Singh
4 | 4,justinbieber,Justin Bieber
5 | 5,Sachin,Sachin Tendulkar
6 | 6,Salman,Salman khan
7 | 7,Kejriwal


--------------------------------------------------------------------------------
/src/main/resources/users1.txt:
--------------------------------------------------------------------------------
1 | 1,Modi,Narender Modi,india
2 | 2,Kohali,Virat Kohali,delhi
3 | 3,Ved,Ved Prakash Sing,up
4 | 4,justinbieber,Justin Bieber,us
5 | 5,Sachin,Sachin Tendulkar,mumbai
6 | 6,Salman,Salman khan,mumbai
7 | 7,Kejriwal,Delhi,india


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Basic.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | import org.apache.spark.rdd.RDD
 6 | 
 7 | /**
 8 |   * Created by ved on 18/2/16.
 9 |   */
10 | object Basic {
11 | 
12 |   val sc = SparkCommon.sparkContext
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 | 
17 |     /**
18 |     *First we load the vertex data in an array.
19 |       */
20 | 
21 |     val vertices = Array((1L, ("Charlotte","NY")),(2L,
22 |       ("Yonkers","NY")),(3L, ("East New York","NY")))
23 | 
24 |     /**
25 |       *Load the array of vertices into the RDD of vertices.
26 |       */
27 | 
28 |     val verRdd=sc.parallelize(vertices)
29 | 
30 |     /**
31 |       *Load the edge data in an array.
32 |       */
33 | 
34 |     val edges = Array(Edge(1L,2L,554.83),Edge(2L,3L,47.94),Edge(3L,
35 |       1L,590.37))
36 | 
37 |     /**
38 |       *Load the data into the RDD of edges.
39 |       */
40 | 
41 |     val edgRdd=sc.parallelize(edges)
42 | 
43 |     /**
44 |       *Create the graph.
45 |       */
46 | 
47 |     val graph = Graph(verRdd,edgRdd)
48 | 
49 |     /**
50 |       *Print all the vertices of the graph.
51 |       */
52 | 
53 |     graph.vertices.collect.foreach(println)
54 | 
55 |     /**
56 |       *Print all the edges of the graph.
57 |       */
58 | 
59 |     graph.edges.collect.foreach(println)
60 | 
61 |     /**
62 |       *Print the edge triplets;
63 |       *Triplet is created by adding source and destination attributes to an edge.
64 |       */
65 | 
66 |     graph.triplets.collect.foreach(println)
67 | 
68 |     /**
69 |       *In-degree of a graph is the number of inward-directed edges it has.
70 |       *  Print the in-degree of each vertex.
71 |       */
72 |     val g= graph.inDegrees
73 | 
74 |     g.foreach(println)
75 |   }
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/CreateGraph.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | import com.tutorial.utils.SparkCommon
 3 | import org.apache.spark.graphx._
 4 | 
 5 | /**
 6 |   * Created by ved on 20/2/16.
 7 |   */
 8 | object CreateGraph {
 9 | 
10 |   val sc = SparkCommon.sparkContext
11 | 
12 |   def main(args: Array[String]) {
13 | 
14 | 
15 |     /**
16 |       * First we Load the vertex data in an array:
17 |       */
18 | 
19 |     val vertices = Array(
20 |       (1L, ("Delhi", "India")),
21 |       (2L, ("Agra" ,"India")),
22 |       (3L, ("Lucknow", "India")))
23 | 
24 |     /**
25 |       * Load the array of vertices into the RDD of vertices:
26 |       *
27 |       */
28 | 
29 |     val verticeRdd = sc.parallelize(vertices)
30 | 
31 |     /**
32 |       *
33 |       * Load the edge data in an array:
34 |       */
35 |     val edges = Array(
36 |       Edge(1L, 2L, 210),
37 |       Edge(2L, 3L, 296),
38 |       Edge(3L, 1L, 420))
39 | 
40 | 
41 |     /**
42 |       * Load the data into the RDD of edges:
43 |       */
44 | 
45 |     val edgesRdd = sc.parallelize(edges)
46 | 
47 |     /**
48 |       * Create the graph:
49 |       */
50 | 
51 |     val graph = Graph(verticeRdd, edgesRdd)
52 | 
53 |     /**
54 |       * Print all the vertices of the graph:
55 |       */
56 | 
57 |     graph.vertices.collect.foreach(println)
58 | 
59 |     /**
60 |       * Print all the edges of the graph:
61 |       */
62 | 
63 |     graph.edges.collect.foreach(println)
64 | 
65 | 
66 | 
67 | 
68 |   }
69 | 
70 | }
71 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/CreateRddEdges.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx.{Graph, Edge}
 5 | 
 6 | /**
 7 |   * Created by ved on 20/2/16.
 8 |   */
 9 | object CreateRddEdges {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 | 
17 |     /**
18 |       * First we Load the vertex data in an array:
19 |       */
20 | 
21 |     val vertices = Array(
22 |       (1L, ("Delhi", "India")),
23 |       (2L, ("Agra", "India")),
24 |       (3L, ("Lucknow", "India")))
25 | 
26 |     /**
27 |       * Load the array of vertices into the RDD of vertices:
28 |       *
29 |       */
30 | 
31 |     val verticeRdd = sc.parallelize(vertices)
32 | 
33 |     /**
34 |       *
35 |       * Load the edge data in an array:
36 |       */
37 |     val edges = Array(
38 |       Edge(1L, 2L, 210),
39 |       Edge(2L, 3L, 296),
40 |       Edge(3L, 1L, 420))
41 | 
42 | 
43 |     /**
44 |       * Load the data into the RDD of edges:
45 |       */
46 | 
47 |     val edgesRdd = sc.parallelize(edges)
48 | 
49 |     /**
50 |       * Create the graph:
51 |       */
52 | 
53 |     val graph = Graph(verticeRdd, edgesRdd)
54 | 
55 |     /**
56 |       * Print all the vertices of the graph:
57 |       */
58 | 
59 |     graph.vertices.collect.foreach(println)
60 | 
61 |     /**
62 |       * Print all the edges of the graph:
63 |       */
64 | 
65 |     graph.edges.collect.foreach(println)
66 | 
67 | 
68 |   }
69 | 
70 | }
71 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/CreateRddVertices.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx.{Graph, Edge}
 5 | 
 6 | /**
 7 |   * Created by ved on 20/2/16.
 8 |   */
 9 | object CreateRddVertices {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 | 
17 |     /**
18 |       * First we Load the vertex data in an array:
19 |       */
20 | 
21 |     val vertices = Array(
22 |       (1L, ("Delhi", "India")),
23 |       (2L, ("Agra", "India")),
24 |       (3L, ("Lucknow", "India")))
25 | 
26 |     /**
27 |       * Load the array of vertices into the RDD of vertices:
28 |       *
29 |       */
30 | 
31 |     val verticeRdd = sc.parallelize(vertices)
32 | 
33 |     /**
34 |       *
35 |       * Load the edge data in an array:
36 |       */
37 |     val edges = Array(
38 |       Edge(1L, 2L, 210),
39 |       Edge(2L, 3L, 296),
40 |       Edge(3L, 1L, 420))
41 | 
42 | 
43 |     /**
44 |       * Load the data into the RDD of edges:
45 |       */
46 | 
47 |     val edgesRdd = sc.parallelize(edges)
48 | 
49 |     /**
50 |       * Create the graph:
51 |       */
52 | 
53 |     val graph = Graph(verticeRdd, edgesRdd)
54 | 
55 |     /**
56 |       * Print all the vertices of the graph:
57 |       */
58 | 
59 |     graph.vertices.collect.foreach(println)
60 | 
61 |     /**
62 |       * Print all the edges of the graph:
63 |       */
64 | 
65 |     graph.edges.collect.foreach(println)
66 | 
67 | 
68 |   }
69 | 
70 | 
71 | }
72 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphAlgorithms/ConnectedGraph.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.GraphAlgorithms
 2 | 
 3 | import org.apache.spark.graphx._
 4 | 
 5 | import com.tutorial.utils.SparkCommon
 6 | import org.apache.spark.rdd.RDD
 7 | 
 8 | /**
 9 |   * Created by ved on 28/2/16.
10 |   */
11 | object ConnectedGraph {
12 | 
13 |   val sc = SparkCommon.sparkContext
14 | 
15 |   //val file_path = "src/main/resources/facebook_edges.csv"
16 |   //val file_path1 = "src/main/resources/facebook_nodes.csv"
17 | 
18 |   val file_path = "src/main/resources/relationedge.csv"
19 |   val file_path1 = "src/main/resources/relationnode.csv"
20 | 
21 |   def main(args: Array[String]) {
22 | 
23 |     /** val edgesFile = sc.textFile(file_path)
24 | 
25 |       * val edges = edgesFile.map(_.split(",")).map(e => Edge(e(0).
26 |       * toLong,e(1).toLong,e(2)))
27 |       */
28 |     val vertices = getVertices(file_path1)
29 |     val edges = getEdges(file_path)
30 | 
31 |     val graph = Graph(vertices, edges)
32 |     val cc = graph.connectedComponents
33 | 
34 | 
35 |     val ccVertices = cc.vertices
36 |     val ccEdges =cc.edges
37 |     //print("connected vertices" )
38 |     //ccVertices.collect.foreach(println )
39 |     println("connected vertices" + ccVertices.collect().mkString(","))
40 |     println("connected edges" + ccEdges.collect().mkString(","))
41 |   }
42 | 
43 |   def getVertices(file_path1: String): RDD[(Long, String)] = {
44 |     val verticesFile = sc.textFile(file_path1)
45 | 
46 |     val vertices = verticesFile.map(_.split(",")).map(e =>
47 |       (e(0).toLong, e(1)))
48 |     vertices
49 |   }
50 | 
51 |   def getEdges(file_path: String): RDD[Edge[String]] = {
52 | 
53 | 
54 |     val edgesFile = sc.textFile(file_path)
55 | 
56 |     val edges = edgesFile.map(_.split(",")).map(e => Edge(e(0).
57 |       toLong, e(1).toLong, e(2)))
58 |     edges
59 | 
60 | 
61 |   }
62 | 
63 | 
64 | }
65 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphAlgorithms/GraphExample.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.GraphAlgorithms
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | 
 6 | /**
 7 |   * Created by ved on 21/2/16.
 8 |   */
 9 | object GraphExample {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 | 
17 |     /**
18 |       * Load my user data and parse into tuples of user id and attribute list
19 |       */
20 | 
21 |     //val users = sc.textFile("src/main/resources/users.txt")
22 |     //  .map(line => line.split(",")).map(parts => (parts.head.toLong, parts.tail))
23 | 
24 |     val users = sc.textFile("src/main/resources/users1.txt")
25 |      .map(line => line.split(",")).map(parts => (parts.head.toLong, parts.tail))
26 | 
27 |     /**
28 |       * Parse the edge data which is already in userId -> userId format.
29 |       */
30 | 
31 | 
32 |    // val followerGraph = GraphLoader.edgeListFile(sc, "src/main/resources/followers.txt")
33 | 
34 |     val followerGraph = GraphLoader.edgeListFile(sc, "src/main/resources/followers1.txt")
35 |     /**
36 |       * Attach the user attributes.
37 |       */
38 | 
39 |     val graph = followerGraph.outerJoinVertices(users) {
40 |       case (uid, deg, Some(attrList)) => attrList
41 | 
42 |       /**
43 |         * Some users may not have attributes so we set them as empty.
44 |         */
45 | 
46 |       case (uid, deg, None) => Array.empty[String]
47 |     }
48 | 
49 |     /**
50 |       * Restrict the graph to users with usernames and names.
51 |       */
52 | 
53 |     val subgraph = graph.subgraph(vpred = (vid, attr) => attr.size == 2)
54 | 
55 |     /**
56 |       * Compute the PageRank
57 |       */
58 |     val pagerankGraph = subgraph.pageRank(0.001)
59 | 
60 |     /**
61 |       * Get the attributes of the top pagerank users
62 |       */
63 | 
64 | 
65 |     val userInfoWithPageRank = subgraph.outerJoinVertices(pagerankGraph.vertices) {
66 |       case (uid, attrList, Some(pr)) => (pr, attrList.toList)
67 |       case (uid, attrList, None) => (0.0, attrList.toList)
68 |     }
69 | 
70 | 
71 |     /**
72 |       * print top 5 rank.
73 |       */
74 |     println(userInfoWithPageRank.vertices.top(5)(Ordering.by(_._2._1)).mkString("\n"))
75 | 
76 | 
77 |   }
78 | 
79 | }
80 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphAlgorithms/PageRank.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.GraphAlgorithms
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx._
  5 | 
  6 | 
  7 | /**
  8 |   * Created by ved on 20/2/16.
  9 |   */
 10 | object PageRank {
 11 | 
 12 |   val sc = SparkCommon.sparkContext
 13 | 
 14 |   def main(args: Array[String]) {
 15 | 
 16 |     val file_Path="src/main/resources/twitter_followers.txt"
 17 | 
 18 | 
 19 |     /**
 20 |       * Load the edges from file
 21 |       * with 20 partitions:
 22 |       *
 23 |       */
 24 |     val edgesFile = sc.textFile(file_Path, 20)
 25 | 
 26 |     /**
 27 |       * Flatten and convert it into an RDD
 28 |       * of "link1,link2" format and then convert it into an RDD of Edge objects:
 29 |       */
 30 | 
 31 |     val edges = edgesFile.flatMap { line =>
 32 |       val links = line.split("\\W+")
 33 |       val from = links(0)
 34 |       val to = links.tail
 35 |       for (link <- to) yield (from, link)
 36 |     }.map(e => Edge(e._1.toLong, e._2.toLong, 1))
 37 | 
 38 |     /**
 39 |       * Load the vertices from file  with 20 partitions:
 40 |       *
 41 |       */
 42 | 
 43 |     val verticesFile = sc.textFile("src/main/resources/twitter_users.txt", 20)
 44 |     /**
 45 |       * Provide an index to the vertices and then swap it to make it in the (index, title) format:
 46 |       */
 47 | 
 48 |     val vertices = verticesFile.zipWithIndex.map(_.swap)
 49 | 
 50 |     /**
 51 |       * Create the graph object:
 52 |       *
 53 |       */
 54 | 
 55 |     val graph = Graph(vertices, edges)
 56 | 
 57 |     /**
 58 |       * Run PageRank and get the vertices:
 59 |       */
 60 |     val startTime = System.currentTimeMillis()
 61 |     val ranks = graph.pageRank(0.001).vertices
 62 |     println("#########ranks#####"+ranks.collect().foreach(println))
 63 |     println(s"Taking time::::::::${System.currentTimeMillis() - startTime}")
 64 | 
 65 | 
 66 | 
 67 |     ranks.foreach(println)
 68 | 
 69 |     /**
 70 |       * As ranks is in the (vertex ID, pagerank) format,
 71 |       * swap it to make it in the (pagerank,vertex ID) format:
 72 |       */
 73 | 
 74 |     val swappedRanks = ranks.map(_.swap)
 75 | 
 76 | 
 77 |     //swappedRanks.foreach(println)
 78 | 
 79 |     println("ranks is in the (vertex ID, pagerank)" + swappedRanks.collect().mkString(" \n"))
 80 | 
 81 | 
 82 |     /**
 83 |       * Sort to get the highest ranked pages first:
 84 |       */
 85 | 
 86 |     val sortedRanks = swappedRanks.sortByKey(false)
 87 | 
 88 | 
 89 |     //sortedRanks.foreach(print)
 90 | 
 91 |     println("highest ranked pages first" + sortedRanks.collect().mkString("\n"))
 92 | 
 93 |     /**
 94 |       * Get the highest ranked page:
 95 |       */
 96 | 
 97 |     val highest = sortedRanks.first
 98 | 
 99 |     //highest.toString().mkString(" ").foreach(print)
100 |     println("highest ranked page" + highest.toString())
101 | 
102 |     /**
103 |       * The preceding command gives the vertex id,
104 |       * which you still have to look up to see the actual title with rank. Let's do a join:
105 |       */
106 | 
107 |     val allJoin = ranks.join(vertices)
108 |     println("##################################################")
109 |     println("allJoin allJoin page" + allJoin.collect().foreach(println))
110 |     println("##################################################")
111 | 
112 | 
113 |     /**
114 |       * Sort the joined RDD again after converting from the
115 |       * (vertex ID, (page rank, title))format to the (page rank, (vertex ID, title)) format:
116 |       */
117 |     val finalAll = allJoin.map(v => (v._2._1, (v._1, v._2._2))).
118 |       sortByKey(false)
119 | 
120 |     /**
121 |       * Print the top five ranked pages
122 |       */
123 | 
124 |     val startTime1 = System.currentTimeMillis()
125 |    println("top five ranked pages" + finalAll.collect().take(5).mkString("\n"))
126 |     //finalAll.collect().take(5).foreach(println)
127 |     println(s"Taking time::::::::${System.currentTimeMillis() - startTime1}")
128 | 
129 | 
130 | 
131 |     //allJoin.collect().take(5).foreach(println)
132 | 
133 | 
134 |   }
135 | 
136 | }
137 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphAlgorithms/PageRank1.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.GraphAlgorithms
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | 
 6 | /**
 7 |   * Created by ved on 20/2/16.
 8 |   */
 9 | object PageRank1 {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   val file_path = "src/main/resources/followers.txt"
14 |   val file_path1 = "src/main/resources/users.txt"
15 | 
16 |   //val file_path = "src/main/resources/relationedge.csv"
17 |   //val file_path1 = "src/main/resources/relationnode.csv"
18 | 
19 |   def main(args: Array[String]) {
20 | 
21 |     /**
22 |       * Load the edges as a graph
23 |       */
24 | 
25 |     val graph = GraphLoader.edgeListFile(sc, file_path)
26 | 
27 |     /**
28 |       * Run PageRank
29 |       */
30 | 
31 |     val startTime = System.currentTimeMillis()
32 |     val ranks = graph.pageRank(0.0001).vertices
33 |     println(s"Taking time::::::::${System.currentTimeMillis() - startTime}")
34 | 
35 |     /**
36 |       * Join the ranks with the usernames
37 |       */
38 | 
39 |     val users = sc.textFile(file_path1).map { line =>
40 |       val fields = line.split(",")
41 |       (fields(0).toLong, fields(1))
42 |     }
43 |     val ranksByUsername = users.join(ranks).map {
44 |       case (id, (username, rank)) => (username, rank)
45 |     }
46 | 
47 |     /**
48 |       * Print the result
49 |       */
50 | 
51 |     println(ranksByUsername.collect().mkString("\n"))
52 | 
53 | 
54 |   }
55 | 
56 | 
57 | }
58 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphAlgorithms/TriangleCounting.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.GraphAlgorithms
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | 
 6 | /**
 7 |   * Triangle
 8 |   * A vertex is part of a triangle when it has two adjacent vertices with an edge between them.
 9 |   * GraphX implements a triangle counting algorithm in the TriangleCount object that determines
10 |   * the number of triangles passing through each vertex, providing a measure of clustering.
11 |   * Created by ved on 21/2/16.
12 |   */
13 | object TriangleCounting {
14 | 
15 |   val sc = SparkCommon.sparkContext
16 | 
17 |   def main(args: Array[String]) {
18 | 
19 |     /**
20 |       * Load the edges in canonical order and partition the graph for triangle count
21 |       */
22 | 
23 |     val graph = GraphLoader.edgeListFile(sc, "src/main/resources/followers.txt", true).
24 |       partitionBy(PartitionStrategy.RandomVertexCut)
25 | 
26 |     /**
27 |       * Find the triangle count for each vertex
28 |       *
29 |       */
30 | 
31 |     val triCounts = graph.triangleCount().vertices
32 | 
33 |     /**
34 |       * Join the triangle counts with the usernames
35 |       *
36 |       */
37 | 
38 |     val users = sc.textFile("src/main/resources/users.txt").map { line =>
39 |       val fields = line.split(",")
40 |       (fields(0).toLong, fields(1))
41 |     }
42 |     val triCountByUsername = users.join(triCounts).map { case (id, (username, tc)) =>
43 |       (username, tc)
44 |     }
45 | 
46 |     /**
47 |       * Print the result
48 |       *
49 |       */
50 |     println(triCountByUsername.collect().mkString("\n"))
51 | 
52 | 
53 |   }
54 | 
55 | }
56 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphxExamples/FlightApp.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.GraphxExamples
  2 | 
  3 | import org.apache.spark._
  4 | import com.tutorial.utils.SparkCommon
  5 | import org.apache.spark.graphx.{Edge, Graph}
  6 | import org.apache.spark.rdd.RDD
  7 | import org.apache.spark.sql.{DataFrame, SQLContext}
  8 | import org.apache.spark.rdd.RDD
  9 | import org.apache.spark.util.IntParam
 10 | import org.apache.spark.sql.SQLContext
 11 | 
 12 | //LAB 9 TODO import classes required for using GraphX
 13 | import org.apache.spark.graphx._
 14 | import org.apache.spark.graphx.util.GraphGenerators
 15 | import org.apache.spark.util.StatCounter
 16 | 
 17 | case class Flight(dofM: String, dofW: String, carrier: String, tailnum: String, flnum: Int,
 18 |                   org_id: Long, origin: String, dest_id: Long, dest: String, crsdeptime: Double,
 19 |                   deptime: Double, depdelaymins: Double, crsarrtime: Double, arrtime: Double,
 20 |                   arrdelay: Double, crselapsedtime: Double, dist: Int)
 21 | 
 22 | /**
 23 |   * Created by ved on 12/3/16.
 24 |   */
 25 | object FlightApp {
 26 | 
 27 |   val sc = SparkCommon.sparkContext
 28 |   val sqlContext = new SQLContext(sc)
 29 | 
 30 |   def main(args: Array[String]) {
 31 | 
 32 | 
 33 |     def parseFlight(str: String): Flight = {
 34 |       val line = str.split(",")
 35 |       Flight(line(0), line(1), line(2), line(3), line(4).toInt, line(5).toLong,
 36 |         line(6), line(7).toLong, line(8), line(9).toDouble, line(10).toDouble,
 37 |         line(11).toDouble, line(12).toDouble, line(13).toDouble, line(14).toDouble, line(15).toDouble, line(16).toInt)
 38 |     }
 39 | 
 40 |     //Create RDD with the January 2014 data
 41 |     val textRDD = sc.textFile("src/main/resources/rita2014jan.csv")
 42 | 
 43 |     val flightsRDD = textRDD.map(parseFlight).cache()
 44 | 
 45 |     val airports = flightsRDD.map(flight => (flight.org_id, flight.origin)).distinct
 46 |     airports.take(1)
 47 | 
 48 |     // Defining a default vertex called nowhere
 49 |     val nowhere = "nowhere"
 50 | 
 51 |     val routes = flightsRDD.map(flight => ((flight.org_id, flight.dest_id), flight.dist)).distinct
 52 | 
 53 |     routes.cache
 54 | 
 55 |     // AirportID is numerical - Mapping airport ID to the 3-letter code
 56 |     val airportMap = airports.map { case ((org_id), name) => (org_id -> name) }.collect.toList.toMap
 57 | 
 58 |     //airportMap: scala.collection.immutable.Map[Long,String] = Map(13024 -> LMT, 10785 -> BTV, 14574 -> ROA, 14057 -> PDX, 13933 -> ORH, 11898 -> GFK, 14709 -> SCC, 15380 -> TVC,
 59 | 
 60 |     // Defining the routes as edges
 61 |     val edges = routes.map { case ((org_id, dest_id), distance) => Edge(org_id.toLong, dest_id.toLong, distance) }
 62 | 
 63 |     //Defining the Graph
 64 |     val graph = Graph(airports, edges, nowhere)
 65 | 
 66 |     // LNumber of airports
 67 |     val numairports = graph.numVertices
 68 | 
 69 |     graph.vertices.take(2)
 70 | 
 71 |     graph.edges.take(2)
 72 | 
 73 | 
 74 | 
 75 |     graph.edges.filter { case (Edge(org_id, dest_id, distance)) => distance > 1000 }.take(3)
 76 |     // res9: Array[org.apache.spark.graphx.Edge[Int]] = Array(Edge(10140,10397,1269), Edge(10140,10821,1670), Edge(10140,12264,1628))
 77 | 
 78 |     // Number of routes
 79 |     val numroutes = graph.numEdges
 80 | 
 81 |     // The EdgeTriplet class extends the Edge class by adding the srcAttr and dstAttr members which contain the source and destination properties respectively.
 82 |     graph.triplets.take(3).foreach(println)
 83 |     /*
 84 |         // Define a reduce operation to compute the highest degree vertex
 85 |         def max(a: (VertexId, Int), b: (VertexId, Int)): (VertexId, Int) = {
 86 |           if (a._2 > b._2) a else b
 87 |         }
 88 | 
 89 |         // Compute the max degrees
 90 |         val maxInDegree: (VertexId, Int) = graph.inDegrees.reduce(max)
 91 |         // maxInDegree: (org.apache.spark.graphx.VertexId, Int) = (10397,152)
 92 |         val maxOutDegree: (VertexId, Int) = graph.outDegrees.reduce(max)
 93 |         // maxOutDegree: (org.apache.spark.graphx.VertexId, Int) = (10397,153)
 94 |         val maxDegrees: (VertexId, Int) = graph.degrees.reduce(max)
 95 |         // maxDegrees: (org.apache.spark.graphx.VertexId, Int) = (10397,305)
 96 | 
 97 |         // we can compute the in-degree of each vertex (defined in GraphOps) by the following:
 98 |         // which airport has the most incoming flights?
 99 |         graph.inDegrees.collect.sortWith(_._2 > _._2).map(x => (airportMap(x._1), x._2))
100 |         //res46: Array[(String, Int)] = Array((ATL,152), (ORD,145), (DFW,143), (DEN,132), (IAH,107), (MSP,96), (LAX,82), (EWR,82), (DTW,81), (SLC,80),
101 |         graph.outDegrees.join(airports).sortBy(_._2._1, ascending = false).take(1)
102 |         val maxout = graph.outDegrees.join(airports).sortBy(_._2._1, ascending = false).take(3)
103 |         //res46: Array[(String, Int)] = Array((ATL,152), (ORD,145), (DFW,143), (DEN,132), (IAH,107), (MSP,96), (LAX,82), (EWR,82), (DTW,81), (SLC,80),
104 |         val maxIncoming = graph.inDegrees.collect.sortWith(_._2 > _._2).map(x => (airportMap(x._1), x._2)).take(3)
105 |         maxIncoming.foreach(println)
106 | 
107 | 
108 |         maxout.foreach(println)
109 | 
110 |         val maxOutgoing = graph.outDegrees.collect.sortWith(_._2 > _._2).map(x => (airportMap(x._1), x._2)).take(3)
111 |         maxOutgoing.foreach(println)
112 | 
113 |         // What are the top 10 flights from airport to airport?
114 |         graph.triplets.sortBy(_.attr, ascending = false).map(triplet =>
115 |           "There were " + triplet.attr.toString + " flights from " + triplet.srcAttr + " to " + triplet.dstAttr + ".").take(10)
116 | 
117 |         val sourceId: VertexId = 13024
118 |         // 50 + distance / 20
119 |         graph.edges.filter { case (Edge(org_id, dest_id, distance)) => distance > 1000 }.take(3)
120 | 
121 |         val gg = graph.mapEdges(e => 50.toDouble + e.attr.toDouble / 20)
122 |         val initialGraph = gg.mapVertices((id, _) => if (id == sourceId) 0.0 else Double.PositiveInfinity)*/
123 | 
124 | 
125 |   }
126 | 
127 | 
128 | }
129 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/GraphxSql.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx.{Graph, Edge}
 5 | import org.apache.spark.rdd.RDD
 6 | 
 7 | /**
 8 |   * Created by ved on 5/3/16.
 9 |   */
10 | object GraphxSql {
11 | 
12 |   def main(args: Array[String]) {
13 | 
14 |     val sc = SparkCommon.sparkContext
15 |     val sqlContext = new org.apache.spark.sql.SQLContext(sc)
16 | 
17 |     import sqlContext._
18 | 
19 | 
20 | 
21 |     val vertexArray = Array(
22 |       (1L, Peep("Kim", 23)),
23 |       (2L, Peep("Pat", 31)),
24 |       (3L, Peep("Chris", 52)),
25 |       (4L, Peep("Kelly", 39)),
26 |       (5L, Peep("Leslie", 45))
27 |     )
28 |     val edgeArray = Array(
29 |       Edge(2L, 1L, 7),
30 |       Edge(2L, 4L, 2),
31 |       Edge(3L, 2L, 4),
32 |       Edge(3L, 5L, 3),
33 |       Edge(4L, 1L, 1),
34 |       Edge(5L, 3L, 9)
35 |     )
36 | 
37 |     val vertexRDD: RDD[(Long, Peep)] = sc.parallelize(vertexArray)
38 |     val edgeRDD: RDD[Edge[Int]] = sc.parallelize(edgeArray)
39 |     val g: Graph[Peep, Int] = Graph(vertexRDD, edgeRDD)
40 | 
41 |     val results = g.triplets.filter(t => t.attr > 7)
42 | 
43 |     for (triplet <- results.collect) {
44 |       println(s"${triplet.srcAttr.name} loves ${triplet.dstAttr.name}")
45 | 
46 |      // val word = sqlContext.sql("word.parquet")
47 |      // word.registerTempTable("word")
48 | 
49 |      // val edge = sql("edge.parquet")
50 |       //edge.registerTempTable("edge")
51 | 
52 |       //sql("SELECT * FROM Peep").take(5)
53 |       //sql("SELECT * FROM edge").take(5)
54 | 
55 |       val n = sql("SELECT id, stem FROM word").distinct()
56 |       val nodes: RDD[(Long, String)] = n.map(p => (p(0).asInstanceOf[Long], p(1).asInstanceOf[String]))
57 | 
58 |       val e = sql("SELECT * FROM edge")
59 |       val edges: RDD[Edge[Int]] = e.map(p => Edge(p(0).asInstanceOf[Long], p(1).asInstanceOf[Long], 0))
60 | 
61 |       val g: Graph[String, Int] = Graph(nodes, edges)
62 | 
63 |       val ranks = g.pageRank(0.0001).vertices
64 |       ranks.join(nodes).sortBy(_._2._1, ascending = false).foreach(println)
65 | 
66 | 
67 |     }
68 |   }
69 | }
70 | case class Peep(name: String, age: Int)


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Operations.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | 
 6 | import org.apache.spark.rdd.RDD
 7 | 
 8 | /**
 9 |   * Basic operations
10 |   * Created by ved on 20/2/16.
11 |   */
12 | object Operations {
13 | 
14 |   val sc = SparkCommon.sparkContext
15 | 
16 |   def main(args: Array[String]) {
17 | 
18 |     /**
19 |       * First we Load the vertex data in an array:
20 |       */
21 | 
22 |     val vertices = Array(
23 |       (1L, ("Delhi", "India")),
24 |       (2L, ("Agra" ,"India")),
25 |       (3L, ("Lucknow", "India")))
26 | 
27 |     /**
28 |       * Load the array of vertices into the RDD of vertices:
29 |       *
30 |       */
31 | 
32 |     val verticeRdd = sc.parallelize(vertices)
33 | 
34 |     /**
35 |       *
36 |       * Load the edge data in an array:
37 |       */
38 |     val edges = Array(
39 |       Edge(1L, 2L, 210),
40 |       Edge(2L, 3L, 296),
41 |       Edge(3L, 1L, 420))
42 | 
43 | 
44 |     /**
45 |       * Load the data into the RDD of edges:
46 |       */
47 | 
48 |     val edgesRdd = sc.parallelize(edges)
49 | 
50 |     /**
51 |       * Create the graph:
52 |       */
53 | 
54 |     val graph = Graph(verticeRdd, edgesRdd)
55 | 
56 |     /**
57 |       * Print all the vertices of the graph:
58 |       */
59 | 
60 |     graph.vertices.collect.foreach(println)
61 | 
62 |     /**
63 |       * Print all the edges of the graph:
64 |       */
65 | 
66 |     graph.edges.collect.foreach(println)
67 | 
68 |     /**
69 |       * Triplet
70 |       * Print the edge triplets;
71 |       * a triplet is created by adding source and destination attributes
72 |       * to an edge:
73 |       */
74 | 
75 |     graph.triplets.collect.foreach(println)
76 | 
77 |     /**
78 |       * In-degree of a graph is the number of inward-directed edges it has. Print the in-degree
79 |       * of each vertex (as VertexRDD[Int] ):
80 |       */
81 | 
82 |     graph.inDegrees.foreach(println)
83 | 
84 | 
85 | 
86 | 
87 |   }
88 | 
89 | }
90 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Operators.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark._
  5 | import org.apache.spark.graphx._
  6 | import org.apache.spark.rdd.RDD
  7 | 
  8 | /**
  9 |   * Created by ved on 18/2/16.
 10 |   */
 11 | object Operators {
 12 | 
 13 |   val sc = SparkCommon.sparkContext
 14 | 
 15 | 
 16 |   def main(args: Array[String]) {
 17 | 
 18 |     /**
 19 |       * First we Create an RDD for vertex
 20 |       */
 21 |     val users: RDD[(VertexId, (String, String))] =
 22 |       sc.parallelize(wrapRefArray
 23 |       (Array((3L, ("Bob", "student")),
 24 |         (7L, ("ved", "postdoc")),
 25 |         (5L, ("Mittal", "prof")),
 26 |         (2L, ("Robin", "prof")),
 27 |         (4L, ("Amit", "student")))))
 28 | 
 29 |     /**
 30 |       * Create an RDD for edges.
 31 |       */
 32 |     val relationships: RDD[Edge[String]] =
 33 |       sc.parallelize(wrapRefArray
 34 |       (Array(Edge(3L, 7L, "collab"),
 35 |         Edge(5L, 3L, "advisor"),
 36 |         Edge(2L, 5L, "colleague"),
 37 |         Edge(5L, 7L, "pi"),
 38 |         Edge(4L, 0L, "student"),
 39 |         Edge(5L, 0L, "colleague"))))
 40 | 
 41 |     /**
 42 |       * Define a default user in case there are relationship with missing user
 43 |       */
 44 |     val defaultUser = ("John Doe", "Missing")
 45 | 
 46 |     /**
 47 |       * Build the initial Graph
 48 |       */
 49 | 
 50 |     val graph = Graph(users, relationships, defaultUser)
 51 | 
 52 |     /**
 53 |       * Notice that there is a user 0 (for which we have no information) connected to users
 54 |       * 4 (peter) and 5 (franklin).
 55 |       */
 56 | 
 57 |     graph.triplets.map(
 58 |       triplet => triplet.srcAttr._1
 59 |         + " is the " + triplet.attr
 60 |         + " of " + triplet.dstAttr._1
 61 |     ).collect.foreach(println(_))
 62 | 
 63 |     /**
 64 |       * Remove missing vertices as well as the edges to connected to them
 65 |       */
 66 | 
 67 | 
 68 |     val validGraph = graph.subgraph(vpred = (id, attr) => attr._2 != "Missing")
 69 | 
 70 |     /**
 71 |       * The valid subgraph will disconnect users 4 and 5 by removing user 0
 72 |       */
 73 | 
 74 | 
 75 |     validGraph.vertices.collect.foreach(println(_))
 76 | 
 77 |     validGraph.triplets.map(
 78 | 
 79 |       triplet => triplet.srcAttr._1
 80 |         + " is the " + triplet.attr
 81 |         + " of " + triplet.dstAttr._1
 82 |     ).collect.foreach(println(_))
 83 | 
 84 | 
 85 | 
 86 |     // Count all users which are postdocs
 87 |   val count_User=  graph.vertices.filter { case (id, (name, pos)) => pos == "postdoc" }.count
 88 |     print(" Count user are postdocs:" +count_User)
 89 | 
 90 | 
 91 |     // Count all the edges where src > dst
 92 |     graph.edges.filter(e => e.srcId > e.dstId).count
 93 | 
 94 |     val count_Edges= graph.edges.filter(e => e.srcId > e.dstId).count
 95 | 
 96 |     print(" count edges: " +count_Edges)
 97 | 
 98 |     val result =graph.edges.filter { case Edge(src, dst, prop) => src > dst }.count
 99 | 
100 |     print("result :" + result)
101 | 
102 |   }
103 | 
104 | }
105 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Page1.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | 
 6 | /**
 7 |   * Created by ved on 19/2/16.
 8 |   */
 9 | object Page1 {
10 |   val sc = SparkCommon.sparkContext
11 | 
12 |   def main(args: Array[String]) {
13 | 
14 | 
15 |     val PATH_ADD = "src/main/resources/followers.txt"
16 | 
17 | 
18 |     /**
19 |       * Load the graph as in the PageRank example
20 |       *
21 |       */
22 | 
23 |     val graph = GraphLoader.edgeListFile(sc, PATH_ADD)
24 | 
25 | 
26 |     /**
27 |       * Find the connected components
28 |       */
29 |     val cc = graph.connectedComponents().vertices
30 | 
31 |     /**
32 |       * Join the connected components with the usernames
33 |       */
34 |     val users = sc.textFile(PATH_ADD).map { line =>
35 |       val fields = line.split(" ")
36 |       (fields(0).toLong, fields(1))
37 |     }
38 |     val ccByUsername = users.join(cc).map {
39 |       case (id, (username, cc)) => (username, cc)
40 |     }
41 | 
42 |     /**
43 |       * Print the result
44 |       */
45 |     println(ccByUsername.collect().mkString("\n"))
46 | 
47 | 
48 |   }
49 | 
50 | }
51 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Properties.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx._
 5 | import org.apache.spark.rdd.RDD
 6 | 
 7 | /**
 8 |   * Created by ved on 19/2/16.
 9 |   */
10 | object Properties {
11 | 
12 |   val sc = SparkCommon.sparkContext
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 | 
17 |     val vertexArray = Array(
18 |       (1L, ("Sachin", 38)),
19 |       (2L, ("shehwag", 37)),
20 |       (3L, ("Dravid", 35)),
21 |       (4L, ("kohali", 22)),
22 |       (5L, ("Dhoni", 35)),
23 |       (6L, ("Raina", 30)),
24 |       (7L, ("Rohit", 28)),
25 |       (8L, ("Ashwin", 31)),
26 |       (9L, ("Zair", 35)),
27 |       (10L, ("Irfan", 32)),
28 |       (11L, ("Nehra", 34))
29 |     )
30 |     val edgeArray = Array(
31 |       Edge(2L, 1L, 7),
32 |       Edge(2L, 4L, 2),
33 |       Edge(3L, 2L, 4),
34 |       Edge(3L, 6L, 3),
35 |       Edge(4L, 1L, 1),
36 |       Edge(5L, 2L, 2),
37 |       Edge(6L, 3L, 8),
38 |       Edge(7L, 6L, 3),
39 |       Edge(8L, 2L, 2),
40 |       Edge(9L, 3L, 8),
41 |       Edge(10L, 6L, 3),
42 |       Edge(11L, 6L, 3)
43 |     )
44 | 
45 |     val vertexRDD: RDD[(Long, (String, Int))] = sc.parallelize(vertexArray)
46 |     val edgeRDD: RDD[Edge[Int]] = sc.parallelize(edgeArray)
47 | 
48 |     val graph: Graph[(String, Int), Int] = Graph(vertexRDD, edgeRDD)
49 |     graph.vertices.collect.foreach(println)
50 | 
51 | 
52 |     graph.vertices.filter { case (id, (name, age)) => age > 20 }.collect.foreach {
53 |       case (id, (name, age)) => println(s"$name is $age")
54 | 
55 | 
56 |     }
57 | 
58 |   }
59 | }
60 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Relation.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | 
 4 | import com.tutorial.utils.SparkCommon
 5 | import org.apache.spark.graphx.{Graph, Edge}
 6 | import org.apache.spark.rdd.RDD
 7 | 
 8 | //import org.graphframes.examples
 9 | 
10 | case class Relat(id: Int, name: String, age: Int, city: String, country: String)
11 | 
12 | //case Relat1()
13 | /**
14 |   * Created by ved on 4/3/16.
15 |   */
16 | object Relation {
17 | 
18 |   val sc = SparkCommon.sparkContext
19 |   val sqlContext = new org.apache.spark.sql.SQLContext(sc)
20 | 
21 |   def main(args: Array[String]) {
22 | 
23 |     /**
24 |       * First we Load the vertex data in an array:
25 |       */
26 | 
27 |     val vertexArray = Array(
28 | 
29 |       (1L, Relat(1, "Ved", 26, "Delhi", "India")),
30 |       (2L, Relat(2, "Himanshu", 24, "Agra", "India")),
31 |       (3L, Relat(3, "Amit", 25, "Lucknow", "India")),
32 |       (4L, Relat(4, "Sumit", 32, "Delhi", "India")),
33 |       (5L, Relat(5, "Sandy", 25, "Agra", "India")),
34 |       (6L, Relat(6, "Puneet", 25, "Lucknow", "India")),
35 |       (7L, Relat(7, "Md", 25, "Lucknow", "India"))
36 |     )
37 | 
38 | 
39 |     /**
40 |       * Load the array of vertices into the RDD of vertices:
41 |       *
42 |       */
43 | 
44 |     val vertexRDD: RDD[(Long, Relat)] = sc.parallelize(vertexArray)
45 | 
46 | 
47 | 
48 |     /**
49 |       *
50 |       * Load the edge data in an array:
51 |       */
52 |     val edgeArray = Array(
53 |       Edge(1L, 2L, "friend"),
54 |       Edge(2L, 3L, "follow"),
55 |       Edge(3L, 2L, "follow"),
56 |       Edge(6L, 3L, "follow"),
57 |       Edge(5L, 6L, "follow"),
58 |       Edge(5L, 4L, "friend"),
59 |       Edge(4L, 1L, "friend"),
60 |       Edge(1L, 5L, "friend")
61 |     )
62 | 
63 |     /**
64 |       * Load the data into the RDD of edges:
65 |       */
66 | 
67 |     val edgeRDD: RDD[Edge[String]] = sc.parallelize(edgeArray)
68 | 
69 |     /**
70 |       * Create the graph:
71 |       */
72 | 
73 |     val graph: Graph[Relat, String] = Graph(vertexRDD, edgeRDD)
74 | 
75 | 
76 | 
77 | 
78 |     /**
79 |       * Print all the vertices of the graph:
80 |       */
81 | 
82 |     graph.vertices.collect.foreach(println)
83 | 
84 | 
85 |     graph.edges.collect().foreach(println)
86 | 
87 |     graph.inDegrees.collect().foreach(println)
88 | 
89 |     graph.outDegrees.collect().foreach(println)
90 | 
91 |     graph.degrees.collect().foreach(println)
92 | 
93 |     //graph.filter("age").collect().groupBy("age").(println)
94 | 
95 | 
96 |   }
97 | 
98 | }
99 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Relationship.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.graphx.{VertexId, Graph, Edge}
 5 | import org.apache.spark.rdd.RDD
 6 | 
 7 | /**
 8 |   * Created by ved on 2/3/16.
 9 |   */
10 | object Relationship {
11 | 
12 |   val sc = SparkCommon.sparkContext
13 | 
14 | 
15 |   //def main(args: Array[String]) {
16 | 
17 |   //val file_path = "src/main/resources/relationship.csv"
18 | 
19 | 
20 |   // val edgesFile = sc.textFile(file_path)
21 | 
22 |   //val edges = edgesFile.map(_.split(",")).map(e => Edge(e(0).
23 |   // toLong,e(1).toLong,e(2)))
24 | 
25 |   // edges.foreach(println)
26 | 
27 |   // val vertxText = sc.textFile("users1.txt")
28 |   // val edgesText = sc.textFile("followers1.txt")
29 | 
30 |   val file_path1 = "src/main/resources/users1.txt"
31 |   val file_path = "src/main/resources/followers1.txt"
32 | 
33 |   def main(args: Array[String]) {
34 | 
35 |     /** val edgesFile = sc.textFile(file_path)
36 | 
37 |       * val edges = edgesFile.map(_.split(",")).map(e => Edge(e(0).
38 |       * toLong,e(1).toLong,e(2)))
39 |       */
40 |     val vertices = getVertices(file_path1)
41 |     val edges = getEdges(file_path)
42 | 
43 |     val graph = Graph(vertices, edges)
44 |     val cc = graph.connectedComponents
45 | 
46 |     val ccVertices = cc.vertices
47 |     ccVertices.collect.foreach(println)
48 |   }
49 | 
50 |   def getVertices(file_path1: String): RDD[(Long, String)] = {
51 |     val verticesFile = sc.textFile(file_path1)
52 | 
53 |     val vertices = verticesFile.map(_.split(",")).map(e =>
54 |       (e(0).toLong, e(1)))
55 |     vertices
56 |   }
57 | 
58 |   def getEdges(file_path: String): RDD[Edge[String]] = {
59 | 
60 | 
61 |     val edgesFile = sc.textFile(file_path)
62 | 
63 |     val edges = edgesFile.map(_.split(",")).map(e => Edge(e(0).
64 |       toLong, e(1).toLong, e(2)))
65 |     edges
66 | 
67 | 
68 |   }
69 | 
70 | 
71 | }
72 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick/Connected_RelationSpecialization.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx.{Edge, Graph}
  5 | import org.apache.spark.rdd.RDD
  6 | import org.apache.spark.sql.{DataFrame, SQLContext}
  7 | 
  8 | 
  9 | /**
 10 |   * Created by ved on 7/3/16.
 11 |   */
 12 | object Connected_RelationSpecialization {
 13 | 
 14 |   def main(args: Array[String]) {
 15 | 
 16 |     val sc = SparkCommon.sparkContext
 17 |     val sqlContext = new SQLContext(sc)
 18 | 
 19 | 
 20 |     val verticesDf = sqlContext.read
 21 |       .format("com.databricks.spark.csv")
 22 |       .option("header", "true")
 23 |       .option("inferSchema", "true")
 24 |       .load("src/main/resources/Cricket_Node.csv")
 25 | 
 26 | 
 27 |     val selectedData = verticesDf.select("id", "name", "age", "location", "specialization")
 28 |     selectedData.write
 29 |       .format("com.databricks.spark.csv")
 30 |       .option("header", "true")
 31 | 
 32 |     val edgeDf = sqlContext.read
 33 |       .format("com.databricks.spark.csv")
 34 |       .option("header", "true")
 35 |       .option("inferSchema", "true")
 36 |       .load("src/main/resources/Cricket_Edges.csv")
 37 |     //.load("src/main/resources/Cricket_Edges1.csv")
 38 | 
 39 |     /**
 40 |       * based on specialization
 41 |       * val selectedData1 = edgeDf.select("id", "id1", "specialization")
 42 |       * selectedData1.write
 43 |       * .format("com.databricks.spark.csv")
 44 |       * .option("header", "true")
 45 | 
 46 |       */
 47 | 
 48 |     /**
 49 |       * based on Location
 50 |       */
 51 |     val selectedData1 = edgeDf.select("id", "id1", "location")
 52 |     selectedData1.write
 53 |       .format("com.databricks.spark.csv")
 54 |       .option("header", "true")
 55 | 
 56 | 
 57 |     // verticesDf.show()
 58 |     //edgeDf.show()
 59 | 
 60 | 
 61 |     def getVertices(df: DataFrame): RDD[(Long, (String, Long, String, String))] = {
 62 |       df.map {
 63 |         case row => (row.getAs[Any]("id").toString.toLong,
 64 |           (row.getAs[Any]("name").toString, row.getAs[Any]("age").toString.toLong,
 65 |             row.getAs[Any]("location").toString, row.getAs[Any]("specialization").toString))
 66 |       }
 67 |     }
 68 | 
 69 |     /**
 70 |       * Based on specialization
 71 |       * def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 72 |       * df1.map {
 73 |       * case row => Edge(row.getAs[Any]("id").toString.toLong,
 74 |       * row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("specialization").toString)
 75 |       * }
 76 |       * }
 77 |       */
 78 | 
 79 |     def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 80 |       df1.map {
 81 |         case row => Edge(row.getAs[Any]("id").toString.toLong,
 82 |           row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("location").toString)
 83 |       }
 84 |     }
 85 | 
 86 |     val vertices = getVertices(verticesDf)
 87 |     val edges = getEdges(edgeDf)
 88 | 
 89 |     /**
 90 |       * Create Graph.
 91 |       */
 92 | 
 93 |     val graph = Graph(vertices, edges)
 94 | 
 95 | 
 96 |     graph.connectedComponents().vertices.collect.foreach(println)
 97 |     graph.connectedComponents().edges.collect.foreach(println)
 98 | 
 99 |     //graph.collectNeighborIds()
100 |     //graph.vertices.collect().foreach(println)
101 | 
102 |     graph.connectedComponents().vertices
103 | 
104 |     /** val connByAllRounder = vertices.join(connected).map {
105 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
106 |       * }
107 |       * val connByBowler = vertices.join(connected).map {
108 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
109 | 
110 |       * }
111 | 
112 |       * val connByBatting = vertices.join(connected).map {
113 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
114 | 
115 |       * }
116 | 
117 |       * connByAllRounder.collect.foreach {
118 |       * case (conn, name, specialization) =>
119 |       * println(f"AllRounder $conn $name $specialization")
120 |       * }
121 |       * connByBatting.collect.foreach {
122 |       * case (conn, name, specialization) =>
123 |       * println(f"Batting $conn $name $specialization")
124 |       * }
125 |       * connByBowler.collect.foreach {
126 |       * case (conn, name, specialization) =>
127 |       * println(f"Bowler$conn $name $specialization")
128 |       * }
129 | 
130 |       */
131 | 
132 |     /**
133 |       * 1.Analysis data on basis of specialization
134 |       *
135 |       */
136 | 
137 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
138 |       specialization.equals("all rounder")
139 |     }.collect.foreach(println)
140 | 
141 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
142 |       specialization.equals("batting")
143 |     }.collect.foreach(println)
144 | 
145 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
146 |       specialization.equals("bowler")
147 |     }.collect.foreach(println)
148 | 
149 | 
150 |     /**
151 |       * 2. Analysis based on location
152 |       *
153 |       */
154 | 
155 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
156 |       location.equals("delhi")
157 |     }.collect.foreach(println)
158 | 
159 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
160 |       location.equals("mumbai")
161 |     }.collect.foreach(println)
162 | 
163 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
164 |       location.equals("chennai")
165 |     }.collect.foreach(println)
166 | 
167 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
168 |       location.equals("up")
169 |     }.collect.foreach(println)
170 | 
171 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
172 |       location.equals("ranchi")
173 |     }.collect.foreach(println)
174 | 
175 | 
176 |     /**
177 |       * Analysis based Age
178 |       *
179 |       */
180 | 
181 | 
182 |     val c2 = graph.vertices.filter { case (id, (name, age, location, specialization)) =>
183 |       age.toLong > 26.toShort
184 |     }
185 | 
186 |     c2.collect.foreach {
187 |       case (id, (name, age, location, specialization)) =>
188 |         println(f"Younger:$id $name $age")
189 | 
190 | 
191 |     }
192 | 
193 |     val c1 = graph.vertices.filter { case (id, (name, age, location, specialization)) =>
194 |       age.toLong < 26.toShort
195 |     }
196 | 
197 |     c1.collect.foreach {
198 |       case (id, (name, age, location, specialization)) =>
199 |         println(f"young :$id $name $age")
200 | 
201 | 
202 |     }
203 | 
204 |     println(c1.top(5)(Ordering.by(_._2._1)).mkString("\n"))
205 | 
206 |     //graph.vertices.filter()
207 | 
208 | 
209 |   }
210 | }
211 | 
212 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick/Cricket_Relation.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick
  2 | import com.tutorial.utils.SparkCommon
  3 | import org.apache.spark.graphx._
  4 | import org.apache.spark.rdd.RDD
  5 | import org.apache.spark.sql.{DataFrame, SQLContext}
  6 | 
  7 | /**
  8 |   * Created by ved on 7/3/16.
  9 |   */
 10 | object Cricket_Relation {
 11 | 
 12 |   val sc = SparkCommon.sparkContext
 13 | 
 14 | 
 15 |   val sqlContext = new SQLContext(sc)
 16 | 
 17 | 
 18 |   def main(args: Array[String]) {
 19 | 
 20 |     //val file_path1 = "src/main/resources/Cricket_Node.csv"
 21 |     // val file_path2 = "src/main/resources/Cricket_Edges.csv"
 22 | 
 23 |     val verticesDf = sqlContext.read
 24 |       .format("com.databricks.spark.csv")
 25 |       .option("header", "true")
 26 |       .option("inferSchema", "true")
 27 |       .load("src/main/resources/Cricket_Node.csv")
 28 | 
 29 | 
 30 |     val selectedData = verticesDf.select("id", "name", "age", "location", "specialization")
 31 |     selectedData.write
 32 |       .format("com.databricks.spark.csv")
 33 |       .option("header", "true")
 34 | 
 35 |     val edgeDf = sqlContext.read
 36 |       .format("com.databricks.spark.csv")
 37 |       .option("header", "true")
 38 |       .option("inferSchema", "true")
 39 |       .load("src/main/resources/Cricket_Edges.csv")
 40 | 
 41 |     val selectedData1 = edgeDf.select("id", "id1", "location")
 42 |     selectedData1.write
 43 |       .format("com.databricks.spark.csv")
 44 |       .option("header", "true")
 45 | 
 46 |     // verticesDf.show()
 47 |     //edgeDf.show()
 48 | 
 49 | 
 50 |     def getVertices(df: DataFrame): RDD[(Long, (String, Long, String, String))] = {
 51 |       df.map {
 52 |         case row => (row.getAs[Any]("id").toString.toLong,
 53 |           (row.getAs[Any]("name").toString, row.getAs[Any]("age").toString.toLong,
 54 |             row.getAs[Any]("location").toString, row.getAs[Any]("specialization").toString))
 55 |       }
 56 |     }
 57 | 
 58 |     def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 59 |       df1.map {
 60 |         case row => Edge(row.getAs[Any]("id").toString.toLong,
 61 |           row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("location").toString)
 62 |       }
 63 |     }
 64 | 
 65 | 
 66 |     val vertices = getVertices(verticesDf)
 67 |     val edges = getEdges(edgeDf)
 68 | 
 69 |     /**
 70 |       * Create Graph.
 71 |       */
 72 | 
 73 |     val graph = Graph(vertices, edges)
 74 | 
 75 |     /**
 76 |       * Connected Vertices Count
 77 |       *
 78 |       */
 79 |     val connected_vertices = vertices.count()
 80 |     println("connected vertices" + connected_vertices)
 81 | 
 82 |     /**
 83 |       * Connected Edges Count
 84 |       *
 85 |       */
 86 |     val connected_edges = edges.count()
 87 |     println("connected edges" + connected_edges)
 88 | 
 89 | 
 90 |     /**
 91 |       * Connected Component.
 92 |       */
 93 | 
 94 |     val cc = graph.connectedComponents()
 95 | 
 96 |     /**
 97 |       * No of Vertices show
 98 |       */
 99 |     val ccVertices = cc.vertices
100 |     println("connected vertices" + ccVertices.collect.mkString("\n"))
101 | 
102 |     /**
103 |       * No of Edges show
104 |       */
105 |     val ccEdges = cc.edges
106 |     println("connected edges" + ccEdges.collect.mkString("\n"))
107 | 
108 |     //page rank
109 | 
110 |     val c1 = graph.vertices.filter { case (id, (name, age, location, specialization)) => age.
111 |       toLong > 20
112 |     }.count
113 | 
114 | 
115 |     // val c2 = graph.edges.collect().foreach(println)
116 |     val c2 = graph.edges.filter { case Edge(src, to, dest)
117 |     => dest == "delhi" | dest == "up"
118 |     }.count
119 |     println("Vertices count : " + c1)
120 |     println("Edges count : " + c2)
121 | 
122 |     val startTime = System.currentTimeMillis()
123 |     val tolerance = 0.0001
124 |     val ranking = graph.pageRank(tolerance).vertices
125 |     val rankByPerson = vertices.join(ranking).map {
126 |       case (id, (name, rank)) => (rank, id, name)
127 |     }
128 | 
129 |     rankByPerson.collect().foreach {
130 |       case (rank, id, name) => println(f"Rank $rank%1.2f id $id name $name")
131 |     }
132 | 
133 |     //println(s"Taking time::::::::${System.currentTimeMillis() - startTime}")
134 | 
135 |     //val finalAll = rankByPerson.sortBy(v => (v._1, v._2, v._2 )).foreach(println)
136 |     //sortByKey(false)
137 | 
138 | 
139 |     val startTime1 = System.currentTimeMillis()
140 |     val finalAll = rankByPerson.sortBy(v => (v._1, v._2, v._2)).foreach(println)
141 |     //println("top five ranked pages" + finalAll.collect().take(5).mkString("\n"))
142 |     //finalAll.collect().take(5).foreach(println)
143 |     println(s"Taking time::::::::${System.currentTimeMillis() - startTime1}")
144 | 
145 |     // traingle count
146 |     val tCount = graph.triangleCount().vertices
147 |     println(tCount.collect().mkString("\n"))
148 | 
149 |     //connected
150 |     val iterations = 1000
151 |     val connected = graph.connectedComponents().vertices
152 |     val connectedS = graph.stronglyConnectedComponents(iterations).vertices
153 | 
154 |     val connByName = vertices.join(connected).map {
155 |       case (id, ((name, age, location, specialization), conn)) => (conn, name, age)
156 |     }
157 |     val connByName1 = vertices.join(connectedS).map {
158 |       case (id, ((name, age, location, specialization), conn)) => (conn, name, age)
159 |     }
160 | 
161 |     connByName.collect().foreach {
162 |       case (conn, name, age) =>
163 |         println(f"Weak $conn $name $age")
164 |     }
165 | 
166 |     connByName1.collect().foreach {
167 |       case (conn, name, age) =>
168 |         println(f"Strong $conn $name $age")
169 |     }
170 |     verticesDf.registerTempTable("Person")
171 | 
172 |     val query = s"""select * from (select age,name from Person group by age,name having count(*) > 1) Person"""
173 |     val result1 = sqlContext.sql(query)
174 | 
175 |     println("Filter data:" + result1.collect.mkString(","))
176 | 
177 | 
178 |   }
179 | 
180 | }
181 | 
182 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick/Social_Relation.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx.{Graph, Edge}
  5 | import org.apache.spark.rdd.RDD
  6 | import org.apache.spark.sql.{DataFrame, SQLContext}
  7 | 
  8 | /**
  9 |   * Created by ved on 8/3/16.
 10 |   */
 11 | object Social_Relation {
 12 | 
 13 |   def main(args: Array[String]) {
 14 | 
 15 |     val sc = SparkCommon.sparkContext
 16 |     val sqlContext = new SQLContext(sc)
 17 | 
 18 | 
 19 |     val verticesDf = sqlContext.read
 20 |       .format("com.databricks.spark.csv")
 21 |       .option("header", "true")
 22 |       .option("inferSchema", "true")
 23 |       .load("src/main/resources/Cricket_Node.csv")
 24 | 
 25 | 
 26 |     val selectedData = verticesDf.select("id", "name", "age", "location", "specialization")
 27 |     selectedData.write
 28 |       .format("com.databricks.spark.csv")
 29 |       .option("header", "true")
 30 | 
 31 |     val edgeDf = sqlContext.read
 32 |       .format("com.databricks.spark.csv")
 33 |       .option("header", "true")
 34 |       .option("inferSchema", "true")
 35 |       // .load("src/main/resources/Cricket_Edges.csv")
 36 |       //.load("src/main/resources/Cricket_Edges1.csv")
 37 |       .load("src/main/resources/Cricket_Edges_Follower.csv")
 38 | 
 39 |     /**
 40 |       * based on specialization
 41 |       * val selectedData1 = edgeDf.select("id", "id1", "specialization")
 42 |       * selectedData1.write
 43 |       * .format("com.databricks.spark.csv")
 44 |       * .option("header", "true")
 45 | 
 46 |       */
 47 | 
 48 |     /**
 49 |       * based on Location
 50 | 
 51 |       * val selectedData1 = edgeDf.select("id", "id1", "location")
 52 |       * selectedData1.write
 53 |       * .format("com.databricks.spark.csv")
 54 |       * .option("header", "true")
 55 |       */
 56 | 
 57 |     /**
 58 |       * based on Location
 59 |       */
 60 | 
 61 |     val selectedData1 = edgeDf.select("id", "id1", "follow")
 62 |     selectedData1.write
 63 |       .format("com.databricks.spark.csv")
 64 |       .option("header", "true")
 65 | 
 66 | 
 67 | 
 68 |     // verticesDf.show()
 69 |     //edgeDf.show()
 70 | 
 71 | 
 72 |     def getVertices(df: DataFrame): RDD[(Long, (String, Long, String, String))] = {
 73 |       df.map {
 74 |         case row => (row.getAs[Any]("id").toString.toLong,
 75 |           (row.getAs[Any]("name").toString, row.getAs[Any]("age").toString.toLong,
 76 |             row.getAs[Any]("location").toString, row.getAs[Any]("specialization").toString))
 77 |       }
 78 |     }
 79 | 
 80 |     /**
 81 |       * Based on specialization
 82 |       * def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 83 |       * df1.map {
 84 |       * case row => Edge(row.getAs[Any]("id").toString.toLong,
 85 |       * row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("specialization").toString)
 86 |       * }
 87 |       * }
 88 |       */
 89 | 
 90 |     /** def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 91 |       * df1.map {
 92 |       * case row => Edge(row.getAs[Any]("id").toString.toLong,
 93 |       * row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("location").toString)
 94 |       * }
 95 |       * }
 96 | 
 97 |       */
 98 | 
 99 |     def getEdges(df1: DataFrame): RDD[Edge[String]] = {
100 |       df1.map {
101 |         case row => Edge(row.getAs[Any]("id").toString.toLong,
102 |           row.getAs[Any]("id1").toString.toLong, row.getAs[Any]("follow").toString)
103 |       }
104 |     }
105 | 
106 |     val vertices = getVertices(verticesDf)
107 |     val edges = getEdges(edgeDf)
108 | 
109 |     /**
110 |       * Create Graph.
111 |       */
112 | 
113 |     val graph = Graph(vertices, edges)
114 | 
115 | 
116 |     graph.connectedComponents().vertices.collect.foreach(println)
117 |     graph.connectedComponents().edges.collect.foreach(println)
118 | 
119 |     //graph.collectNeighborIds()
120 |     //graph.vertices.collect().foreach(println)
121 | 
122 |     graph.connectedComponents().vertices
123 | 
124 |     /** val connByAllRounder = vertices.join(connected).map {
125 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
126 |       * }
127 |       * val connByBowler = vertices.join(connected).map {
128 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
129 | 
130 |       * }
131 | 
132 |       * val connByBatting = vertices.join(connected).map {
133 |       * case (id, ((name, age, location, specialization), conn)) => (conn, name, specialization)
134 | 
135 |       * }
136 | 
137 |       * connByAllRounder.collect.foreach {
138 |       * case (conn, name, specialization) =>
139 |       * println(f"AllRounder $conn $name $specialization")
140 |       * }
141 |       * connByBatting.collect.foreach {
142 |       * case (conn, name, specialization) =>
143 |       * println(f"Batting $conn $name $specialization")
144 |       * }
145 |       * connByBowler.collect.foreach {
146 |       * case (conn, name, specialization) =>
147 |       * println(f"Bowler$conn $name $specialization")
148 |       * }
149 | 
150 |       */
151 | 
152 |     /*/**
153 |       * 1.Analysis data on basis of specialization
154 |       *
155 |       */
156 | 
157 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
158 |       specialization.equals("all rounder")
159 |     }.collect.foreach(println)
160 | 
161 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
162 |       specialization.equals("batting")
163 |     }.collect.foreach(println)
164 | 
165 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
166 |       specialization.equals("bowler")
167 |     }.collect.foreach(println)
168 | 
169 | 
170 |     /**
171 |       * 2. Analysis based on location
172 |       *
173 |       */
174 | 
175 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
176 |       location.equals("delhi")
177 |     }.collect.foreach(println)
178 | 
179 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
180 |       location.equals("mumbai")
181 |     }.collect.foreach(println)
182 | 
183 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
184 |       location.equals("chennai")
185 |     }.collect.foreach(println)
186 | 
187 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
188 |       location.equals("up")
189 |     }.collect.foreach(println)
190 | 
191 |     graph.vertices.filter { case (id, (name, age, location, specialization)) =>
192 |       location.equals("ranchi")
193 |     }.collect.foreach(println)
194 | 
195 | */
196 |     /**
197 |       * Analysis based Age
198 |       *
199 |       */
200 | 
201 | 
202 |     //  graph.vertices.filter { case (id, (name, age, location, specialization)) =>
203 |     //   age.toLong > 26
204 |     //  }.collect.foreach(println)
205 | 
206 |     val c2 = graph.edges.filter { case Edge(from, to, property)
207 |     => property == "friend" | property == "good friend" | property == "best friend" | property == "follow" | property == "compititor"
208 |     }.collect.foreach(println)
209 | 
210 |     /**
211 |       * count the edges
212 |       */
213 |     //println("count edge:"+c2)
214 |   }
215 | }
216 | 
217 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick/Test.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx.{Graph, Edge}
  5 | import org.apache.spark.rdd.RDD
  6 | import org.apache.spark.sql.{DataFrame, SQLContext}
  7 | 
  8 | /**
  9 |   * Created by ved on 9/3/16.
 10 |   */
 11 | object Test {
 12 | 
 13 |   val sc = SparkCommon.sparkContext
 14 | 
 15 | 
 16 |   val sqlContext = new SQLContext(sc)
 17 | 
 18 | 
 19 |   def main(args: Array[String]) {
 20 | 
 21 |     //val file_path1 = "src/main/resources/Cricket_Node.csv"
 22 |     // val file_path2 = "src/main/resources/Cricket_Edges.csv"
 23 | 
 24 |     val verticesDf = sqlContext.read
 25 |       .format("com.databricks.spark.csv")
 26 |       .option("header", "true")
 27 |       .option("inferSchema", "true")
 28 |       .load("src/main/resources/Cricket_Node.csv")
 29 | 
 30 | 
 31 |     val selectedData = verticesDf.select("id", "name", "age", "location", "specialization")
 32 |     selectedData.write
 33 |       .format("com.databricks.spark.csv")
 34 |       .option("header", "true")
 35 | 
 36 |     val edgeDf = sqlContext.read
 37 |       .format("com.databricks.spark.csv")
 38 |       .option("header", "true")
 39 |       .option("inferSchema", "true")
 40 |       .load("src/main/resources/Cricket_Node.csv")
 41 | 
 42 |     val selectedData1 = edgeDf.select("id", "name", "age", "location", "specialization")
 43 |     selectedData1.write
 44 |       .format("com.databricks.spark.csv")
 45 |       .option("header", "true")
 46 | 
 47 |     // verticesDf.show()
 48 |     //edgeDf.show()
 49 | 
 50 | 
 51 |     def getVertices(df: DataFrame): RDD[(Long, (String, Long, String, String))] = {
 52 |       df.map {
 53 |         case row => (row.getAs[Any]("id").toString.toLong,
 54 |           (row.getAs[Any]("name").toString, row.getAs[Any]("age").toString.toLong,
 55 |             row.getAs[Any]("location").toString, row.getAs[Any]("specialization").toString))
 56 |       }
 57 |     }
 58 | 
 59 |     def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 60 |       df1.map {
 61 |         case row => Edge(row.getAs[Any]("id").toString.toLong,
 62 |           row.getAs[Any]("age").toString.toLong, row.getAs[Any]("location").toString)
 63 |       }
 64 |     }
 65 | 
 66 | 
 67 |     val vertices = getVertices(verticesDf)
 68 |     val edges = getEdges(edgeDf)
 69 | 
 70 |     /**
 71 |       * Create Graph.
 72 |       */
 73 | 
 74 |     val graph = Graph(vertices, edges)
 75 | 
 76 |     /**
 77 |       * Connected Vertices Count
 78 |       *
 79 |       */
 80 |     val connected_vertices = vertices.count()
 81 |     println("connected vertices" + connected_vertices)
 82 | 
 83 |     /**
 84 |       * Connected Edges Count
 85 |       *
 86 |       */
 87 |     val connected_edges = edges.count()
 88 |     println("connected edges" + connected_edges)
 89 | 
 90 | 
 91 |     /**
 92 |       * Connected Component.
 93 |       */
 94 | 
 95 |     val cc = graph.connectedComponents()
 96 | 
 97 |     /**
 98 |       * No of Vertices show
 99 |       */
100 |     val ccVertices = cc.vertices
101 |     println("connected vertices" + ccVertices.collect.mkString("\n"))
102 | 
103 |     /**
104 |       * No of Edges show
105 |       */
106 |     val ccEdges = cc.edges
107 |     println("connected edges" + ccEdges.collect.mkString("\n"))
108 | 
109 |     //page rank
110 | 
111 |    // val c1 = graph.vertices.filter { case (id, (name, age, location, specialization)) => age.
112 |    //   toLong > 20
113 |   //  }.count
114 | 
115 | 
116 |     // val c2 = graph.edges.collect().foreach(println)
117 |    // val c2 = graph.edges.filter { case Edge(src, to, dest)
118 |    // => dest == "delhi" | dest == "up"
119 |   //  }.count
120 |   //  println("Vertices count : " + c1)
121 |    // println("Edges count : " + c2)
122 | 
123 |     val startTime = System.currentTimeMillis()
124 |     val tolerance = 0.0001
125 |     val ranking = graph.pageRank(tolerance).vertices
126 |     val rankByPerson = vertices.join(ranking).map {
127 |       case (id, (name, rank)) => (rank, id, name)
128 |     }
129 | 
130 |     rankByPerson.collect().foreach {
131 |       case (rank, id, name) => println(f"Rank $rank%1.2f id $id name $name")
132 |     }
133 | 
134 |     //println(s"Taking time::::::::${System.currentTimeMillis() - startTime}")
135 | 
136 |     //val finalAll = rankByPerson.sortBy(v => (v._1, v._2, v._2 )).foreach(println)
137 |     //sortByKey(false)
138 | 
139 | 
140 |     val startTime1 = System.currentTimeMillis()
141 |     val finalAll = rankByPerson.sortBy(v => (v._1, v._2, v._2)).foreach(println)
142 |     //println("top five ranked pages" + finalAll.collect().take(5).mkString("\n"))
143 |     //finalAll.collect().take(5).foreach(println)
144 |     println(s"Taking time::::::::${System.currentTimeMillis() - startTime1}")
145 | 
146 |     // traingle count
147 |     val tCount = graph.triangleCount().vertices
148 |     println(tCount.collect().mkString("\n"))
149 | 
150 |     //connected
151 |     val iterations = 1000
152 |     val connected = graph.connectedComponents().vertices
153 |     val connectedS = graph.stronglyConnectedComponents(iterations).vertices
154 | 
155 |     val connByName = vertices.join(connected).map {
156 |       case (id, ((name, age, location, specialization), conn)) => (conn, name, age)
157 |     }
158 |     val connByName1 = vertices.join(connectedS).map {
159 |       case (id, ((name, age, location, specialization), conn)) => (conn, name, age)
160 |     }
161 | 
162 |     connByName.collect().foreach {
163 |       case (conn, name, age) =>
164 |         println(f"Weak $conn $name $age")
165 |     }
166 | 
167 |     connByName1.collect().foreach {
168 |       case (conn, name, age) =>
169 |         println(f"Strong $conn $name $age")
170 |     }
171 |     verticesDf.registerTempTable("Person")
172 | 
173 |     val query = s"""select * from (select age,name from Person group by age,name having count(*) > 1) Person"""
174 |     val result1 = sqlContext.sql(query)
175 | 
176 |     println("Filter data:" + result1.collect.mkString(","))
177 | 
178 | 
179 |   }
180 | }
181 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick_Implement/GraphConnection.scala:
--------------------------------------------------------------------------------
1 | package com.tutorial.graphX.Rklick_Implement
2 | 
3 | /**
4 |   * Created by ved on 12/3/16.
5 |   */
6 |  case class GraphConnection(primaryIndex: String, vertexAttr: String, edgeAttr: String, relation: String)
7 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick_Implement/GraphXImplement.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.Rklick_Implement
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.SQLContext
 5 | 
 6 | /**
 7 |   * Created by ved on 12/3/16.
 8 |   */
 9 | object GraphXImplement {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 | 
14 |   val sqlContext = new SQLContext(sc)
15 | 
16 | }


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick_Implement/GraphXMain.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick_Implement
  2 | 
  3 | import com.tutorial.graphX.Rklick_Implement
  4 | import org.apache.spark.graphx.lib.{ConnectedComponents, PageRank, StronglyConnectedComponents}
  5 | import org.apache.spark.graphx.{Edge, Graph, PartitionStrategy}
  6 | import org.apache.spark.rdd.RDD
  7 | import org.apache.spark.sql.{DataFrame, Row}
  8 | import org.slf4j.{Logger, LoggerFactory}
  9 | 
 10 | import scala.util.control.NonFatal
 11 | 
 12 | 
 13 | /**
 14 |   * Created by ved on 12/3/16.
 15 |   */
 16 | trait GraphXMain {
 17 |   type GraphRDD = RDD[Map[String, Any]]
 18 |   val logger: Logger = LoggerFactory.getLogger(classOf[GraphXMain])
 19 | 
 20 | 
 21 |   /**
 22 |     * Create edges from dataframe
 23 |     *
 24 |     * @param df
 25 |     * @param graph
 26 |     * @return
 27 |     */
 28 |   def getEdges(df: DataFrame, graph: GraphConnection): RDD[Edge[String]] = {
 29 |     df.map {
 30 |       case row: Row =>
 31 |         val primaryValue = row.getAs[Any](graph.primaryIndex).toString.toLong
 32 |         val relIndex = row.getAs[Any](graph.relation).toString.toLong
 33 |         val edgeAttr = row.getAs[Any](graph.edgeAttr).toString
 34 |         Edge(primaryValue, relIndex, edgeAttr)
 35 |     }
 36 |   }
 37 | 
 38 |   /**
 39 |     * Create vertices from dataframe
 40 |     *
 41 |     * @param df    The DataFrame
 42 |     * @param graph The domain.GraphComponent
 43 |     * @return
 44 |     */
 45 |   def getVertices(df: DataFrame, graph: GraphConnection): RDD[(Long, String)] = df.map {
 46 |     case row =>
 47 |       val primaryValue = row.getAs[Any](graph.primaryIndex).toString.toLong
 48 |       val vertexAttr = row.getAs[Any](graph.vertexAttr).toString
 49 |       (primaryValue, vertexAttr)
 50 |   }
 51 | 
 52 |   /**
 53 |     * Process graph
 54 |     *
 55 |     * @param df             The DataFrame
 56 |     * @param graphComponent The domain.GraphComponent
 57 |     */
 58 |   def processGraph(df: DataFrame, graphComponent: GraphConnection): Either[String, GraphRDD] = {
 59 |     try {
 60 |       val edges = getEdges(df, graphComponent)
 61 |       val vertices = getVertices(df, graphComponent)
 62 |       val graph = Graph(vertices, edges).cache()
 63 |       for {
 64 |         result <- applyGraphAlgorithm(graph, graphComponent.primaryIndex).right
 65 |       } yield result
 66 |     } catch {
 67 |       case NonFatal(t: Throwable) =>
 68 |         Left(t.getMessage)
 69 |     }
 70 |   }
 71 | 
 72 |   /**
 73 |     * Apply graph algorithm & find the output
 74 |     *
 75 |     * @param graph
 76 |     * @param primaryIndex
 77 |     * @return
 78 |     */
 79 |   def applyGraphAlgorithm(graph: Graph[String, String], primaryIndex: String): Either[String, GraphRDD] = {
 80 |     try {
 81 |       //Page rank Algorithm
 82 |       val pageRankGraph = PageRank.run(graph, 1, 0.001)
 83 |       // Page Rank join with graph object
 84 |       val graphWithPageRank = graph.outerJoinVertices(pageRankGraph.vertices) {
 85 |         case (id, attr, Some(pr)) => (pr, attr)
 86 |         case (id, attr, None) => (0.0, attr)
 87 |       }
 88 | 
 89 |       //Triangle Count Algorithm
 90 |       val triangleComponent = graph.partitionBy(PartitionStrategy.RandomVertexCut)
 91 |         .triangleCount().vertices
 92 | 
 93 |       //Triangle Count join with page rank graph object
 94 |       val triByGraph = graphWithPageRank.outerJoinVertices(triangleComponent) {
 95 |         case (id, (rank, attr), Some(tri)) => (rank, tri, attr)
 96 |         case (id, (rank, attr), None) => (rank, 0, attr)
 97 |       }
 98 | 
 99 |       //Connected Component Algorithm
100 |       val cComponent = ConnectedComponents.run(graph).vertices
101 |       //Connected Component join with triangle component graph object
102 |       val ccByGraph = triByGraph.outerJoinVertices(cComponent) {
103 |         case (id, (rank, tri, attr), Some(cc)) => (rank, tri, cc, attr)
104 |         case (id, (rank, tri, attr), None) => (rank, tri, -1L, attr)
105 |       }
106 | 
107 |       //Strongly Connected Component Algorithm
108 |       val stronglyConnected = StronglyConnectedComponents.run(graph, 3).vertices
109 |       //Strongly Connected Component join with connected component object
110 |       val stByGraph = ccByGraph.outerJoinVertices(stronglyConnected) {
111 |         case (id, (rank, tri, cc, attr), Some(st)) => (rank, tri, cc, st, attr)
112 |         case (id, (rank, tri, cc, attr), None) => (rank, tri, cc, id.toLong, attr)
113 |       }
114 | 
115 |       //Result output
116 |       val result = stByGraph.triplets.map {
117 |         case line =>
118 |           val relation = Map("from" -> line.srcId.toString, "to" -> line.dstId.toString, "type" -> line.attr)
119 |           Map(primaryIndex -> line.srcId.toLong, "pageRank" -> line.srcAttr._1.toDouble,
120 |             "triangleComponent" -> line.srcAttr._2.toInt, "connectedComponent" -> line.srcAttr._3.toLong,
121 |             "stConnectedComponent" -> line.srcAttr._4.toLong, "link_relation" -> relation)
122 |       }
123 | 
124 |       //Find top 10 users
125 |       println("top 10 user")
126 |       println(graphWithPageRank.vertices.top(10)(Ordering.by(_._2._1)).mkString("\n"))
127 |       Right(result)
128 |     } catch {
129 |       case NonFatal(t: Throwable) => Left(t.getMessage)
130 |     }
131 |   }
132 | }
133 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick_Implement/GraphXMainImplementation.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.graphX.Rklick_Implement
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.SQLContext
 5 | 
 6 | /**
 7 |   * Created by ved on 12/3/16.
 8 |   */
 9 | object GraphXMainImplementation extends GraphXMain {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   val sqlContext = new SQLContext(sc)
14 | 
15 | 
16 |   /**
17 |     *
18 |     * @return
19 |     */
20 |   def main(args: Array[String]) {
21 |     val SCHEMA_OPTIONS = Map("header" -> "true", "inferSchema" -> "true")
22 |     val path = "src/main/resources/graph_data.csv"
23 |     val df = sqlContext.read.format("csv").options(SCHEMA_OPTIONS).load(path)
24 |     val graphComponent = GraphConnection("ID", "User", "Relationship", "RelationId")
25 |     processGraph(df, graphComponent) match {
26 |       case Right(data) => println(s"GraphX Implementation Done.")
27 |         sqlContext.sparkContext.stop()
28 |       case Left(error) => println(s"See Error In graphX:::${error}")
29 |         sqlContext.sparkContext.stop()
30 |     }
31 |   }
32 | }


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/Rklick_Test/Test1.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX.Rklick_Test
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx.{Edge, Graph}
  5 | import org.apache.spark.rdd.RDD
  6 | import org.apache.spark.sql.{DataFrame, SQLContext}
  7 | 
  8 | /**
  9 |   * Created by ved on 10/3/16.
 10 |   */
 11 | object Test1 {
 12 | 
 13 |   def main(args: Array[String]) {
 14 | 
 15 |     val sc = SparkCommon.sparkContext
 16 |     val sqlContext = new SQLContext(sc)
 17 | 
 18 |     val dataSet = sqlContext.read
 19 |       .format("com.databricks.spark.csv")
 20 |       .option("header", "true")
 21 |       .option("inferSchema", "true")
 22 |       .load("src/main/resources/Cricket_Node_Rklick_Test.csv")
 23 | 
 24 |     def getVertices(df: DataFrame): RDD[(Long, (Long, Long, String, String, Long, String, String, String, String, String, String))] = {
 25 |       df.map {
 26 |         case row => (row.getAs[Any]("id").toString.toLong,
 27 |           (row.getAs[Any]("fid").toString.toLong, row.getAs[Any]("lid").toString.toLong, row.getAs[Any]("follow").toString, row.getAs[Any]("name").toString,
 28 |             row.getAs[Any]("age").toString.toLong, row.getAs[Any]("location").toString, row.getAs[Any]("specialization").toString, row.getAs[Any]("marital").toString
 29 |             , row.getAs[Any]("twitter").toString, row.getAs[Any]("fb").toString, row.getAs[Any]("movie").toString))
 30 |       }
 31 |     }
 32 | 
 33 |     def getEdges(df1: DataFrame): RDD[Edge[String]] = {
 34 |       df1.map {
 35 |         case row => Edge(row.getAs[Any]("id").toString.toLong,
 36 |           row.getAs[Any]("id").toString.toLong, row.getAs[Any]("specialization").toString)
 37 |       }
 38 |     }
 39 | 
 40 |     val vertices = getVertices(dataSet)
 41 |     val edges = getEdges(dataSet)
 42 | 
 43 |     /**
 44 |       * Create Graph.
 45 |       */
 46 |     val graph = Graph(vertices, edges)
 47 |     graph.vertices.collect.foreach(println)
 48 |     graph.edges.collect().foreach(println)
 49 | 
 50 |     /**
 51 |       * 1.Analysis data on basis of specialization
 52 |       *
 53 |       */
 54 |     printSpecialization("all rounder", "batting", "bowler")
 55 |     def printSpecialization(specs: String*): Unit = {
 56 |       specs.foreach { spec =>
 57 |         graph.vertices.filter { case (id, (_, _, _, _, _, _, specialization, _, _, _, _)) =>
 58 |           specialization.equals(spec)
 59 |         }.collect.foreach { case (id, (_, _, _, name, _, _, _, _, _, _, _)) =>
 60 |           println(f"$spec:$id $name")
 61 |         }
 62 |       }
 63 |     }
 64 | 
 65 |     /**
 66 |       * 2. Analysis based on location
 67 |       *
 68 |       */
 69 |     printLocations("delhi", "mumbai", "chennai", "up", "ranchi")
 70 | 
 71 |     def printLocations(locations: String*): Unit = {
 72 |       locations.foreach { name =>
 73 |         graph.vertices.filter { case (id, (_, _, _, _, _, location, _, _, _, _, _)) =>
 74 |           location.equals(name)
 75 |         }.collect.foreach(println)
 76 |       }
 77 |     }
 78 | 
 79 |     /**
 80 |       * 3.Analysis data on basis of follower
 81 |       *
 82 |       */
 83 |     printFollowers("friend", "best friend", "good friend")
 84 | 
 85 |     def printFollowers(followers: String*): Unit = {
 86 |       followers.foreach { value =>
 87 |         graph.vertices.filter { case (id, (_, _, follow, _, _, _, _, _, _, _, _)) =>
 88 |           follow.equals(value)
 89 |         }.collect.foreach(println)
 90 |       }
 91 |     }
 92 | 
 93 |     graph.edges.filter { case Edge(from, to, property) =>
 94 |       property == "friend" | property == "good friend" | property == "best friend"
 95 |     }.collect.foreach(println)
 96 | 
 97 |     graph.edges.filter { case Edge(from, to, property) =>
 98 |       property == "bowler" | property == "batting" | property == "all rounder"
 99 |     }.collect.foreach(println)
100 | 
101 |     graph.edges.filter { case Edge(from, to, property) =>
102 |       property == "delhi" | property == "up" | property == "chennai" | property == "mumbai" | property == "ranchi"
103 |     }.collect.foreach(println)
104 | 
105 | 
106 |     /**
107 |       * 4.Analysis based Age
108 |       *
109 |       */
110 |     graph.vertices.filter { case (id, (_, _, _, name, age, _, _, _, _, _, _)) =>
111 |       age.toLong > 26.toShort
112 |     }.collect.foreach { case (id, (_, _, _, name, age, _, _, _, _, _, _)) =>
113 |       println(f"Younger:$id $name $age")
114 |     }
115 | 
116 |     graph.vertices.filter { case (id, (_, _, _, name, age, _, _, _, _, _, _)) =>
117 |       age.toLong < 26.toShort
118 |     }.collect.foreach { case (id, (_, _, _, name, age, _, _, _, _, _, _)) =>
119 |       println(f"young :$id $name $age")
120 |     }
121 | 
122 |     /**
123 |       * 5.Analysis based on marital
124 |       *
125 |       */
126 |     List("Married", "Unmarried").foreach { status =>
127 |       graph.vertices.filter { case (id, (_, _, _, _, _, _, _, marital, _, _, _)) =>
128 |         marital.equals(status)
129 |       }.collect.foreach { case (id, (_, _, _, name, age, _, _, _, _, _, _)) =>
130 |           println(f"$status :$id $name $age")
131 |       }
132 |     }
133 | 
134 |     /**
135 |       * 5.Analysis based on twitter
136 |       *
137 |       */
138 |     val t1 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
139 |       twitter.equals("y")
140 |     }
141 | 
142 |     t1.collect.foreach {
143 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
144 |         println(f"y :$id $name $age")
145 |     }
146 | 
147 |     val t2 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
148 |       twitter.equals("n")
149 |     }
150 | 
151 |     t2.collect.foreach {
152 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
153 |         println(f"n :$id $name $age")
154 |     }
155 | 
156 |     /**
157 |       * 5.Analysis based on fb
158 |       *
159 |       */
160 |     val f1 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
161 |       fb.equals("y")
162 |     }
163 | 
164 |     f1.collect.foreach {
165 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
166 |         println(f"y :$id $name $age")
167 |     }
168 | 
169 |     val f2 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
170 |       fb.equals("n")
171 |     }
172 | 
173 |     f2.collect.foreach {
174 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
175 |         println(f"n :$id $name $age")
176 |     }
177 | 
178 |     /**
179 |       * 5.Analysis based on movie
180 |       *
181 |       */
182 |     val movie1 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
183 |       movie.equals("Hollywood")
184 |     }
185 | 
186 |     movie1.collect.foreach {
187 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
188 |         println(f"Hollywood :$id $name $age")
189 |     }
190 | 
191 |     val movie2 = graph.vertices.filter { case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
192 |       movie.equals("Bollywood")
193 |     }
194 | 
195 |     movie2.collect.foreach {
196 |       case (id, (fid, lid, follow, name, age, location, specialization, marital, twitter, fb, movie)) =>
197 |         println(f"Bollywood :$id $name $age")
198 |     }
199 | 
200 |   }
201 | 
202 | }
203 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/graphX/examlples.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.graphX
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.graphx.{Graph, Edge, VertexId}
  5 | import org.apache.spark.rdd.RDD
  6 | import org.apache.spark.sql.SQLContext
  7 | import org.apache.spark.sql.{SQLContext, Row}
  8 | 
  9 | /**
 10 |   * Created by ved on 5/3/16.
 11 |   */
 12 | object examlples {
 13 | 
 14 |   val sc = SparkCommon.sparkContext
 15 |   val sqlContext = new SQLContext(sc)
 16 | 
 17 |   import sqlContext._
 18 | 
 19 |   def main(args: Array[String]) {
 20 | 
 21 | 
 22 |     /**
 23 |       * Next you'll need to get your data registered as a table.
 24 |       * Note that in order to infer the schema (column names and types) it must be a case class.
 25 |       * A normal class won't work, unless you manually implement much of what a case class would do,
 26 |       * e.g. extend Product and Serializable.
 27 | 
 28 |       * Any data source can be used, as long as you can transform it to an rdd of case classes.  For now, we'll just use some dummy data:
 29 |       */
 30 | 
 31 | 
 32 |     val people = sc.parallelize(Array(
 33 |       Person("adam", 19, Seq("123-5767", "234-1544")),
 34 |       Person("bob", 23, Seq()),
 35 |       Person("cathy", 14, Seq("454-1222", "433-1212", "345-2343")),
 36 |       Person("dave", 44, Seq("454-1222"))
 37 |     ))
 38 | 
 39 |     //people.registerTempTable("Persons")
 40 | 
 41 | 
 42 |     val likes = sc.parallelize(Array(
 43 |       Likes("adam", "cathy"),
 44 |       Likes("bob", "dave"),
 45 |       Likes("bob", "adam"),
 46 |       Likes("cathy", "adam"),
 47 |       Likes("cathy", "bob")
 48 |     ))
 49 | 
 50 | 
 51 | 
 52 |     def id(s: String) = s.head.toLong
 53 | 
 54 |     val vertices: RDD[(VertexId, String)] = sql("select name from people").map {
 55 |       case Row(name: String) =>
 56 |         id(name) -> name
 57 |     }
 58 | 
 59 |     /*
 60 |     helper to make vertex ids more readable
 61 |     */
 62 | 
 63 |     val nameOf = vertices.collect.toMap
 64 | 
 65 |     /*
 66 |     GraphX edges are directional, so if you want the equivalent of an undirected graph, you need to make edges in both directions
 67 |     Edges are assumed to have a label of some kind (e.g. a weight or a name), if you don't need it, just use 0
 68 |     */
 69 | 
 70 |     val edges: RDD[Edge[Int]] = sql("select name, likes from likes").map {
 71 |       case Row(name: String, likes: String) =>
 72 |         Edge(id(name), id(likes), 0)
 73 |     }
 74 | 
 75 |     /*
 76 |     Passing a default avoids errors when an edge refers to an unknown vertex
 77 |     */
 78 | 
 79 |     val default = "UNKNOWN"
 80 | 
 81 |     val graph = Graph(vertices, edges, default)
 82 | 
 83 |     /*
 84 |     Now that you have a graph object, you can perform operations on it:
 85 |     */
 86 | 
 87 |     val degrees = graph.degrees.collect.sortWith((a, b) => (a._2 > b._2))
 88 | 
 89 |     println(
 90 |       s"""
 91 | the vertex with the highest degree is "${nameOf(degrees(0)._1)}",
 92 | having ${degrees(0)._2} relationships
 93 | """)
 94 | 
 95 |     val scc = graph.stronglyConnectedComponents(numIter = 5)
 96 | 
 97 |     scc.vertices.map {
 98 |       case (member, leader) => s"${nameOf(member)} is in ${nameOf(leader)}'s clique"
 99 |     }.collect.foreach(println)
100 | 
101 | 
102 |   }
103 | 
104 | }
105 | 
106 | case class Person(name: String, age: Int, phones: Seq[String])
107 | 
108 | case class Likes(name: String, likes: String)
109 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparkcore/CreateRDD.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparkcore
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | 
 5 | /**
 6 |     * An example to create RDD from different data sources
 7 |     * Created by ved on 7/1/16.
 8 |     */
 9 | object CreateRDD {
10 | 
11 |   /**
12 |     * Create a Scala Spark Context.
13 |     */
14 |   val sc = SparkCommon.sparkContext
15 |   def main(args: Array[String]) {
16 | 
17 |     /**
18 |       * Create RDDs using parallelize() method of SparkContext
19 |       */
20 |     val lines = sc.parallelize(List("pandas", "i like pandas"))
21 |     lines.collect().map(println)
22 | 
23 |     /**
24 |       * Create RDDs is to load data from external storage
25 |       */
26 |     val rddDataset = sc.textFile("src/main/resources/test_file.txt")
27 |     rddDataset.collect().map(println)
28 |     //sc.stop()
29 |   }
30 | 
31 | }
32 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparkcore/RDDAction.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.sparkcore
  2 | 
  3 | import java.util.UUID
  4 | 
  5 | import com.tutorial.utils.SparkCommon
  6 | 
  7 | /**
  8 |   *  Actions are operations that return a result to the driver program or write it to storage
  9 |   * Created by ved on 8/1/16.
 10 |   */
 11 | object RDDAction {
 12 | 
 13 |   val sc = SparkCommon.sparkContext
 14 | 
 15 |   def main(args: Array[String]) {
 16 |     /**
 17 |       * The most common action on basic RDDs
 18 |       * reduce(func) Combine the elements of the RDD together in parallel.
 19 |       */
 20 |     val input = sc.parallelize(List(3, 2, 4, 6))
 21 |     val inputs = sc.parallelize(List(2, 4, 2, 3))
 22 | 
 23 |     val rUnion = input.union(inputs)
 24 |     val resultReduce = rUnion.reduce((x, y) => x + y)
 25 |     println("reduce:" + resultReduce + " ")
 26 | 
 27 |     /**
 28 |       * The most common action on basic RDDs
 29 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 30 |       * collect() Return all elements from the RDD.
 31 |       */
 32 |     val inputElement = sc.parallelize(List(2, 3, 4, 4))
 33 |     println("collect" + inputElement.collect().mkString(","))
 34 | 
 35 |     /**
 36 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 37 |       * count() returns a count of the elements the RDD.
 38 |       */
 39 |     val inputCount = sc.parallelize(List(2, 3, 4, 4))
 40 |     println(" count:" + inputCount.count())
 41 | 
 42 |     /**
 43 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 44 |       * countByValue()  returns Number of times each element occurs in the RDD  .
 45 |       */
 46 |     val inputCountByValue = sc.parallelize(List(2, 3, 4, 4))
 47 |     println("countByValue :" + inputCountByValue.countByValue().mkString(","))
 48 | 
 49 |     /**
 50 |       * first()
 51 |       * Returns the first element of the dataset (similar to take (1)).
 52 |       */
 53 |     val inputFirst = sc.parallelize(List(2, 3, 4, 4))
 54 |     println("fist:" + inputFirst.first())
 55 | 
 56 |     /**
 57 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 58 |       * take(num) Return num elements from the RDD.
 59 |       */
 60 |     val inputTake = sc.parallelize(List(2, 3, 4, 4))
 61 |     println("take :" + inputTake.take(2).mkString(","))
 62 | 
 63 |     /**
 64 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 65 |       * top(num) Return the top num elements the RDD.
 66 |       */
 67 |     val inputTop = sc.parallelize(List(2, 3, 4, 4))
 68 |     println("Top:" + inputTop.top(2).mkString(","))
 69 | 
 70 |     /**
 71 |       * Basic actions on an RDD containing {2, 3, 4, 4}
 72 |       * takeOrdered(num)(ordering) Return num elements based on provided ordering
 73 |       */
 74 |     val inputOrder = sc.parallelize(List(2, 3, 4, 4))
 75 |     println("Take Order :" + inputOrder.takeOrdered(2).mkString(","))
 76 | 
 77 |     /**
 78 |       * foreach()
 79 |       * Runs a function func on each element of the dataset.
 80 |       */
 81 |     val inputForeach = sc.parallelize(List(2, 3, 4, 4))
 82 |     inputForeach.foreach((x => println(x + 1)))
 83 | 
 84 |     /**
 85 |       * example of map() that squares all of the numbers in an RDD
 86 |       */
 87 |     val inputNumbers = sc.parallelize(List(2, 3, 4, 4))
 88 |     val resultSquare = inputNumbers.map(x => x * x)
 89 |     println("Square:" + resultSquare.collect().mkString(","))
 90 | 
 91 |     /**
 92 |       * Actions are available on pair RDDs
 93 |       * Actions on pair RDDs (example ({(1, 2), (2, 3), (5, 4)}))
 94 |       * countByKey() Count the number of elements for each key.
 95 |       */
 96 |     val inputAction = sc.parallelize(List((1, 2), (2, 3), (5, 4)))
 97 |     println("countByKey :" + inputAction.countByKey().mkString(","))
 98 | 
 99 |     /**
100 |       * Actions on pair RDDs (example ({(1, 2), (2, 3), (5, 4)}))
101 |       * collectAsMap() Collect the result as a map to provide easy lookup.
102 |       */
103 |     val inputCollectAsMap = sc.parallelize(List((1, 2), (2, 3), (5, 4)))
104 |     println("Collect map:" + inputCollectAsMap.collectAsMap().mkString(","))
105 | 
106 |     /**
107 |       * Actions on pair RDDs (example ({(1, 2), (2, 3), (3, 4)}))
108 |       * lookup(key) Return all values associated with the provided key.
109 |       */
110 |     val inputLookUp = sc.parallelize(List((1, 2), (2, 3), (3, 4)))
111 |     println("lookup key:" + inputLookUp.lookup(3).mkString(","))
112 | 
113 |     /**
114 |       * saveAsTextFile()
115 |       * Write the elements of the dataset as a text file (or set of text files) in a given directory in the local filesystem.
116 |       */
117 |     val inputFile = sc.textFile("src/main/resources/test_file.txt") // Load our input data.
118 |     val count = inputFile.flatMap(line => line.split(" ")) // Split it up into words.
119 |         .map(word => (word, 1)).reduceByKey(_ + _) // Transform into pairs and count.
120 | 
121 |     //Save the word count back out to a text file, causing evaluation.
122 |     count.saveAsTextFile(s"src/main/resources/${UUID.randomUUID()}")
123 |     println("OK")
124 | 
125 |     /**
126 |       * takeSample()
127 |       * Similar to take, in return type of array with size of n.
128 |       * Includes boolean option of with or without replacement and random generator seed.
129 |       */
130 |     val takeSampleInput = sc.parallelize(List(2, 3, 4, 4))
131 |     val takeSampleResult = takeSampleInput.takeSample(true, 1)
132 |     println("takeSample :" + takeSampleResult.mkString(","))
133 | 
134 |   }
135 | 
136 | 
137 | }
138 | 
139 | 
140 | 
141 | 
142 | 
143 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparkcore/RDDOperations.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparkcore
 2 | 
 3 | import java.util.UUID
 4 | 
 5 | import com.tutorial.utils.SparkCommon
 6 | 
 7 | /**
 8 |   * An example  of two types of operations: transformations and actions.
 9 |   * Created by ved on 7/1/16.
10 |   */
11 | object RDDOperations {
12 | 
13 |   val sc = SparkCommon.sparkContext
14 | 
15 |   def main(args: Array[String]) {
16 | 
17 |     /**
18 |       * RDD operation for  Word count .
19 |       */
20 |     val inputFile = sc.textFile("src/main/resources/test_file.txt") // Load our input data.
21 |     val count = inputFile.flatMap(line => line.split(" ")) // Split it up into words.
22 |         .map(word => (word, 1)).reduceByKey(_ + _) // Transform into pairs and count.
23 | 
24 |     //Save the word count back out to a text file, causing evaluation.
25 |     count.saveAsTextFile(s"src/main/resources/${UUID.randomUUID()}")
26 |     println("OK")
27 | 
28 |     /**
29 |       * Operations on RDDs.
30 |       * count() returns a count of the elements the RDD.
31 |       */
32 |     val inputRDD = sc.textFile("src/main/resources/test_file.txt")
33 |     val dataRDD = inputRDD.filter(line => line.contains("data"))
34 |     println(dataRDD.count())
35 | 
36 |     /**
37 |       * The most common action on basic RDDs
38 |       * reduce(func) Combine the elements of the RDD together in parallel (e.g., sum ).
39 |       */
40 |     val x = sc.parallelize(List(1, 2, 4, 4))
41 |     val y = sc.parallelize(List(6, 5, 1, 4))
42 | 
43 |     val rUnion = x.union(y)
44 |     val resultReduce = rUnion.reduce((x, y) => x + y)
45 |     println("reduce:" + resultReduce + " ")
46 | 
47 |     /**
48 |       * Basic actions on an RDD containing {1, 2, 3, 3}
49 |       * count() returns a count of the elements the RDD.
50 |       */
51 |     val inputCount = sc.parallelize(List(1, 2, 4, 4))
52 |     println(" count:" + inputCount.count())
53 | 
54 |     /**
55 |       * example of map() that squares all of the numbers in an RDD
56 |       */
57 |     val inputNumbers = sc.parallelize(List(1, 2, 3, 4))
58 |     val resultSquare = inputNumbers.map(x => x * x)
59 |     println("Square:" + resultSquare.collect().mkString(","))
60 | 
61 |   }
62 | }
63 | 
64 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparkcore/RDDTransformations.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.sparkcore
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | 
  5 | /**
  6 |   * Transformations are operations on RDDs that return a new RDD
  7 |   * Created by ved on 8/1/16.
  8 |   */
  9 | object RDDTransformations {
 10 | 
 11 |   val sc = SparkCommon.sparkContext
 12 | 
 13 |   def main(args: Array[String]) {
 14 | 
 15 |     val inputText = sc.textFile("src/main/resources/test_file.txt")
 16 |     val inputLine = sc.parallelize(List("hello VED", "hi"))
 17 |     val input = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7, 8))
 18 | 
 19 |     /**
 20 |       * Element-wise transformations
 21 |       * map() transformation  function being the new value of each element in the resulting RDD
 22 |       */
 23 |     //val input1 = sc.parallelize(List(1, 2, 3, 4,5,6,7,8,9))
 24 |     val result1 = input.map(x => x + 1)
 25 |     println("Mapping:" + result1.collect().mkString(","))
 26 | 
 27 |     /**
 28 |       * Basic RDD transformations in Spark
 29 |       * flatMap() to produce multiple output elements for each input element
 30 |       */
 31 |     //val flatMapLines = sc.parallelize(List("hello VED", "hi"))
 32 |     val flatMapResult = inputLine.flatMap(line => line.split(" "))
 33 |     println("flatMap:" + flatMapResult.first())
 34 | 
 35 |     /**
 36 |       * Example of filter() Transformations  operations from external storage.
 37 |       */
 38 |     //val inputText = sc.textFile("src/main/resources/test_file.txt")
 39 |     val resultFilterText = inputText.filter(line => line.contains("data"))
 40 |     println("filterData" + resultFilterText.count())
 41 | 
 42 |     /**
 43 |       * Element-wise transformations
 44 |       * Example of filter()  Transformations  operations from existing collection in your program and
 45 |       * pass it to SparkContext’s parallelize() method.
 46 |       */
 47 | 
 48 |     val result = input.filter(x => x != 1)
 49 |     println("Filter of element:" + result.collect().mkString(","))
 50 | 
 51 |     /**
 52 |       * mapPartitions()
 53 |       * Similar to map, but runs separately on each partition (block) of the RDD,
 54 |       * so func must be of type Iterator<T> ⇒ Iterator<U> when running on an RDD of type T.
 55 |       */
 56 | 
 57 |     val inputData = sc.parallelize(1 to 9, 3)
 58 |     val inputData1 = sc.parallelize(1 to 9)
 59 |     val mapPartitionResult = inputData.mapPartitions(x => List(x.next).iterator)
 60 |     println("mapPartition is :" + mapPartitionResult.collect().mkString(","))
 61 | 
 62 |     /**
 63 |       * mapPartitionsWithIndex()
 64 |       * Similar to mapPartitions, but also provides a function
 65 |       * with an Int value to indicate the index position of the partition.
 66 |       */
 67 |     //val inputData =sc.parallelize(1 to 9 ,3)
 68 |     //val inputData1 =sc.parallelize(1 to 9 )
 69 |     val mapPartitionsWithIndexRseult = inputData.mapPartitionsWithIndex((index: Int, it: Iterator[Int])
 70 |     => it.toList.map(x => index + ", " + x).iterator)
 71 |     println("mapPartitionsWithIndex :" + mapPartitionsWithIndexRseult.collect().mkString(","))
 72 | 
 73 | 
 74 |     /**
 75 |       * sample()
 76 |       * Sample a fraction of the data, with or without replacement, using a given random number generator seed.
 77 |       */
 78 | 
 79 |     val sampleResult = inputData1.sample(true, .1)
 80 |     println("sample:" + sampleResult.count())
 81 | 
 82 |     /**
 83 |       * Two-RDD transformations on RDDs containing {1, 2, 3} and {3, 5,7 }
 84 |       * union() transformation  function Produce an RDD containing elements from both RDDs.
 85 |       */
 86 |     val inputRdd1 = sc.parallelize(List(1, 2, 3))
 87 |     val inputRdd2 = sc.parallelize(List(3, 5, 7))
 88 |     val resultInputUnion = inputRdd1.union(inputRdd2)
 89 |     println("Union:" + resultInputUnion.collect().mkString(","))
 90 | 
 91 |     /**
 92 |       * Two-RDD transformations on RDDs containing {1, 2, 3} and {3, 5, 7}
 93 |       * intersection() transformation  function containing only elements found in both RDDs.
 94 |       */
 95 |     //val input113 = sc.parallelize(List(1, 2, 3))
 96 |     ///val input223 = sc.parallelize(List(3, 5, 7))
 97 |     val resultIntersection = inputRdd1.intersection(inputRdd2)
 98 |     println("Intersection:" + resultIntersection.collect().mkString(","))
 99 | 
100 |     /**
101 |       * RDDs support many of the operations of mathematical sets such as distinct,union,intersection and subtract.
102 |       * distinct() transformation  function Return an RDD consisting of Remove duplicates.
103 |       */
104 |     val distinctInput = sc.parallelize(List(1, 2, 3, 4, 2, 1, 3, 4, 2, 5, 6))
105 |     val distinctResult = distinctInput.distinct()
106 |     println("distinct:" + distinctResult.collect().mkString(","))
107 | 
108 |     /**
109 |       * Two-RDD transformations on RDDs containing {1, 2, 3} and {3, 5, 6}
110 |       * subtract() transformation  function Remove the contents of one RDD(e.g.,remove training data).
111 |       */
112 |     val inputSubtract1 = sc.parallelize(List(1, 2, 3))
113 |     val inputSubtract2 = sc.parallelize(List(3, 5, 6))
114 |     val resultSub = inputSubtract1.subtract(inputSubtract2)
115 |     println("subtract:" + resultSub.collect().mkString(","))
116 | 
117 |     /**
118 |       * Two-RDD transformations on RDDs containing {1, 2, 3} and {3, 5, 7}
119 |       * cartesian() Cartesian product with the other RDD.
120 |       */
121 |     val inputCartesian1 = sc.parallelize(List(1, 2, 3))
122 |     val inputCartesian2 = sc.parallelize(List(3, 5, 7))
123 |     val resultCartesian = inputCartesian1.cartesian(inputCartesian2)
124 |     println("cartesian:" + resultCartesian.collect().mkString(","))
125 | 
126 |     /**
127 |       * Transformations on Pair RDDs.
128 |       * Transformations on one pair RDD (example: {(1, 2), (3, 4), (4, 6)})
129 |       * reduceByKey(func) Combine values with the same key.
130 |       */
131 |     val inputTupplesIs = sc.parallelize(List((1, 2), (3, 4), (4, 6)))
132 |     val resultReduceByKey = inputTupplesIs.reduceByKey((x, y) => x + y)
133 |     println("Combine values with the same Key:" + resultReduceByKey.collect().mkString(","))
134 | 
135 |     /**
136 |       * Transformations on one pair RDD (example: {(1, 2), (3, 5), (3, 7)})
137 |       * groupByKey() Group values with the same key.
138 |       */
139 |     val inputTupplesList = sc.parallelize(List((1, 2), (3, 5), (3, 7)))
140 |     val resultGroupByKey = inputTupplesList.groupByKey()
141 |     println("Group values  with the same Key:" +resultGroupByKey.collect.mkString(","))
142 | 
143 |     //println("Group values with the same key:" + resultGroupByKey.collect().mkString(","))
144 | 
145 |     /**
146 |       * Transformations on two pair RDDs (rdd = {(1, 2), (3, 5), (3, 7)} other = {(5, 9)})
147 |       * subtractByKey Remove elements with a key present in the other RDD.
148 |       */
149 | 
150 |     val inputTupples = sc.parallelize(List((1, 2), (3, 5), (3, 7)))
151 |     val inputTuppless = sc.parallelize(List((5, 9)))
152 | 
153 |     val resultSubtractByKey = inputTupples.subtractByKey(inputTuppless)
154 |     println("Remove elements by subtractByKey:." + resultSubtractByKey.collect().mkString(","))
155 | 
156 | 
157 |     /**
158 |       * sortByKey()
159 |       * When called on a dataset of (K, V) pairs where K implements Ordered, returns a dataset of (K, V) pairs sorted by keys
160 |       * in ascending or descending order, as specified in the Boolean ascending argument.
161 |       */
162 |     val sortKey = inputTupples.sortByKey()
163 |     println("sortkey:" + sortKey.collect().mkString(","))
164 | 
165 |     /**
166 |       * Transformations on two pair RDDs (rdd = {(1, 2), (3, 4), (4, 7)} other = {(5, 9)})
167 |       * join Perform an inner join between two RDDs.
168 |       */
169 |     val inputTupple1 = sc.parallelize(List((1, 2), (3, 4), (4, 7)))
170 |     val inputTupple2 = sc.parallelize(List((5, 9)))
171 |     val resultJoin = inputTupple1.join(inputTupple2)
172 |     val resultRightOuterJoin = inputTupple1.rightOuterJoin(inputTupple2)
173 |     val resultLeftOuterJoin = inputTupple1.leftOuterJoin(inputTupple2)
174 | 
175 |     println("inner join between two RDDs." + resultJoin.collect().mkString(","))
176 |     println(" RightOuter  join between two RDDs." + resultRightOuterJoin.collect().mkString(","))
177 |     println(" leftOuter join between two RDDs." + resultLeftOuterJoin.collect().mkString(","))
178 | 
179 |     /**
180 |       * Transformations on two pair RDDs (rdd = {(1, 2), (3, 4), (4, 6)} other = {(4, 9)})
181 |       * cogroup Group data from both RDDs sharing the same key.
182 |       */
183 |     val inputElementTupple1 = sc.parallelize(List((1, 2), (3, 4), (4, 6)))
184 |     val inpuElementtTupple2 = sc.parallelize(List((4, 9)))
185 | 
186 |     val resultGroup = inputElementTupple1.cogroup(inputElementTupple1)
187 |     println("Group data from both RDDs sharing the same key :" + resultGroup.collect().mkString(","))
188 | 
189 | 
190 |   }
191 | 
192 | }
193 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/BasicQueryExample.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import org.apache.spark.sql
 4 | import org.apache.spark.sql.SQLContext
 5 | 
 6 | import com.tutorial.utils.SparkCommon
 7 | 
 8 | /**
 9 |   * Created by ved on 16/2/16.
10 |   */
11 | object BasicQueryExample {
12 | 
13 |   val sc = SparkCommon.sparkContext
14 |   val sqlContext = new org.apache.spark.sql.SQLContext(sc)
15 | 
16 |   def main(args: Array[String]) {
17 | 
18 |     //import sqlContext.implicits._
19 | 
20 |     val input = sqlContext.read.json("src/main/resources/cars1.json")
21 | 
22 |     input.registerTempTable("Cars1")
23 | 
24 | 
25 |     val result = sqlContext.sql("SELeCT * FROM Cars1")
26 | 
27 |     result.show()
28 | 
29 |     sqlContext.sql("SELECT COUNT(*) FROM Cars1").collect().foreach(println)
30 | 
31 |     sqlContext.sql("SELECT name, COUNT(*) AS cnt FROM Cars1 WHERE name <> '' GROUP BY name ORDER BY cnt DESC LIMIT 10")
32 |       .collect().foreach(println)
33 | 
34 | 
35 |   }
36 | 
37 | }
38 | 
39 | case class Cars1(name: String)
40 | 
41 | 
42 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/CreatingDataFarmes.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | 
 5 | /**
 6 |   * Created by ved on 29/1/16.
 7 |   */
 8 | object CreatingDataFarmes {
 9 | 
10 |   val sc = SparkCommon.sparkContext
11 | 
12 |   /**
13 |     * Create a Scala Spark SQL Context.
14 |     */
15 |   val sqlContext = new org.apache.spark.sql.SQLContext(sc)
16 | 
17 |   def main(args: Array[String]) {
18 |     /**
19 |       * Create the DataFrame
20 |       */
21 |     val df = sqlContext.read.json("src/main/resources/cars.json")
22 | 
23 |     /**
24 |       * Show the Data
25 |       */
26 |     df.show()
27 | 
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/CreatingDatasets.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | 
 5 | /**
 6 |   * Created by ved on 20/1/16.
 7 |   */
 8 | object CreatingDatasets {
 9 | 
10 |   val sc = SparkCommon.sparkContext
11 |   val sqlContext = new org.apache.spark.sql.SQLContext(sc)
12 | 
13 |   def main(args: Array[String]) {
14 | 
15 |     import sqlContext.implicits._
16 | 
17 |     val lines = sqlContext.read.text("src/main/resources/test_file.txt").as[String]
18 |     val words = lines
19 |       .flatMap(_.split(" "))
20 |       .filter(_ != "")
21 |       .groupBy(_.toLowerCase)
22 |       .count()
23 |       .show()
24 | 
25 |     /**
26 |       * Encoders are also created for case classes.
27 |       */
28 |       //case class Cars(name: String, kph: Long)
29 |     val ds = Seq(Cars("lamborghini", 32)).toDS()
30 |     ds.show()
31 | 
32 |     /**
33 |       *DataFrames can be converted to a Dataset by providing a class.
34 |       *  Mapping will be done by name.
35 |       */
36 | 
37 |     val people = sqlContext.read.json("src/main/resources/cars.json").as[Cars]
38 |     
39 |     people.show()
40 | 
41 | 
42 | 
43 |   }
44 | }
45 | 
46 | case class Cars(name: String, kph: Long)
47 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/DataFrameAPI.scala:
--------------------------------------------------------------------------------
  1 | package com.tutorial.sparksql
  2 | 
  3 | import com.tutorial.utils.SparkCommon
  4 | import org.apache.spark.sql.DataFrame
  5 | 
  6 | import scala.reflect.internal.util.TableDef.Column
  7 | 
  8 | /**
  9 |   * DataFarme API Example Using Different types of Functionality.
 10 | 
 11 |   * Created by ved on 9/2/16.
 12 |   */
 13 | 
 14 | /**
 15 |   *
 16 |   */
 17 | object DataFrameAPI {
 18 | 
 19 |   val sc = SparkCommon.sparkContext
 20 | 
 21 |   val sqlContext = SparkCommon.sparkSQLContext
 22 |   /**
 23 |     * Use the following command to create SQLContext.
 24 |     */
 25 |   val ssc = SparkCommon.sparkSQLContext
 26 | 
 27 |   val schemaOptions = Map("header" -> "true", "inferSchema" -> "true")
 28 | 
 29 |   import sqlContext.implicits._
 30 |   import org.apache.spark.sql.functions._
 31 | 
 32 |   def main(args: Array[String]) {
 33 | 
 34 |     val cars = "src/main/resources/cars.json"
 35 | 
 36 |     val employee1 = "src/main/resources/employee1.json"
 37 | 
 38 |     val carsPrice = "src/main/resources/cars_price.json"
 39 | 
 40 |     val carDataFrame: DataFrame = ssc.read.format("json").options(schemaOptions).load(cars)
 41 | 
 42 |     val empDataFrame2: DataFrame = ssc.read.format("json").options(schemaOptions).load(employee1)
 43 | 
 44 |     val carDataFrame1: DataFrame = ssc.read.format("json").options(schemaOptions).load(carsPrice)
 45 | 
 46 |     /**
 47 |       * Diiferent type of DataFrame operatios.
 48 |       * Action: Action are operations (such as take, count, first, and so on)
 49 |       * that return a value after running a computation on an DataFrame.
 50 |       * Some Action Operation with examples:
 51 |       */
 52 | 
 53 |     /**
 54 |       * show()
 55 |       * If you want to see top 20 rows of DataFrame in a tabular form then use the following command.
 56 |       */
 57 | 
 58 |     carDataFrame.show()
 59 | 
 60 | 
 61 |     /**
 62 |       * show(n)
 63 |       * If you want to see n rows of DataFrame in a tabular form then use the following command.
 64 |       */
 65 | 
 66 |     carDataFrame.show(2)
 67 | 
 68 | 
 69 |     /**
 70 |       * take()
 71 |       * take(n) Returns the first n rows in the DataFrame.
 72 |       */
 73 |     carDataFrame.take(2).foreach(println)
 74 | 
 75 | 
 76 |     /**
 77 |       * count()
 78 |       * Returns the number of rows.
 79 |       */
 80 | 
 81 |     carDataFrame.groupBy("speed").count().show()
 82 | 
 83 | 
 84 |     /**
 85 |       * head()
 86 |       * head () is used to returns first row.
 87 |       */
 88 | 
 89 |     val resultHead = carDataFrame.head()
 90 | 
 91 |     println(resultHead.mkString(","))
 92 | 
 93 |     /**
 94 |       * head(n)
 95 |       * head(n) returns first n rows.
 96 |       */
 97 | 
 98 |     val resultHeadNo = carDataFrame.head(3)
 99 | 
100 |     println(resultHeadNo.mkString(","))
101 | 
102 |     /**
103 |       * first()
104 |       * Returns the first row.
105 |       */
106 | 
107 |     val resultFirst = carDataFrame.first()
108 | 
109 |     println("fist:" + resultFirst.mkString(","))
110 | 
111 | 
112 |     /**
113 |       * collect()
114 |       * Returns an array that contains all of Rows in this DataFrame.
115 |       */
116 | 
117 |     val resultCollect = carDataFrame.collect()
118 | 
119 |     println(resultCollect.mkString(","))
120 | 
121 | 
122 |     /**
123 |       * Basic DataFrame functions:
124 |       *
125 |       */
126 |     /**
127 |       * printSchema()
128 |       * If you want to see the Structure (Schema) of the DataFrame, then use the following command.
129 |       */
130 | 
131 |     carDataFrame.printSchema()
132 | 
133 | 
134 |     /**
135 |       * toDF()
136 |       * toDF() Returns a new DataFrame with columns renamed.
137 |       * It can be quite convenient in conversion from a RDD of tuples into a DataFrame with meaningful names.
138 |       */
139 | 
140 |     val car = sc.textFile("src/main/resources/fruits.txt")
141 |       .map(_.split(","))
142 |       .map(f => Fruit(f(0).trim.toInt, f(1), f(2).trim.toInt))
143 |       .toDF().show()
144 | 
145 | 
146 |     /**
147 |       * dtypes()
148 |       * Returns all column names and their data types as an array.
149 |       */
150 | 
151 |     carDataFrame.dtypes.foreach(println)
152 | 
153 | 
154 |     /**
155 |       * columns ()
156 |       * Returns all column names as an array.
157 |       */
158 | 
159 |     carDataFrame.columns.foreach(println)
160 | 
161 |     /**
162 |       * cache()
163 |       * cache() explicitly to store the data into memory.
164 |       * Or data stored in a distributed way in the memory by default.
165 |       */
166 | 
167 |     val resultCache = carDataFrame.filter(carDataFrame("speed") > 300)
168 | 
169 |     resultCache.cache().show()
170 | 
171 | 
172 |     /**
173 |       * Data Frame operations:
174 |       */
175 |     /**
176 |       * orderBy()
177 |       * Returns a new DataFrame sorted by the specified column(s).
178 |       */
179 |     carDataFrame.orderBy(desc("speed")).show()
180 | 
181 | 
182 |     /**
183 |       * groupBy()
184 |       * counting the number of cars who are of the same speed.
185 |       */
186 | 
187 |     carDataFrame.groupBy("speed").count().show()
188 | 
189 | 
190 |     /**
191 |       * na()
192 |       * Returns a DataFrame  for working with missing data.
193 |       */
194 | 
195 | 
196 |     carDataFrame.na.drop().show()
197 | 
198 | 
199 |     /**
200 |       * sort()
201 |       * Returns a new DataFrame sorted by the given expressions.
202 |       */
203 |     carDataFrame.sort($"itemNo".desc).show()
204 | 
205 | 
206 |     /**
207 |       * as()
208 |       * Returns a new DataFrame with an alias set.
209 |       */
210 | 
211 |     carDataFrame.select(avg($"speed").as("avg_speed")).show()
212 | 
213 | 
214 |     /**
215 |       * alias()
216 |       * Returns a new DataFrame with an alias set. Same as `as`.
217 |       */
218 | 
219 |     carDataFrame.select(avg($"weight").alias("avg_weight")).show()
220 | 
221 | 
222 |     /**
223 |       * select()
224 |       * to fetch speed-column among all columns from the DataFrame.
225 |       */
226 | 
227 |     carDataFrame.select("speed").show()
228 | 
229 | 
230 |     /**
231 |       * filter()
232 |       * filter the cars whose speed is greater than 300 (speed > 300).
233 |       */
234 | 
235 |     carDataFrame.filter(carDataFrame("speed") > 300).show()
236 | 
237 | 
238 |     /**
239 |       * where()
240 |       * Filters speed using the given SQL expression.
241 |       */
242 |     carDataFrame.where($"speed" > 300).show()
243 | 
244 | 
245 |     /**
246 |       * agg()
247 |       * Aggregates on the entire DataFrame without groups.
248 |       * returns the average of the values in a group.
249 |       */
250 | 
251 |     carDataFrame.agg(max($"speed")).show()
252 | 
253 | 
254 |     /**
255 |       * limit()
256 |       * Returns a new DataFrame by taking the first n rows.
257 |       * The difference between this function and head is that head returns an array
258 |       * while limit returns a new DataFrame.
259 |       */
260 | 
261 |     carDataFrame1.limit(3).show()
262 | 
263 | 
264 |     /**
265 |       * unionAll()
266 |       * Returns a new DataFrame containing union of rows in this frame and another frame.
267 |       */
268 | 
269 |     carDataFrame.unionAll(empDataFrame2).show()
270 | 
271 | 
272 |     /**
273 |       * intersect()
274 |       * Returns a new DataFrame containing rows only in both this frame and another frame.
275 |       */
276 | 
277 |     carDataFrame1.intersect(carDataFrame).show()
278 | 
279 |     /**
280 |       * except()
281 |       * Returns a new DataFrame containing rows in this frame but not in another frame.
282 |       */
283 | 
284 |     carDataFrame.except(carDataFrame1).show()
285 | 
286 |     /**
287 |       * withColumn()
288 |       * Returns a new DataFrame by adding a column or replacing the existing column that has the same name.
289 |       */
290 | 
291 |     val coder: (Int => String) = (arg: Int) => {
292 |       if (arg < 300) "slow" else "high"
293 |     }
294 | 
295 |     val sqlfunc = udf(coder)
296 | 
297 |     carDataFrame.withColumn("First", sqlfunc(col("speed"))).show()
298 | 
299 | 
300 |     /**
301 |       * withColumnRenamed()
302 |       * Returns a new DataFrame with a column renamed.
303 |       *
304 |       */
305 | 
306 |     empDataFrame2.withColumnRenamed("id", "employeeId").show()
307 | 
308 | 
309 |     /**
310 |       * drop()
311 |       * Returns a new DataFrame with a column dropped.
312 |       */
313 | 
314 |     carDataFrame.drop("speed").show()
315 | 
316 | 
317 |     /**
318 |       * dropDuplicates()
319 |       * Returns a new DataFrame that contains only the unique rows from this DataFrame.
320 |       * This is an alias for distinct.
321 |       */
322 | 
323 |     carDataFrame.dropDuplicates().show()
324 | 
325 | 
326 |     /**
327 |       * describe()
328 |       * describe returns a DataFrame containing information such as number of non-null entries (count),
329 |       * mean, standard deviation, and minimum and maximum value for each numerical column.
330 |       */
331 |     carDataFrame.describe("speed").show()
332 | 
333 |   }
334 | 
335 | 
336 | }
337 | 
338 | case class Fruit(id: Int, name: String, quantity: Int)
339 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/DataFrameOperations.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.DataFrame
 5 | 
 6 | 
 7 | /**
 8 |   * Created by ved on 19/1/16.
 9 |   */
10 | object DataFrameOperations {
11 | 
12 |   val sc = SparkCommon.sparkContext
13 | 
14 |   /**
15 |     * Use the following command to create SQLContext.
16 |     */
17 |   val ssc = SparkCommon.sparkSQLContext
18 | 
19 |   val schemaOptions = Map("header" -> "true", "inferSchema" -> "true")
20 | 
21 |   def main(args: Array[String]) {
22 | 
23 |     /**
24 |       * Create the DataFrame
25 |       */
26 |     val cars = "src/main/resources/cars.json"
27 | 
28 |     /**
29 |       * read the JSON document
30 |       * Use the following command to read the JSON document named cars.json.
31 |       * The data is shown as a table with the fields − itemNo, name, speed and weight.
32 |       */
33 |     val carsDataFrame: DataFrame = ssc.read.format("json").options(schemaOptions).load(cars)
34 | 
35 |     /**
36 |       * Show the Data
37 |       * If you want to see the data in the DataFrame, then use the following command.
38 |       */
39 |     carsDataFrame.show()
40 | 
41 |     /**
42 |       * printSchema Method
43 |       * If you want to see the Structure (Schema) of the DataFrame, then use the following command
44 |       */
45 |     carsDataFrame.printSchema()
46 | 
47 |     /**
48 |       * Select Method
49 |       * Use the following command to fetch name-column among three columns from the DataFrame
50 |       */
51 |     carsDataFrame.select("name").show()
52 | 
53 |     /**
54 |       * Filter
55 |       * cars whose speed is greater than 300 (speed > 300).
56 |       */
57 |     carsDataFrame.filter(carsDataFrame("speed") > 300).show()
58 | 
59 |     /**
60 |       * groupBy Method
61 |       * counting the number of cars who are of the same speed.
62 |       */
63 |     carsDataFrame.groupBy("speed").count().show()
64 | 
65 | 
66 |   }
67 | 
68 | 
69 | }
70 | 
71 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/DataSources/CsvFile.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql.DataSources
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.SQLContext
 5 | 
 6 | /**
 7 |   * Created by ved on 29/1/16.
 8 |   */
 9 | object CsvFile {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   val sqlContext = SparkCommon.sparkSQLContext
14 | 
15 |   def main(args: Array[String]) {
16 | 
17 |     val sqlContext = new SQLContext(sc)
18 |     val df = sqlContext.read
19 |       .format("com.databricks.spark.csv")
20 |       .option("header", "true") // Use first line of all files as header
21 |       .option("inferSchema", "true") // Automatically infer data types
22 |       .load("src/main/resources/cars.csv")
23 |     df.show()
24 |     df.printSchema()
25 | 
26 |     val selectedData = df.select("year", "model")
27 |     selectedData.write
28 |       .format("com.databricks.spark.csv")
29 |       .option("header", "true")
30 |     selectedData.show()
31 | 
32 | 
33 |     //.save(s"src/main/resources/${UUID.randomUUID()}")
34 |     //println("OK")
35 | 
36 |   }
37 | 
38 | }
39 | 
40 | 
41 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/DataSources/JSONFile.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql.DataSources
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.DataFrame
 5 | 
 6 | /**
 7 |   * Created by ved on 22/2/16.
 8 |   */
 9 | object JSONFile {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   /**
14 |     * Use the following command to create SQLContext.
15 |     */
16 |   val ssc = SparkCommon.sparkSQLContext
17 | 
18 |   val schemaOptions = Map("header" -> "true", "inferSchema" -> "true")
19 | 
20 |   def main(args: Array[String]) {
21 | 
22 |     /**
23 |       * Create the DataFrame
24 |       */
25 |     val cars = "src/main/resources/cars.json"
26 | 
27 |     /**
28 |       * read the JSON document
29 |       * Use the following command to read the JSON document named cars.json.
30 |       * The data is shown as a table with the fields − itemNo, name, speed and weight.
31 |       */
32 |     val empDataFrame: DataFrame = ssc.read.format("json").options(schemaOptions).load(cars)
33 | 
34 |     /**
35 |       * Show the Data
36 |       * If you want to see the data in the DataFrame, then use the following command.
37 |       */
38 |     empDataFrame.show()
39 | 
40 |     /**
41 |       * printSchema Method
42 |       * If you want to see the Structure (Schema) of the DataFrame, then use the following command
43 |       */
44 |     empDataFrame.printSchema()
45 | 
46 |     /**
47 |       * Select Method
48 |       * Use the following command to fetch name-column among three columns from the DataFrame
49 |       */
50 |     empDataFrame.select("name").show()
51 | 
52 |     /**
53 |       * Filter used to
54 |       * cars whose speed is greater than 300 (speed > 300).
55 |       */
56 |     empDataFrame.filter(empDataFrame("speed") > 300).show()
57 | 
58 |     /**
59 |       * groupBy Method
60 |       * counting the number of cars who are of the same speed.
61 |       */
62 |     empDataFrame.groupBy("speed").count().show()
63 | 
64 | 
65 |   }
66 | 
67 | 
68 | }
69 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/DataSources/TextFile.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql.DataSources
 2 | 
 3 | import com.tutorial.sparksql.Fruits
 4 | import com.tutorial.utils.SparkCommon
 5 | 
 6 | /**
 7 |   * Created by ved on 22/2/16.
 8 |   */
 9 | object TextFile {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   val sqlContext = SparkCommon.sparkSQLContext
14 | 
15 |   import sqlContext.implicits._
16 | 
17 |   def main(args: Array[String]) {
18 | 
19 |     /**
20 |       * Create RDD and Apply Transformations
21 |       */
22 | 
23 |     val fruits = sc.textFile("src/main/resources/fruits.txt")
24 |       .map(_.split(","))
25 |       .map(frt => Fruits(frt(0).trim.toInt, frt(1), frt(2).trim.toInt))
26 |       .toDF()
27 | 
28 |     /**
29 |       * Store the DataFrame Data in a Table
30 |       */
31 |     fruits.registerTempTable("fruits")
32 | 
33 |     /**
34 |       * Select Query on DataFrame
35 |       */
36 |     val records = sqlContext.sql("SELECT * FROM fruits")
37 | 
38 | 
39 |     /**
40 |       * To see the result data of allrecords DataFrame
41 |       */
42 |     records.show()
43 | 
44 |   }
45 | }
46 | 
47 | case class Fruits(id: Int, name: String, quantity: Int)
48 | 
49 | 
50 | 
51 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/InferringTheSchema.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | 
 5 | 
 6 | /**
 7 |   * Created by ved on 29/1/16.
 8 |   */
 9 | object InferringTheSchema {
10 | 
11 |   val sc = SparkCommon.sparkContext
12 | 
13 |   val sqlContext = SparkCommon.sparkSQLContext
14 | 
15 |   import sqlContext.implicits._
16 | 
17 |   def main(args: Array[String]) {
18 | 
19 |     /**
20 |       * Create RDD and Apply Transformations
21 |       */
22 | 
23 |     val fruits = sc.textFile("src/main/resources/fruits.txt")
24 |       .map(_.split(","))
25 |       .map(frt => Fruits(frt(0).trim.toInt, frt(1), frt(2).trim.toInt))
26 |       .toDF()
27 | 
28 |     /**
29 |       * Store the DataFrame Data in a Table
30 |       */
31 |     fruits.registerTempTable("fruits")
32 | 
33 |     /**
34 |       * Select Query on DataFrame
35 |       */
36 |     val records = sqlContext.sql("SELECT * FROM fruits")
37 | 
38 | 
39 |     /**
40 |       * To see the result data of allrecords DataFrame
41 |       */
42 |     records.show()
43 | 
44 |   }
45 | }
46 | 
47 | case class Fruits(id: Int, name: String, quantity: Int)


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/JsonSQLContext.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.DataFrame
 5 | 
 6 | /**
 7 |   * Created by anand on 1/9/16.
 8 |   */
 9 | object JsonSQLContext {
10 |   val ssc = SparkCommon.sparkSQLContext
11 | 
12 |   val schemaOptions = Map("header" -> "true", "inferSchema" -> "true")
13 | 
14 |   def main(args: Array[String]) {
15 | 
16 |     val pocOnePath = "src/main/resources/sales.json"
17 |     val dataset: DataFrame = ssc.read.format("json").options(schemaOptions).load(pocOnePath)
18 | 
19 |     dataset.printSchema()
20 |     val columns = dataset.dtypes.filter { case (col, cType) =>
21 |       println(cType)
22 |       cType.startsWith("StringType")
23 |     }.toMap.keySet.toList
24 |     dataset.describe(columns: _*).show()
25 | 
26 | 
27 |   }
28 | }


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/sparksql/ProgrammaticallySchema.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.sparksql
 2 | 
 3 | import com.tutorial.utils.SparkCommon
 4 | import org.apache.spark.sql.Row
 5 | import org.apache.spark.sql.types.{StringType, StructField, StructType}
 6 | 
 7 | /**
 8 |   * Method for creating DataFrame is through programmatic interface
 9 |   * Programmatically Specifying the Schema
10 |   *
11 |   * Created by ved on 19/1/16.
12 |   */
13 | object ProgrammaticallySchema {
14 |   val sc = SparkCommon.sparkContext
15 |   val schemaOptions = Map("header" -> "true", "inferSchema" -> "true")
16 | 
17 |   //sc is an existing SparkContext.
18 |   val sqlContext = new org.apache.spark.sql.SQLContext(sc)
19 | 
20 |   def main(args: Array[String]) {
21 | 
22 |     // Create an RDD
23 |     val fruit = sc.textFile("src/main/resources/fruits.txt")
24 | 
25 |     // The schema is encoded in a string
26 |     val schemaString = "id name"
27 | 
28 |     // Generate the schema based on the string of schema
29 |     val schema =
30 |       StructType(
31 |         schemaString.split(" ").map(fieldName => StructField(fieldName, StringType, true)))
32 | 
33 |     schema.foreach(println)
34 | 
35 |     // Convert records of the RDD (fruit) to Rows.
36 |     val rowRDD = fruit.map(_.split(",")).map(p => Row(p(0), p(1).trim))
37 | 
38 |     rowRDD.foreach(println)
39 | 
40 |     // Apply the schema to the RDD.
41 |     val fruitDataFrame = sqlContext.createDataFrame(rowRDD, schema)
42 | 
43 |     fruitDataFrame.foreach(println)
44 | 
45 |     // Register the DataFrames as a table.
46 |     fruitDataFrame.registerTempTable("fruit")
47 | 
48 |     /**
49 |       * SQL statements can be run by using the sql methods provided by sqlContext.
50 |       */
51 |     val results = sqlContext.sql("SELECT * FROM fruit")
52 |     results.show()
53 | 
54 | 
55 |   }
56 | 
57 | 
58 | }
59 | 


--------------------------------------------------------------------------------
/src/main/scala/com/tutorial/utils/SparkCommon.scala:
--------------------------------------------------------------------------------
 1 | package com.tutorial.utils
 2 | 
 3 | import org.apache.spark.sql.SQLContext
 4 | import org.apache.spark.streaming.{Seconds, StreamingContext}
 5 | import org.apache.spark.{SparkConf, SparkContext}
 6 | 
 7 | /**
 8 |   * Created by ved on 21/11/15.
 9 |   */
10 | object SparkCommon {
11 | 
12 |   lazy val conf = {
13 |     new SparkConf(false)
14 |       .setMaster("local[*]")
15 |       .setAppName("Spark Tutorial")
16 |   }
17 | 
18 |   lazy val sparkContext = new SparkContext(conf)
19 |   lazy val sparkSQLContext = SQLContext.getOrCreate(sparkContext)
20 |   lazy val streamingContext = StreamingContext.getActive()
21 |     .getOrElse(new StreamingContext(sparkContext, Seconds(2)))
22 | 
23 | }
24 | 


--------------------------------------------------------------------------------
/src/test/scala/HelloSpec.scala:
--------------------------------------------------------------------------------
1 | import org.scalatest._
2 | 
3 | class HelloSpec extends FlatSpec with Matchers {
4 |   "Hello" should "have tests" in {
5 |     true should === (true)
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/tutorial/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | <head>
  4 |     <meta charset="UTF-8">
  5 |     <title>spark-tutorial by rklick-solutions</title>
  6 | </head>
  7 | <body>
  8 | <section class="page-header">
  9 |       <h1 class="project-name">spark-tutorial</h1>
 10 |       <h2 class="project-tagline">This tutorial provides a quick introduction to using Spark</h2>
 11 |       <a href="https://github.com/rklick-solutions/spark-tutorial" class="btn">View on GitHub</a>
 12 |       <a href="https://github.com/rklick-solutions/spark-tutorial/zipball/master" class="btn">Download .zip</a>
 13 |       <a href="https://github.com/rklick-solutions/spark-tutorial/tarball/master" class="btn">Download .tar.gz</a>
 14 |     </section>
 15 | 
 16 |     <section class="main-content">
 17 |       <p><img src="https://github.com/rklick-solutions/spark-tutorial/blob/master/spark-tutorial-logo.png" alt="Spark Tutorial"></p>
 18 | 
 19 | <blockquote>
 20 | <p>This tutorial provides a quick introduction to using Spark. It demonstrates the basic functionality of RDD and DataFrame API</p>
 21 | 
 22 | <p>This is the start of using Spark with Scala, from next week onwards we would be working on this tutorial to make it grow.
 23 | We would look at how we can add more functionality into it , then we would be adding more modules to it together. If you have any
 24 | changes then feel free to send in pull requests and we would do the merges :) Stay tuned.</p>
 25 | </blockquote>
 26 | 
 27 | <h4>
 28 | <a id="spark-core" class="anchor" href="#spark-core" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#spark-core">Spark Core</a>
 29 | </h4>
 30 | 
 31 | <ul>
 32 | <li>
 33 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#introduction-to-apache-spark">Introduction </a>
 34 | 
 35 | <ul>
 36 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#features-of-apache-spark">Features </a></li>
 37 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#initializing-spark">Initializing </a></li>
 38 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#rdds">RDDs</a></li>
 39 | </ul>
 40 | </li>
 41 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#create-rdds">Create</a></li>
 42 | <li>
 43 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#rdds-operations">Operations</a>
 44 | 
 45 | <ul>
 46 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#1transformations">Transformations</a></li>
 47 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#2actions">Actions</a></li>
 48 | </ul>
 49 | </li>
 50 | </ul>
 51 | 
 52 | <h4>
 53 | <a id="spark-sql" class="anchor" href="#spark-sql" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#spark-sql">Spark SQL</a>
 54 | </h4>
 55 | 
 56 | <ul>
 57 | <li>
 58 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#Introduction">Introduction</a>
 59 | 
 60 | <ul>
 61 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#create-sql-context">Create SQL Context</a></li>
 62 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-query">Basic Query</a></li>
 63 | </ul>
 64 | </li>
 65 | <li> <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframes">DataFrames</a>
 66 | 
 67 | <ul>
 68 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#creating-dataframes">Creating DataFrames</a></li>
 69 | <li>
 70 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#datafarme-api-example-using-different-types-of-functionality">DataFarme API Functionality</a>
 71 | 
 72 | <ul>
 73 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#action">Action</a></li>
 74 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-dataframe-functions">Basic DataFrame functions</a></li>
 75 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#data-frame-operations">DataFrame Operations</a></li>
 76 | </ul>
 77 | </li>
 78 | </ul>
 79 | </li>
 80 | <li>
 81 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#interoperating-with-rdds">Interoperating with RDDs</a>
 82 | 
 83 | <ul>
 84 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#1-inferring-the-schema-using-reflection">Inferring the Schema using Reflection</a></li>
 85 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#2-programmatically-specifying-the-schema">Programmatically Specifying the Schema</a></li>
 86 | </ul>
 87 | </li>
 88 | <li>
 89 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#data-sources">Data Sources</a>
 90 | 
 91 | <ul>
 92 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframe-operations-in-json-file">DataFrame Operations in JSON file</a></li>
 93 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataframe-operations-in-text-file">DataFrame Operations in Text file</a></li>
 94 | <li><a href="https://github.com/rklick-solutions/sparhttps://github.com/rklick-solutions/spark-tutorial/wiki/Spark-Core#initializing-sparkk-tutorial/wiki/Spark-SQL#dataframe-operations-in-csv-file">DataFrame Operations in CSV file</a></li>
 95 | </ul>
 96 | </li>
 97 | <li>
 98 | <a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#dataset">Datasets</a>
 99 | 
100 | <ul>
101 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#creating-datasets">Creating Datasets</a></li>
102 | <li><a href="https://github.com/rklick-solutions/spark-tutorial/wiki/Spark-SQL#basic-opeartion">Basic Opeartion</a></li>
103 | </ul>
104 | </li>
105 | </ul>
106 | 
107 |       <footer class="site-footer">
108 |         <span class="site-footer-owner"><a href="https://github.com/rklick-solutions/spark-tutorial">spark-tutorial</a> is maintained by <a href="https://github.com/rklick-solutions">rklick-solutions</a>.</span>
109 |       </footer>
110 | 
111 |     </section>
112 | </body>
113 | </html>
114 | 


--------------------------------------------------------------------------------