├── .gitignore ├── README.md ├── dashboard ├── api_browser │ ├── __init__.py │ ├── admin.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ └── views.py ├── dashboard │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── libspark │ ├── README.md │ ├── SparkRDD.py │ ├── __init__.py │ ├── admin.py │ ├── migrations │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ └── views.py ├── manage.py └── templates │ └── api_browser │ └── index.html ├── dashboard_ui ├── .bowerrc ├── .editorconfig ├── .gitattributes ├── .gitignore ├── .jshintrc ├── .travis.yml ├── Gruntfile.js ├── app │ ├── .buildignore │ ├── .htaccess │ ├── 404.html │ ├── favicon.ico │ ├── images │ │ └── yeoman.png │ ├── index.html │ ├── robots.txt │ ├── scripts │ │ ├── app.js │ │ ├── controllers │ │ │ ├── about.js │ │ │ ├── dashboard.js │ │ │ └── main.js │ │ └── services │ │ │ └── request.js │ ├── styles │ │ └── main.css │ └── views │ │ ├── about.html │ │ ├── dashboard.html │ │ └── main.html ├── bower.json ├── package.json └── test │ ├── .jshintrc │ ├── karma.conf.js │ └── spec │ ├── controllers │ ├── about.js │ ├── dashboard.js │ └── main.js │ └── services │ └── request.js ├── dump └── foodRevPy-dev │ ├── PaginationColl.bson │ ├── PaginationColl.metadata.json │ ├── ReviewColl.bson │ ├── ReviewColl.metadata.json │ └── system.indexes.bson ├── examples ├── Foodista │ ├── Foodista │ │ ├── __init__.py │ │ ├── settings.py │ │ ├── urls.py │ │ └── wsgi.py │ ├── libspark │ │ ├── SparkRDD.py │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── tests.py │ │ └── views.py │ ├── manage.py │ └── review │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── migrations │ │ └── __init__.py │ │ ├── models.py │ │ ├── tests.py │ │ └── views.py └── README.md ├── libspark ├── README.md ├── SparkRDD.py ├── __init__.py ├── admin.py ├── migrations │ └── __init__.py ├── models.py ├── tests.py └── views.py ├── manage.py └── spark_lib ├── __init__.py ├── settings.py ├── urls.py └── wsgi.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sqlite3 3 | .idea/* 4 | idea/ 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django-LibSpark 2 | Apache Spark API for Django. The API uses PySpark which communicates with Spark. Django-LibSpark provides easy to use interface 3 | with chaining of various methods. 4 | 5 | Read Lib Spark Documentation [here](https://github.com/djangothon/django-libspark/tree/master/libspark) 6 | 7 | # Examples 8 | We have provided with an example app which works on the `Lib Spark` API. 9 | 10 | # Dashboard UI 11 | A web dashboard exposing the Spark API is built with AnguarJS which provides an interface to play around with the Lib Spark API 12 | 13 | # Dashboard 14 | The backend for Dashboard built with Django using the Lib Spark API 15 | -------------------------------------------------------------------------------- /dashboard/api_browser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/api_browser/__init__.py -------------------------------------------------------------------------------- /dashboard/api_browser/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /dashboard/api_browser/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/api_browser/migrations/__init__.py -------------------------------------------------------------------------------- /dashboard/api_browser/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /dashboard/api_browser/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /dashboard/api_browser/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from django.http import JsonResponse 3 | from django.views.decorators.csrf import csrf_exempt 4 | import json 5 | from libspark.SparkRDD import SparkRDD 6 | 7 | def index(request): 8 | return render(request, 'api_browser/index.html',{}) 9 | 10 | @csrf_exempt 11 | def evaluate(request): 12 | if request.method == "POST": 13 | print request.body 14 | decodedJson = json.loads(str(request.body)) 15 | i = 0 16 | queryString = '' 17 | exec('data = '+decodedJson[-1]) 18 | dataRDD = SparkRDD(data=data) 19 | for method in decodedJson[0]: 20 | if i > 0: 21 | queryString = queryString + '.' 22 | i = i+1 23 | if (method['func'] is True): 24 | queryString = queryString + method['name'] + '(' + method['lambda'] + ')' 25 | else: 26 | queryString = queryString + method['name'] + '()' 27 | queryString = 'dataRDD.' + queryString + '.' + decodedJson[1]['name'] + '()' 28 | print queryString 29 | exec('output = '+queryString) 30 | print output 31 | return JsonResponse({'success':output}) 32 | -------------------------------------------------------------------------------- /dashboard/dashboard/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/dashboard/__init__.py -------------------------------------------------------------------------------- /dashboard/dashboard/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for dashboard project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.8.4. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.8/ref/settings/ 11 | """ 12 | 13 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 14 | import os 15 | 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | PROJECT_APP_DIR = os.path.realpath(os.path.dirname(__file__)) 19 | PROJECT_DIR = os.path.dirname(PROJECT_APP_DIR) 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = 'l=a*l2673^rc)juq_c@_ghb(p5wqg0%t#ntj3f5trv7f4o8i)l' 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = True 29 | 30 | ALLOWED_HOSTS = [] 31 | 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = ( 36 | 'django.contrib.admin', 37 | 'django.contrib.auth', 38 | 'django.contrib.contenttypes', 39 | 'django.contrib.sessions', 40 | 'django.contrib.messages', 41 | 'django.contrib.staticfiles', 42 | 'corsheaders', 43 | 'api_browser', 44 | 'libspark', 45 | ) 46 | 47 | MIDDLEWARE_CLASSES = ( 48 | 'django.contrib.sessions.middleware.SessionMiddleware', 49 | 'corsheaders.middleware.CorsMiddleware', 50 | 'django.middleware.common.CommonMiddleware', 51 | 'django.middleware.csrf.CsrfViewMiddleware', 52 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 53 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 54 | 'django.contrib.messages.middleware.MessageMiddleware', 55 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 56 | 'django.middleware.security.SecurityMiddleware', 57 | ) 58 | 59 | ROOT_URLCONF = 'dashboard.urls' 60 | 61 | TEMPLATES = [ 62 | { 63 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 64 | 'DIRS': [os.path.join(os.path.dirname(__file__),'templates'), 65 | os.path.join(PROJECT_DIR, 'templates')], 66 | 'APP_DIRS': True, 67 | 'OPTIONS': { 68 | 'context_processors': [ 69 | 'django.template.context_processors.debug', 70 | 'django.template.context_processors.request', 71 | 'django.contrib.auth.context_processors.auth', 72 | 'django.contrib.messages.context_processors.messages', 73 | ], 74 | }, 75 | }, 76 | ] 77 | 78 | WSGI_APPLICATION = 'dashboard.wsgi.application' 79 | 80 | 81 | # Database 82 | # https://docs.djangoproject.com/en/1.8/ref/settings/#databases 83 | 84 | DATABASES = { 85 | 'default': { 86 | 'ENGINE': 'django.db.backends.sqlite3', 87 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 88 | } 89 | } 90 | 91 | 92 | # Internationalization 93 | # https://docs.djangoproject.com/en/1.8/topics/i18n/ 94 | 95 | LANGUAGE_CODE = 'en-us' 96 | 97 | TIME_ZONE = 'UTC' 98 | 99 | USE_I18N = True 100 | 101 | USE_L10N = True 102 | 103 | USE_TZ = True 104 | 105 | 106 | # Static files (CSS, JavaScript, Images) 107 | # https://docs.djangoproject.com/en/1.8/howto/static-files/ 108 | STATIC_URL = '/static/' 109 | MEDIA_URL = '/media/' 110 | MEDIA_ROOT = '/media/' 111 | 112 | PROJECT_APP_DIR = os.path.realpath(os.path.dirname(__file__)) 113 | PROJECT_DIR = os.path.dirname(PROJECT_APP_DIR) 114 | 115 | # TEMPLATE_DIRS = ( 116 | # os.path.join(os.path.dirname(__file__),'templates'), 117 | # os.path.join(PROJECT_DIR, 'templates'), 118 | # ) 119 | 120 | #Assets Folder 121 | STATICFILES_DIRS = ( 122 | os.path.join(PROJECT_DIR,'static'), 123 | ) 124 | 125 | STATICFILES_FINDERS = ( 126 | 'django.contrib.staticfiles.finders.FileSystemFinder', 127 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 128 | ) 129 | 130 | # CORS Headers 131 | CORS_ORIGIN_ALLOW_ALL = True 132 | -------------------------------------------------------------------------------- /dashboard/dashboard/urls.py: -------------------------------------------------------------------------------- 1 | """dashboard URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.8/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add an import: from blog import urls as blog_urls 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) 15 | """ 16 | from django.conf.urls import include, url 17 | from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | url(r'^admin/', include(admin.site.urls)), 21 | url(r'^$','api_browser.views.evaluate'), 22 | ] 23 | -------------------------------------------------------------------------------- /dashboard/dashboard/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for dashboard project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /dashboard/libspark/README.md: -------------------------------------------------------------------------------- 1 | Django Lib Spark 2 | ================ 3 | 4 | This is the core Django Lib Spark. This library is built on top of [PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) available for [Apache Spark](http://spark.apache.org/). 5 | 6 | What makes Spark Lib easy to use? 7 | ================================= 8 | 9 | Lib Spark makes communication to a Spark instacnce a cake walk. Lib Spark takes care of all the connection related initialisation scripts. It also supports powerfull method chaining mechanism along with lazy evaluation. 10 | 11 | Packages 12 | ======== 13 | 14 | 1. [pyspark.RDD](http://spark.apache.org/docs/latest/api/python/pyspark.html#pyspark.RDD) 15 | 16 | Functions Implemented 17 | ===================== 18 | 19 | * map 20 | * filter 21 | * collect 22 | * cache 23 | * count 24 | * distinct 25 | * first 26 | * flatMap 27 | * flatMapValues 28 | * groupBy 29 | * groupByKey 30 | * intersection 31 | * join 32 | * keyBy 33 | * keys 34 | * leftOuterJoin 35 | * lookup 36 | * mapPartitions 37 | * mapPartitionsWithIndex 38 | * mapPartitionsWithSplit 39 | * mapValues 40 | * max 41 | * mean 42 | * min 43 | * name 44 | * reduceByKey 45 | * reduceByKeyLocally 46 | * rightOuterJoin 47 | * sortBy 48 | * sortByKey 49 | * substract 50 | * sum 51 | * take 52 | * takeOrdered 53 | * top 54 | * union 55 | * values 56 | * variance 57 | * zip 58 | * zipWithIndex 59 | * zipWithUniqueId 60 | -------------------------------------------------------------------------------- /dashboard/libspark/SparkRDD.py: -------------------------------------------------------------------------------- 1 | # Configure the necessary Spark environment 2 | import os 3 | import sys 4 | 5 | # Spark home 6 | spark_home = os.environ.get("SPARK_HOME") 7 | 8 | # If Spark V1.4.x is detected, then add ' pyspark-shell' to 9 | # the end of the 'PYSPARK_SUBMIT_ARGS' environment variable 10 | spark_release_file = spark_home + "/RELEASE" 11 | if os.path.exists(spark_release_file) and "Spark 1.4" in open(spark_release_file).read(): 12 | pyspark_submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "") 13 | if not "pyspark-shell" in pyspark_submit_args: pyspark_submit_args += " pyspark-shell" 14 | os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args 15 | 16 | # Add the spark python sub-directory to the path 17 | sys.path.insert(0, spark_home + "/python") 18 | 19 | # Add the py4j to the path. 20 | # You may need to change the version number to match your install 21 | sys.path.insert(0, os.path.join(spark_home, "python/lib/py4j-0.8.2.1-src.zip")) 22 | # Initialize PySpark to predefine the SparkContext variable 'sc' 23 | execfile(os.path.join(spark_home, "python/pyspark/shell.py")) 24 | 25 | class SparkRDD(): 26 | # When the SparkRDD is initialized - data and function 27 | # can be passed as arguments 28 | def __init__(self, data=[], **kwargs): 29 | self.func = kwargs.get("func",None) 30 | self.data = data 31 | self.is_cached = False 32 | try: 33 | self.dataRDD = sc.parallelize (data) 34 | except TypeError: 35 | self.dataRDD = data 36 | 37 | def map(self,func): 38 | mappedData = self.dataRDD.map(func) 39 | tempObj = SparkRDD(data = mappedData) 40 | return tempObj 41 | 42 | def filter(self,func): 43 | filteredData = self.dataRDD.filter(func) 44 | tempObj = SparkRDD(data = filteredData) 45 | return tempObj 46 | 47 | def collect(self): 48 | return self.dataRDD.collect() 49 | 50 | def cache(self): 51 | self.dataRDD.cache() 52 | self.is_cached = True 53 | return self 54 | 55 | def count(self): 56 | return self.dataRDD.count() 57 | 58 | def distinct(self): 59 | distinctRDD = self.dataRDD.distinct() 60 | distinctObj = SparkRDD(data=distinctRDD) 61 | return distinctObj 62 | 63 | def first(self): 64 | return self.dataRDD.first() 65 | 66 | def flatMap(self,func): 67 | flatMapRDD = self.dataRDD.flatMap(func) 68 | tempObj = SparkRDD(data=flatMapRDD) 69 | return tempObj 70 | 71 | def flatMapValues(self,func): 72 | flatMapRDD = self.dataRDD.flatMapValues(func) 73 | tempObj = SparkRDD(data=flatMapRDD) 74 | return tempObj 75 | 76 | def groupBy(self,func): 77 | groupByRDD = self.dataRDD.groupBy(func) 78 | tempObj = SparkRDD(data=groupByRDD) 79 | return tempObj 80 | 81 | def groupByKey(self): 82 | groupByRDD = self.dataRDD.groupByKey() 83 | tempObj = SparkRDD(data=groupByRDD) 84 | return tempObj 85 | 86 | def intersection(self,toBeIntersected): 87 | if isinstance(toBeIntersected,SparkRDD): 88 | intersected = self.dataRDD.intersection(toBeIntersected.dataRDD) 89 | tempObj = SparkRDD(data=intersected) 90 | else: 91 | #TODO: rasie err 92 | pass 93 | return tempObj 94 | 95 | def join(self,toBeJoined): 96 | if isinstance(toBeJoined,SparkRDD): 97 | joined = self.dataRDD.join(toBeJoined.dataRDD) 98 | tempObj = SparkRDD(data=joined) 99 | else: 100 | #TODO: rasie err 101 | pass 102 | return tempObj 103 | 104 | def keyBy(self,func): 105 | keyByRDD = self.dataRDD.keyBy(func) 106 | tempObj = SparkRDD(data=keyByRDD) 107 | return tempObj 108 | 109 | def keys(self): 110 | keysRDD = self.dataRDD.keys() 111 | tempObj = SparkRDD(data=keysRDD) 112 | return tempObj 113 | 114 | def leftOuterJoin(self,toBeJoined): 115 | if isinstance(toBeJoined,SparkRDD): 116 | leftJoined = self.dataRDD.join(toBeJoined.dataRDD) 117 | tempObj = SparkRDD(data=leftJoined) 118 | else: 119 | #TODO: rasie err 120 | pass 121 | return tempObj 122 | 123 | 124 | def lookup(self,key): 125 | valuesList = self.dataRDD.lookup(key) 126 | return valuesList 127 | 128 | def mapPartitions(self,func): 129 | mapPartitionsRDD = self.dataRDD.mapPartitions(func) 130 | tempObj = SparkRDD(data=mapPartitionsRDD) 131 | return tempObj 132 | 133 | def mapPartitionsWithIndex(self,func): 134 | mapPartitionsWithIndexRDD = self.dataRDD.mapPartitionsWithIndex(func) 135 | tempObj = SparkRDD(data=mapPartitionsWithIndexRDD) 136 | return tempObj 137 | 138 | def mapPartitionsWithSplit(self,func): 139 | mapPartitionsWithSplitRDD = self.dataRDD.mapPartitionsWithSplit(func) 140 | tempObj = SparkRDD(data=mapPartitionsWithSplitRDD) 141 | return tempObj 142 | 143 | def mapValues(self,func): 144 | mapValuesRDD = self.dataRDD.mapValues(func) 145 | tempObj = SparkRDD(data=mapValuesRDD) 146 | return tempObj 147 | 148 | def max(self,func=None): 149 | if func is None: 150 | maxed = self.dataRDD.max() 151 | else: 152 | maxed = self.dataRDD.max(func) 153 | return maxed 154 | 155 | def mean(self): 156 | meanRDD = self.dataRDD.mean() 157 | tempObj = SparkRDD(data=meanRDD) 158 | return tempObj 159 | 160 | def min(self,func=None): 161 | if func is None: 162 | mined = self.dataRDD.min() 163 | else: 164 | mined = self.dataRDD.min(func) 165 | return mined 166 | 167 | def name(self): 168 | return self.dataRDD.name 169 | 170 | def reduceByKey(self,func=None): 171 | if func is None: 172 | reduceByKeyRDD = self.dataRDD.reduceByKey() 173 | else: 174 | reduceByKeyRDD = self.dataRDD.reduceByKey(func) 175 | tempObj = SparkRDD(data=reduceByKeyRDD) 176 | return tempObj 177 | 178 | def reduceByKeyLocally(self,func=None): 179 | if key is None: 180 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally() 181 | else: 182 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally(func) 183 | tempObj = SparkRDD(data=reduceByKeyLocallyRDD) 184 | return tempObj 185 | 186 | 187 | def rightOuterJoin(self,toBeJoined): 188 | if isinstance(toBeJoined,SparkRDD): 189 | rightOuterJoinRDD = self.dataRDD.join(toBeJoined.dataRDD) 190 | tempObj = SparkRDD(data=rightOuterJoinRDD) 191 | else: 192 | #TODO: rasie err 193 | pass 194 | return tempObj 195 | 196 | def sortBy(self,func,ascending=True): 197 | sortedBy = self.dataRDD.sortBy(func) 198 | sortedObj = SparkRDD(data=sortedBy) 199 | return sortedObj 200 | 201 | def sortByKey(self,func,ascending=True): 202 | sortedByKey = self.dataRDD.sortByKey(func) 203 | sortedObj = SparkRDD(data=sortedByKey) 204 | return sortedObj 205 | 206 | def substract(self,toBeSubtracted): 207 | if isinstance(toBeJoined,SparkRDD): 208 | subtracted = self.dataRDD.subtract(toBeSubtracted.dataRDD) 209 | subtractedObj = SparkRDD(data=subtracted) 210 | else: 211 | #TODO: raise err 212 | pass 213 | return subtractedObj 214 | 215 | def sum(self): 216 | summed = self.dataRDD.sum() 217 | sumObj = SparkRDD(data=summed) 218 | return sumObj 219 | 220 | def take(self,num): 221 | took = self.dataRDD.take(num) 222 | tookObj = SparkRDD(data=took) 223 | return tookObj 224 | 225 | def takeOrdered(self,num,func=None): 226 | if func is None: 227 | tookOredered = self.dataRDD.takeOrdered(num) 228 | else: 229 | tookOredered = self.dataRDD.takeOrdered(num,key=func) 230 | return tookOredered 231 | 232 | def top(self,num,func): 233 | if func is None: 234 | topped = self.dataRDD.top(num) 235 | else: 236 | topped = self.dataRDD.top(num,key=func) 237 | return topped 238 | 239 | def union(self): 240 | unioned = self.dataRDD.union() 241 | unionObj = SparkRDD(data=unioned) 242 | return unionObj 243 | 244 | def values(self): 245 | valued = self.dataRDD.values() 246 | valueObj = SparkRDD(data=valued) 247 | return valueObj 248 | 249 | def variance(self): 250 | return self.dataRDD.variance() 251 | 252 | 253 | def zip(self,toBeZipped): 254 | if isinstance(toBeZipped,SparkRDD): 255 | zipped = self.dataRDD.zip(toBeZipped) 256 | else: 257 | #TODO: Raise err 258 | pass 259 | zipObj = SparkRDD(data=zipped) 260 | return zipObj 261 | 262 | def zipWithIndex(self): 263 | zipped = self.dataRDD.zipWithIndex() 264 | zipObj = SparkRDD(data=zipped) 265 | return zipObj 266 | 267 | def zipWithUniqueId(self): 268 | zipped = self.dataRDD.zipWithUniqueId() 269 | zippedObj = SparkRDD(data = zipped) 270 | return zipObj 271 | -------------------------------------------------------------------------------- /dashboard/libspark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/libspark/__init__.py -------------------------------------------------------------------------------- /dashboard/libspark/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /dashboard/libspark/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/libspark/migrations/__init__.py -------------------------------------------------------------------------------- /dashboard/libspark/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /dashboard/libspark/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /dashboard/libspark/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard/libspark/views.py -------------------------------------------------------------------------------- /dashboard/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /dashboard/templates/api_browser/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | API Browser 8 | 9 | 10 | 11 | 12 | 13 | 14 | 19 | 20 | 21 | 22 |
23 | {{1+2}} 24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 | 40 |
41 |
42 | 43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 | 74 |
75 |
76 | 2nd 77 |
78 |
79 |
80 |
81 | 3rd 82 |
83 |
84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /dashboard_ui/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory": "bower_components" 3 | } 4 | -------------------------------------------------------------------------------- /dashboard_ui/.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain consistent 2 | # coding styles between different editors and IDEs 3 | # editorconfig.org 4 | 5 | root = true 6 | 7 | 8 | [*] 9 | 10 | # Change these settings to your own preference 11 | indent_style = space 12 | indent_size = 2 13 | 14 | # We recommend you to keep these unchanged 15 | end_of_line = lf 16 | charset = utf-8 17 | trim_trailing_whitespace = true 18 | insert_final_newline = true 19 | 20 | [*.md] 21 | trim_trailing_whitespace = false 22 | -------------------------------------------------------------------------------- /dashboard_ui/.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto -------------------------------------------------------------------------------- /dashboard_ui/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .tmp 4 | .sass-cache 5 | bower_components 6 | -------------------------------------------------------------------------------- /dashboard_ui/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "browser": true, 4 | "esnext": true, 5 | "bitwise": true, 6 | "camelcase": true, 7 | "curly": true, 8 | "eqeqeq": true, 9 | "immed": true, 10 | "indent": 2, 11 | "latedef": true, 12 | "newcap": true, 13 | "noarg": true, 14 | "quotmark": "single", 15 | "regexp": true, 16 | "undef": true, 17 | "unused": true, 18 | "strict": true, 19 | "trailing": true, 20 | "smarttabs": true, 21 | "globals": { 22 | "angular": false 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dashboard_ui/.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '0.10' 4 | before_script: 5 | - 'npm install -g bower grunt-cli' 6 | - 'bower install' 7 | -------------------------------------------------------------------------------- /dashboard_ui/Gruntfile.js: -------------------------------------------------------------------------------- 1 | // Generated on 2015-08-23 using generator-angular 0.9.2 2 | 'use strict'; 3 | 4 | // # Globbing 5 | // for performance reasons we're only matching one level down: 6 | // 'test/spec/{,*/}*.js' 7 | // use this if you want to recursively match all subfolders: 8 | // 'test/spec/**/*.js' 9 | 10 | module.exports = function (grunt) { 11 | 12 | // Load grunt tasks automatically 13 | require('load-grunt-tasks')(grunt); 14 | 15 | // Time how long tasks take. Can help when optimizing build times 16 | require('time-grunt')(grunt); 17 | 18 | // Configurable paths for the application 19 | var appConfig = { 20 | app: require('./bower.json').appPath || 'app', 21 | dist: 'dist' 22 | }; 23 | 24 | // Define the configuration for all the tasks 25 | grunt.initConfig({ 26 | 27 | // Project settings 28 | yeoman: appConfig, 29 | 30 | // Watches files for changes and runs tasks based on the changed files 31 | watch: { 32 | bower: { 33 | files: ['bower.json'], 34 | tasks: ['wiredep'] 35 | }, 36 | js: { 37 | files: ['<%= yeoman.app %>/scripts/{,*/}*.js'], 38 | tasks: ['newer:jshint:all'], 39 | options: { 40 | livereload: '<%= connect.options.livereload %>' 41 | } 42 | }, 43 | jsTest: { 44 | files: ['test/spec/{,*/}*.js'], 45 | tasks: ['newer:jshint:test', 'karma'] 46 | }, 47 | styles: { 48 | files: ['<%= yeoman.app %>/styles/{,*/}*.css'], 49 | tasks: ['newer:copy:styles', 'autoprefixer'] 50 | }, 51 | gruntfile: { 52 | files: ['Gruntfile.js'] 53 | }, 54 | livereload: { 55 | options: { 56 | livereload: '<%= connect.options.livereload %>' 57 | }, 58 | files: [ 59 | '<%= yeoman.app %>/{,*/}*.html', 60 | '.tmp/styles/{,*/}*.css', 61 | '<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}' 62 | ] 63 | } 64 | }, 65 | 66 | // The actual grunt server settings 67 | connect: { 68 | options: { 69 | port: 9000, 70 | // Change this to '0.0.0.0' to access the server from outside. 71 | hostname: 'localhost', 72 | livereload: 35729 73 | }, 74 | livereload: { 75 | options: { 76 | open: true, 77 | middleware: function (connect) { 78 | return [ 79 | connect.static('.tmp'), 80 | connect().use( 81 | '/bower_components', 82 | connect.static('./bower_components') 83 | ), 84 | connect.static(appConfig.app) 85 | ]; 86 | } 87 | } 88 | }, 89 | test: { 90 | options: { 91 | port: 9001, 92 | middleware: function (connect) { 93 | return [ 94 | connect.static('.tmp'), 95 | connect.static('test'), 96 | connect().use( 97 | '/bower_components', 98 | connect.static('./bower_components') 99 | ), 100 | connect.static(appConfig.app) 101 | ]; 102 | } 103 | } 104 | }, 105 | dist: { 106 | options: { 107 | open: true, 108 | base: '<%= yeoman.dist %>' 109 | } 110 | } 111 | }, 112 | 113 | // Make sure code styles are up to par and there are no obvious mistakes 114 | jshint: { 115 | options: { 116 | jshintrc: '.jshintrc', 117 | reporter: require('jshint-stylish') 118 | }, 119 | all: { 120 | src: [ 121 | 'Gruntfile.js', 122 | '<%= yeoman.app %>/scripts/{,*/}*.js' 123 | ] 124 | }, 125 | test: { 126 | options: { 127 | jshintrc: 'test/.jshintrc' 128 | }, 129 | src: ['test/spec/{,*/}*.js'] 130 | } 131 | }, 132 | 133 | // Empties folders to start fresh 134 | clean: { 135 | dist: { 136 | files: [{ 137 | dot: true, 138 | src: [ 139 | '.tmp', 140 | '<%= yeoman.dist %>/{,*/}*', 141 | '!<%= yeoman.dist %>/.git*' 142 | ] 143 | }] 144 | }, 145 | server: '.tmp' 146 | }, 147 | 148 | // Add vendor prefixed styles 149 | autoprefixer: { 150 | options: { 151 | browsers: ['last 1 version'] 152 | }, 153 | dist: { 154 | files: [{ 155 | expand: true, 156 | cwd: '.tmp/styles/', 157 | src: '{,*/}*.css', 158 | dest: '.tmp/styles/' 159 | }] 160 | } 161 | }, 162 | 163 | // Automatically inject Bower components into the app 164 | wiredep: { 165 | options: { 166 | cwd: '<%= yeoman.app %>' 167 | }, 168 | app: { 169 | src: ['<%= yeoman.app %>/index.html'], 170 | ignorePath: /\.\.\// 171 | } 172 | }, 173 | 174 | // Renames files for browser caching purposes 175 | filerev: { 176 | dist: { 177 | src: [ 178 | '<%= yeoman.dist %>/scripts/{,*/}*.js', 179 | '<%= yeoman.dist %>/styles/{,*/}*.css', 180 | '<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}', 181 | '<%= yeoman.dist %>/styles/fonts/*' 182 | ] 183 | } 184 | }, 185 | 186 | // Reads HTML for usemin blocks to enable smart builds that automatically 187 | // concat, minify and revision files. Creates configurations in memory so 188 | // additional tasks can operate on them 189 | useminPrepare: { 190 | html: '<%= yeoman.app %>/index.html', 191 | options: { 192 | dest: '<%= yeoman.dist %>', 193 | flow: { 194 | html: { 195 | steps: { 196 | js: ['concat', 'uglifyjs'], 197 | css: ['cssmin'] 198 | }, 199 | post: {} 200 | } 201 | } 202 | } 203 | }, 204 | 205 | // Performs rewrites based on filerev and the useminPrepare configuration 206 | usemin: { 207 | html: ['<%= yeoman.dist %>/{,*/}*.html'], 208 | css: ['<%= yeoman.dist %>/styles/{,*/}*.css'], 209 | options: { 210 | assetsDirs: ['<%= yeoman.dist %>','<%= yeoman.dist %>/images'] 211 | } 212 | }, 213 | 214 | // The following *-min tasks will produce minified files in the dist folder 215 | // By default, your `index.html`'s will take care of 216 | // minification. These next options are pre-configured if you do not wish 217 | // to use the Usemin blocks. 218 | // cssmin: { 219 | // dist: { 220 | // files: { 221 | // '<%= yeoman.dist %>/styles/main.css': [ 222 | // '.tmp/styles/{,*/}*.css' 223 | // ] 224 | // } 225 | // } 226 | // }, 227 | // uglify: { 228 | // dist: { 229 | // files: { 230 | // '<%= yeoman.dist %>/scripts/scripts.js': [ 231 | // '<%= yeoman.dist %>/scripts/scripts.js' 232 | // ] 233 | // } 234 | // } 235 | // }, 236 | // concat: { 237 | // dist: {} 238 | // }, 239 | 240 | imagemin: { 241 | dist: { 242 | files: [{ 243 | expand: true, 244 | cwd: '<%= yeoman.app %>/images', 245 | src: '{,*/}*.{png,jpg,jpeg,gif}', 246 | dest: '<%= yeoman.dist %>/images' 247 | }] 248 | } 249 | }, 250 | 251 | svgmin: { 252 | dist: { 253 | files: [{ 254 | expand: true, 255 | cwd: '<%= yeoman.app %>/images', 256 | src: '{,*/}*.svg', 257 | dest: '<%= yeoman.dist %>/images' 258 | }] 259 | } 260 | }, 261 | 262 | htmlmin: { 263 | dist: { 264 | options: { 265 | collapseWhitespace: true, 266 | conservativeCollapse: true, 267 | collapseBooleanAttributes: true, 268 | removeCommentsFromCDATA: true, 269 | removeOptionalTags: true 270 | }, 271 | files: [{ 272 | expand: true, 273 | cwd: '<%= yeoman.dist %>', 274 | src: ['*.html', 'views/{,*/}*.html'], 275 | dest: '<%= yeoman.dist %>' 276 | }] 277 | } 278 | }, 279 | 280 | // ngmin tries to make the code safe for minification automatically by 281 | // using the Angular long form for dependency injection. It doesn't work on 282 | // things like resolve or inject so those have to be done manually. 283 | ngmin: { 284 | dist: { 285 | files: [{ 286 | expand: true, 287 | cwd: '.tmp/concat/scripts', 288 | src: '*.js', 289 | dest: '.tmp/concat/scripts' 290 | }] 291 | } 292 | }, 293 | 294 | // Replace Google CDN references 295 | cdnify: { 296 | dist: { 297 | html: ['<%= yeoman.dist %>/*.html'] 298 | } 299 | }, 300 | 301 | // Copies remaining files to places other tasks can use 302 | copy: { 303 | dist: { 304 | files: [{ 305 | expand: true, 306 | dot: true, 307 | cwd: '<%= yeoman.app %>', 308 | dest: '<%= yeoman.dist %>', 309 | src: [ 310 | '*.{ico,png,txt}', 311 | '.htaccess', 312 | '*.html', 313 | 'views/{,*/}*.html', 314 | 'images/{,*/}*.{webp}', 315 | 'fonts/*' 316 | ] 317 | }, { 318 | expand: true, 319 | cwd: '.tmp/images', 320 | dest: '<%= yeoman.dist %>/images', 321 | src: ['generated/*'] 322 | }, { 323 | expand: true, 324 | cwd: 'bower_components/bootstrap/dist', 325 | src: 'fonts/*', 326 | dest: '<%= yeoman.dist %>' 327 | }] 328 | }, 329 | styles: { 330 | expand: true, 331 | cwd: '<%= yeoman.app %>/styles', 332 | dest: '.tmp/styles/', 333 | src: '{,*/}*.css' 334 | } 335 | }, 336 | 337 | // Run some tasks in parallel to speed up the build process 338 | concurrent: { 339 | server: [ 340 | 'copy:styles' 341 | ], 342 | test: [ 343 | 'copy:styles' 344 | ], 345 | dist: [ 346 | 'copy:styles', 347 | 'imagemin', 348 | 'svgmin' 349 | ] 350 | }, 351 | 352 | // Test settings 353 | karma: { 354 | unit: { 355 | configFile: 'test/karma.conf.js', 356 | singleRun: true 357 | } 358 | } 359 | }); 360 | 361 | 362 | grunt.registerTask('serve', 'Compile then start a connect web server', function (target) { 363 | if (target === 'dist') { 364 | return grunt.task.run(['build', 'connect:dist:keepalive']); 365 | } 366 | 367 | grunt.task.run([ 368 | 'clean:server', 369 | 'wiredep', 370 | 'concurrent:server', 371 | 'autoprefixer', 372 | 'connect:livereload', 373 | 'watch' 374 | ]); 375 | }); 376 | 377 | grunt.registerTask('server', 'DEPRECATED TASK. Use the "serve" task instead', function (target) { 378 | grunt.log.warn('The `server` task has been deprecated. Use `grunt serve` to start a server.'); 379 | grunt.task.run(['serve:' + target]); 380 | }); 381 | 382 | grunt.registerTask('test', [ 383 | 'clean:server', 384 | 'concurrent:test', 385 | 'autoprefixer', 386 | 'connect:test', 387 | 'karma' 388 | ]); 389 | 390 | grunt.registerTask('build', [ 391 | 'clean:dist', 392 | 'wiredep', 393 | 'useminPrepare', 394 | 'concurrent:dist', 395 | 'autoprefixer', 396 | 'concat', 397 | 'ngmin', 398 | 'copy:dist', 399 | 'cdnify', 400 | 'cssmin', 401 | 'uglify', 402 | 'filerev', 403 | 'usemin', 404 | 'htmlmin' 405 | ]); 406 | 407 | grunt.registerTask('default', [ 408 | 'newer:jshint', 409 | 'test', 410 | 'build' 411 | ]); 412 | }; 413 | -------------------------------------------------------------------------------- /dashboard_ui/app/.buildignore: -------------------------------------------------------------------------------- 1 | *.coffee -------------------------------------------------------------------------------- /dashboard_ui/app/.htaccess: -------------------------------------------------------------------------------- 1 | # Apache Configuration File 2 | 3 | # (!) Using `.htaccess` files slows down Apache, therefore, if you have access 4 | # to the main server config file (usually called `httpd.conf`), you should add 5 | # this logic there: http://httpd.apache.org/docs/current/howto/htaccess.html. 6 | 7 | # ############################################################################## 8 | # # CROSS-ORIGIN RESOURCE SHARING (CORS) # 9 | # ############################################################################## 10 | 11 | # ------------------------------------------------------------------------------ 12 | # | Cross-domain AJAX requests | 13 | # ------------------------------------------------------------------------------ 14 | 15 | # Enable cross-origin AJAX requests. 16 | # http://code.google.com/p/html5security/wiki/CrossOriginRequestSecurity 17 | # http://enable-cors.org/ 18 | 19 | # 20 | # Header set Access-Control-Allow-Origin "*" 21 | # 22 | 23 | # ------------------------------------------------------------------------------ 24 | # | CORS-enabled images | 25 | # ------------------------------------------------------------------------------ 26 | 27 | # Send the CORS header for images when browsers request it. 28 | # https://developer.mozilla.org/en/CORS_Enabled_Image 29 | # http://blog.chromium.org/2011/07/using-cross-domain-images-in-webgl-and.html 30 | # http://hacks.mozilla.org/2011/11/using-cors-to-load-webgl-textures-from-cross-domain-images/ 31 | 32 | 33 | 34 | 35 | SetEnvIf Origin ":" IS_CORS 36 | Header set Access-Control-Allow-Origin "*" env=IS_CORS 37 | 38 | 39 | 40 | 41 | # ------------------------------------------------------------------------------ 42 | # | Web fonts access | 43 | # ------------------------------------------------------------------------------ 44 | 45 | # Allow access from all domains for web fonts 46 | 47 | 48 | 49 | Header set Access-Control-Allow-Origin "*" 50 | 51 | 52 | 53 | 54 | # ############################################################################## 55 | # # ERRORS # 56 | # ############################################################################## 57 | 58 | # ------------------------------------------------------------------------------ 59 | # | 404 error prevention for non-existing redirected folders | 60 | # ------------------------------------------------------------------------------ 61 | 62 | # Prevent Apache from returning a 404 error for a rewrite if a directory 63 | # with the same name does not exist. 64 | # http://httpd.apache.org/docs/current/content-negotiation.html#multiviews 65 | # http://www.webmasterworld.com/apache/3808792.htm 66 | 67 | Options -MultiViews 68 | 69 | # ------------------------------------------------------------------------------ 70 | # | Custom error messages / pages | 71 | # ------------------------------------------------------------------------------ 72 | 73 | # You can customize what Apache returns to the client in case of an error (see 74 | # http://httpd.apache.org/docs/current/mod/core.html#errordocument), e.g.: 75 | 76 | ErrorDocument 404 /404.html 77 | 78 | 79 | # ############################################################################## 80 | # # INTERNET EXPLORER # 81 | # ############################################################################## 82 | 83 | # ------------------------------------------------------------------------------ 84 | # | Better website experience | 85 | # ------------------------------------------------------------------------------ 86 | 87 | # Force IE to render pages in the highest available mode in the various 88 | # cases when it may not: http://hsivonen.iki.fi/doctype/ie-mode.pdf. 89 | 90 | 91 | Header set X-UA-Compatible "IE=edge" 92 | # `mod_headers` can't match based on the content-type, however, we only 93 | # want to send this header for HTML pages and not for the other resources 94 | 95 | Header unset X-UA-Compatible 96 | 97 | 98 | 99 | # ------------------------------------------------------------------------------ 100 | # | Cookie setting from iframes | 101 | # ------------------------------------------------------------------------------ 102 | 103 | # Allow cookies to be set from iframes in IE. 104 | 105 | # 106 | # Header set P3P "policyref=\"/w3c/p3p.xml\", CP=\"IDC DSP COR ADM DEVi TAIi PSA PSD IVAi IVDi CONi HIS OUR IND CNT\"" 107 | # 108 | 109 | # ------------------------------------------------------------------------------ 110 | # | Screen flicker | 111 | # ------------------------------------------------------------------------------ 112 | 113 | # Stop screen flicker in IE on CSS rollovers (this only works in 114 | # combination with the `ExpiresByType` directives for images from below). 115 | 116 | # BrowserMatch "MSIE" brokenvary=1 117 | # BrowserMatch "Mozilla/4.[0-9]{2}" brokenvary=1 118 | # BrowserMatch "Opera" !brokenvary 119 | # SetEnvIf brokenvary 1 force-no-vary 120 | 121 | 122 | # ############################################################################## 123 | # # MIME TYPES AND ENCODING # 124 | # ############################################################################## 125 | 126 | # ------------------------------------------------------------------------------ 127 | # | Proper MIME types for all files | 128 | # ------------------------------------------------------------------------------ 129 | 130 | 131 | 132 | # Audio 133 | AddType audio/mp4 m4a f4a f4b 134 | AddType audio/ogg oga ogg 135 | 136 | # JavaScript 137 | # Normalize to standard type (it's sniffed in IE anyways): 138 | # http://tools.ietf.org/html/rfc4329#section-7.2 139 | AddType application/javascript js jsonp 140 | AddType application/json json 141 | 142 | # Video 143 | AddType video/mp4 mp4 m4v f4v f4p 144 | AddType video/ogg ogv 145 | AddType video/webm webm 146 | AddType video/x-flv flv 147 | 148 | # Web fonts 149 | AddType application/font-woff woff 150 | AddType application/vnd.ms-fontobject eot 151 | 152 | # Browsers usually ignore the font MIME types and sniff the content, 153 | # however, Chrome shows a warning if other MIME types are used for the 154 | # following fonts. 155 | AddType application/x-font-ttf ttc ttf 156 | AddType font/opentype otf 157 | 158 | # Make SVGZ fonts work on iPad: 159 | # https://twitter.com/FontSquirrel/status/14855840545 160 | AddType image/svg+xml svg svgz 161 | AddEncoding gzip svgz 162 | 163 | # Other 164 | AddType application/octet-stream safariextz 165 | AddType application/x-chrome-extension crx 166 | AddType application/x-opera-extension oex 167 | AddType application/x-shockwave-flash swf 168 | AddType application/x-web-app-manifest+json webapp 169 | AddType application/x-xpinstall xpi 170 | AddType application/xml atom rdf rss xml 171 | AddType image/webp webp 172 | AddType image/x-icon ico 173 | AddType text/cache-manifest appcache manifest 174 | AddType text/vtt vtt 175 | AddType text/x-component htc 176 | AddType text/x-vcard vcf 177 | 178 | 179 | 180 | # ------------------------------------------------------------------------------ 181 | # | UTF-8 encoding | 182 | # ------------------------------------------------------------------------------ 183 | 184 | # Use UTF-8 encoding for anything served as `text/html` or `text/plain`. 185 | AddDefaultCharset utf-8 186 | 187 | # Force UTF-8 for certain file formats. 188 | 189 | AddCharset utf-8 .atom .css .js .json .rss .vtt .webapp .xml 190 | 191 | 192 | 193 | # ############################################################################## 194 | # # URL REWRITES # 195 | # ############################################################################## 196 | 197 | # ------------------------------------------------------------------------------ 198 | # | Rewrite engine | 199 | # ------------------------------------------------------------------------------ 200 | 201 | # Turning on the rewrite engine and enabling the `FollowSymLinks` option is 202 | # necessary for the following directives to work. 203 | 204 | # If your web host doesn't allow the `FollowSymlinks` option, you may need to 205 | # comment it out and use `Options +SymLinksIfOwnerMatch` but, be aware of the 206 | # performance impact: http://httpd.apache.org/docs/current/misc/perf-tuning.html#symlinks 207 | 208 | # Also, some cloud hosting services require `RewriteBase` to be set: 209 | # http://www.rackspace.com/knowledge_center/frequently-asked-question/why-is-mod-rewrite-not-working-on-my-site 210 | 211 | 212 | Options +FollowSymlinks 213 | # Options +SymLinksIfOwnerMatch 214 | RewriteEngine On 215 | # RewriteBase / 216 | 217 | 218 | # ------------------------------------------------------------------------------ 219 | # | Suppressing / Forcing the "www." at the beginning of URLs | 220 | # ------------------------------------------------------------------------------ 221 | 222 | # The same content should never be available under two different URLs especially 223 | # not with and without "www." at the beginning. This can cause SEO problems 224 | # (duplicate content), therefore, you should choose one of the alternatives and 225 | # redirect the other one. 226 | 227 | # By default option 1 (no "www.") is activated: 228 | # http://no-www.org/faq.php?q=class_b 229 | 230 | # If you'd prefer to use option 2, just comment out all the lines from option 1 231 | # and uncomment the ones from option 2. 232 | 233 | # IMPORTANT: NEVER USE BOTH RULES AT THE SAME TIME! 234 | 235 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 236 | 237 | # Option 1: rewrite www.example.com → example.com 238 | 239 | 240 | RewriteCond %{HTTPS} !=on 241 | RewriteCond %{HTTP_HOST} ^www\.(.+)$ [NC] 242 | RewriteRule ^ http://%1%{REQUEST_URI} [R=301,L] 243 | 244 | 245 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 246 | 247 | # Option 2: rewrite example.com → www.example.com 248 | 249 | # Be aware that the following might not be a good idea if you use "real" 250 | # subdomains for certain parts of your website. 251 | 252 | # 253 | # RewriteCond %{HTTPS} !=on 254 | # RewriteCond %{HTTP_HOST} !^www\..+$ [NC] 255 | # RewriteRule ^ http://www.%{HTTP_HOST}%{REQUEST_URI} [R=301,L] 256 | # 257 | 258 | 259 | # ############################################################################## 260 | # # SECURITY # 261 | # ############################################################################## 262 | 263 | # ------------------------------------------------------------------------------ 264 | # | Content Security Policy (CSP) | 265 | # ------------------------------------------------------------------------------ 266 | 267 | # You can mitigate the risk of cross-site scripting and other content-injection 268 | # attacks by setting a Content Security Policy which whitelists trusted sources 269 | # of content for your site. 270 | 271 | # The example header below allows ONLY scripts that are loaded from the current 272 | # site's origin (no inline scripts, no CDN, etc). This almost certainly won't 273 | # work as-is for your site! 274 | 275 | # To get all the details you'll need to craft a reasonable policy for your site, 276 | # read: http://html5rocks.com/en/tutorials/security/content-security-policy (or 277 | # see the specification: http://w3.org/TR/CSP). 278 | 279 | # 280 | # Header set Content-Security-Policy "script-src 'self'; object-src 'self'" 281 | # 282 | # Header unset Content-Security-Policy 283 | # 284 | # 285 | 286 | # ------------------------------------------------------------------------------ 287 | # | File access | 288 | # ------------------------------------------------------------------------------ 289 | 290 | # Block access to directories without a default document. 291 | # Usually you should leave this uncommented because you shouldn't allow anyone 292 | # to surf through every directory on your server (which may includes rather 293 | # private places like the CMS's directories). 294 | 295 | 296 | Options -Indexes 297 | 298 | 299 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 300 | 301 | # Block access to hidden files and directories. 302 | # This includes directories used by version control systems such as Git and SVN. 303 | 304 | 305 | RewriteCond %{SCRIPT_FILENAME} -d [OR] 306 | RewriteCond %{SCRIPT_FILENAME} -f 307 | RewriteRule "(^|/)\." - [F] 308 | 309 | 310 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 311 | 312 | # Block access to backup and source files. 313 | # These files may be left by some text editors and can pose a great security 314 | # danger when anyone has access to them. 315 | 316 | 317 | Order allow,deny 318 | Deny from all 319 | Satisfy All 320 | 321 | 322 | # ------------------------------------------------------------------------------ 323 | # | Secure Sockets Layer (SSL) | 324 | # ------------------------------------------------------------------------------ 325 | 326 | # Rewrite secure requests properly to prevent SSL certificate warnings, e.g.: 327 | # prevent `https://www.example.com` when your certificate only allows 328 | # `https://secure.example.com`. 329 | 330 | # 331 | # RewriteCond %{SERVER_PORT} !^443 332 | # RewriteRule ^ https://example-domain-please-change-me.com%{REQUEST_URI} [R=301,L] 333 | # 334 | 335 | # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 336 | 337 | # Force client-side SSL redirection. 338 | 339 | # If a user types "example.com" in his browser, the above rule will redirect him 340 | # to the secure version of the site. That still leaves a window of opportunity 341 | # (the initial HTTP connection) for an attacker to downgrade or redirect the 342 | # request. The following header ensures that browser will ONLY connect to your 343 | # server via HTTPS, regardless of what the users type in the address bar. 344 | # http://www.html5rocks.com/en/tutorials/security/transport-layer-security/ 345 | 346 | # 347 | # Header set Strict-Transport-Security max-age=16070400; 348 | # 349 | 350 | # ------------------------------------------------------------------------------ 351 | # | Server software information | 352 | # ------------------------------------------------------------------------------ 353 | 354 | # Avoid displaying the exact Apache version number, the description of the 355 | # generic OS-type and the information about Apache's compiled-in modules. 356 | 357 | # ADD THIS DIRECTIVE IN THE `httpd.conf` AS IT WILL NOT WORK IN THE `.htaccess`! 358 | 359 | # ServerTokens Prod 360 | 361 | 362 | # ############################################################################## 363 | # # WEB PERFORMANCE # 364 | # ############################################################################## 365 | 366 | # ------------------------------------------------------------------------------ 367 | # | Compression | 368 | # ------------------------------------------------------------------------------ 369 | 370 | 371 | 372 | # Force compression for mangled headers. 373 | # http://developer.yahoo.com/blogs/ydn/posts/2010/12/pushing-beyond-gzipping 374 | 375 | 376 | SetEnvIfNoCase ^(Accept-EncodXng|X-cept-Encoding|X{15}|~{15}|-{15})$ ^((gzip|deflate)\s*,?\s*)+|[X~-]{4,13}$ HAVE_Accept-Encoding 377 | RequestHeader append Accept-Encoding "gzip,deflate" env=HAVE_Accept-Encoding 378 | 379 | 380 | 381 | # Compress all output labeled with one of the following MIME-types 382 | # (for Apache versions below 2.3.7, you don't need to enable `mod_filter` 383 | # and can remove the `` and `` lines 384 | # as `AddOutputFilterByType` is still in the core directives). 385 | 386 | AddOutputFilterByType DEFLATE application/atom+xml \ 387 | application/javascript \ 388 | application/json \ 389 | application/rss+xml \ 390 | application/vnd.ms-fontobject \ 391 | application/x-font-ttf \ 392 | application/x-web-app-manifest+json \ 393 | application/xhtml+xml \ 394 | application/xml \ 395 | font/opentype \ 396 | image/svg+xml \ 397 | image/x-icon \ 398 | text/css \ 399 | text/html \ 400 | text/plain \ 401 | text/x-component \ 402 | text/xml 403 | 404 | 405 | 406 | 407 | # ------------------------------------------------------------------------------ 408 | # | Content transformations | 409 | # ------------------------------------------------------------------------------ 410 | 411 | # Prevent some of the mobile network providers from modifying the content of 412 | # your site: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.5. 413 | 414 | # 415 | # Header set Cache-Control "no-transform" 416 | # 417 | 418 | # ------------------------------------------------------------------------------ 419 | # | ETag removal | 420 | # ------------------------------------------------------------------------------ 421 | 422 | # Since we're sending far-future expires headers (see below), ETags can 423 | # be removed: http://developer.yahoo.com/performance/rules.html#etags. 424 | 425 | # `FileETag None` is not enough for every server. 426 | 427 | Header unset ETag 428 | 429 | 430 | FileETag None 431 | 432 | # ------------------------------------------------------------------------------ 433 | # | Expires headers (for better cache control) | 434 | # ------------------------------------------------------------------------------ 435 | 436 | # The following expires headers are set pretty far in the future. If you don't 437 | # control versioning with filename-based cache busting, consider lowering the 438 | # cache time for resources like CSS and JS to something like 1 week. 439 | 440 | 441 | 442 | ExpiresActive on 443 | ExpiresDefault "access plus 1 month" 444 | 445 | # CSS 446 | ExpiresByType text/css "access plus 1 year" 447 | 448 | # Data interchange 449 | ExpiresByType application/json "access plus 0 seconds" 450 | ExpiresByType application/xml "access plus 0 seconds" 451 | ExpiresByType text/xml "access plus 0 seconds" 452 | 453 | # Favicon (cannot be renamed!) 454 | ExpiresByType image/x-icon "access plus 1 week" 455 | 456 | # HTML components (HTCs) 457 | ExpiresByType text/x-component "access plus 1 month" 458 | 459 | # HTML 460 | ExpiresByType text/html "access plus 0 seconds" 461 | 462 | # JavaScript 463 | ExpiresByType application/javascript "access plus 1 year" 464 | 465 | # Manifest files 466 | ExpiresByType application/x-web-app-manifest+json "access plus 0 seconds" 467 | ExpiresByType text/cache-manifest "access plus 0 seconds" 468 | 469 | # Media 470 | ExpiresByType audio/ogg "access plus 1 month" 471 | ExpiresByType image/gif "access plus 1 month" 472 | ExpiresByType image/jpeg "access plus 1 month" 473 | ExpiresByType image/png "access plus 1 month" 474 | ExpiresByType video/mp4 "access plus 1 month" 475 | ExpiresByType video/ogg "access plus 1 month" 476 | ExpiresByType video/webm "access plus 1 month" 477 | 478 | # Web feeds 479 | ExpiresByType application/atom+xml "access plus 1 hour" 480 | ExpiresByType application/rss+xml "access plus 1 hour" 481 | 482 | # Web fonts 483 | ExpiresByType application/font-woff "access plus 1 month" 484 | ExpiresByType application/vnd.ms-fontobject "access plus 1 month" 485 | ExpiresByType application/x-font-ttf "access plus 1 month" 486 | ExpiresByType font/opentype "access plus 1 month" 487 | ExpiresByType image/svg+xml "access plus 1 month" 488 | 489 | 490 | 491 | # ------------------------------------------------------------------------------ 492 | # | Filename-based cache busting | 493 | # ------------------------------------------------------------------------------ 494 | 495 | # If you're not using a build process to manage your filename version revving, 496 | # you might want to consider enabling the following directives to route all 497 | # requests such as `/css/style.12345.css` to `/css/style.css`. 498 | 499 | # To understand why this is important and a better idea than `*.css?v231`, read: 500 | # http://stevesouders.com/blog/2008/08/23/revving-filenames-dont-use-querystring 501 | 502 | # 503 | # RewriteCond %{REQUEST_FILENAME} !-f 504 | # RewriteCond %{REQUEST_FILENAME} !-d 505 | # RewriteRule ^(.+)\.(\d+)\.(js|css|png|jpg|gif)$ $1.$3 [L] 506 | # 507 | 508 | # ------------------------------------------------------------------------------ 509 | # | File concatenation | 510 | # ------------------------------------------------------------------------------ 511 | 512 | # Allow concatenation from within specific CSS and JS files, e.g.: 513 | # Inside of `script.combined.js` you could have 514 | # 515 | # 516 | # and they would be included into this single file. 517 | 518 | # 519 | # 520 | # Options +Includes 521 | # AddOutputFilterByType INCLUDES application/javascript application/json 522 | # SetOutputFilter INCLUDES 523 | # 524 | # 525 | # Options +Includes 526 | # AddOutputFilterByType INCLUDES text/css 527 | # SetOutputFilter INCLUDES 528 | # 529 | # 530 | 531 | # ------------------------------------------------------------------------------ 532 | # | Persistent connections | 533 | # ------------------------------------------------------------------------------ 534 | 535 | # Allow multiple requests to be sent over the same TCP connection: 536 | # http://httpd.apache.org/docs/current/en/mod/core.html#keepalive. 537 | 538 | # Enable if you serve a lot of static content but, be aware of the 539 | # possible disadvantages! 540 | 541 | # 542 | # Header set Connection Keep-Alive 543 | # 544 | -------------------------------------------------------------------------------- /dashboard_ui/app/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Page Not Found :( 6 | 141 | 142 | 143 |
144 |

Not found :(

145 |

Sorry, but the page you were trying to view does not exist.

146 |

It looks like this was the result of either:

147 | 151 | 154 | 155 |
156 | 157 | 158 | -------------------------------------------------------------------------------- /dashboard_ui/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard_ui/app/favicon.ico -------------------------------------------------------------------------------- /dashboard_ui/app/images/yeoman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dashboard_ui/app/images/yeoman.png -------------------------------------------------------------------------------- /dashboard_ui/app/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lib Spark 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 22 | 23 | 24 |
25 |
26 | 29 |

Django Lib Spark API Explorer

30 |
31 | 32 |
33 | 34 |
35 |

from the Lib Spark Team

36 |
37 |
38 | 39 | 40 | 41 | 50 | 51 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /dashboard_ui/app/robots.txt: -------------------------------------------------------------------------------- 1 | # robotstxt.org 2 | 3 | User-agent: * 4 | -------------------------------------------------------------------------------- /dashboard_ui/app/scripts/app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @ngdoc overview 5 | * @name dashboardUiApp 6 | * @description 7 | * # dashboardUiApp 8 | * 9 | * Main module of the application. 10 | */ 11 | angular 12 | .module('dashboardUiApp', [ 13 | 'ngAnimate', 14 | 'ngCookies', 15 | 'ngResource', 16 | 'ngRoute', 17 | 'ngSanitize', 18 | 'ngTouch' 19 | ]) 20 | .config(function ($routeProvider) { 21 | $routeProvider 22 | .when('/', { 23 | templateUrl: 'views/main.html', 24 | controller: 'MainCtrl' 25 | }) 26 | .when('/about', { 27 | templateUrl: 'views/about.html', 28 | controller: 'AboutCtrl' 29 | }) 30 | .when('/dashboard', { 31 | templateUrl: 'views/dashboard.html', 32 | controller: 'DashboardCtrl' 33 | }) 34 | .otherwise({ 35 | redirectTo: '/' 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /dashboard_ui/app/scripts/controllers/about.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @ngdoc function 5 | * @name dashboardUiApp.controller:AboutCtrl 6 | * @description 7 | * # AboutCtrl 8 | * Controller of the dashboardUiApp 9 | */ 10 | angular.module('dashboardUiApp') 11 | .controller('AboutCtrl', function ($scope) { 12 | $scope.awesomeThings = [ 13 | 'HTML5 Boilerplate', 14 | 'AngularJS', 15 | 'Karma' 16 | ]; 17 | }); 18 | -------------------------------------------------------------------------------- /dashboard_ui/app/scripts/controllers/dashboard.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @ngdoc function 5 | * @name dashboardUiApp.controller:DashboardCtrl 6 | * @description 7 | * # DashboardCtrl 8 | * Controller of the dashboardUiApp 9 | */ 10 | angular.module('dashboardUiApp') 11 | .controller('DashboardCtrl', function ($scope,request) { 12 | $scope.methodOneChecked = []; 13 | $scope.endMethodChecked = null; 14 | $scope.methodTypeOne = [ 15 | { 16 | 'name': 'map', 17 | 'func' : true, 18 | 'checked': false 19 | }, 20 | 21 | { 22 | 'name': 'distinct', 23 | 'func' : false, 24 | 'checked': false 25 | }, 26 | { 27 | 'name': 'filter', 28 | 'func' : true, 29 | 'checked': false 30 | }, 31 | { 32 | 'name': 'flatMap', 33 | 'func' : true, 34 | 'checked': false 35 | }, 36 | { 37 | 'name': 'flatMapValues', 38 | 'func' : true, 39 | 'checked': false 40 | }, 41 | { 42 | 'name': 'groupBy', 43 | 'func' : true, 44 | 'checked': false 45 | }, 46 | { 47 | 'name': 'groupByKey', 48 | 'func' : false, 49 | 'checked': false 50 | }, 51 | { 52 | 'name': 'intersection', 53 | 'func' : false, 54 | 'checked': false 55 | }, 56 | { 57 | 'name': 'join', 58 | 'func' : false, 59 | 'checked': false 60 | }, 61 | { 62 | 'name': 'keyBy', 63 | 'func' : true, 64 | 'checked': false 65 | }, 66 | { 67 | 'name': 'keys', 68 | 'func' : false, 69 | 'checked': false 70 | }, 71 | { 72 | 'name': 'leftOuterJoin', 73 | 'func' : false, 74 | 'checked': false 75 | }, 76 | { 77 | 'name': 'mapPartitions', 78 | 'func' : true, 79 | 'checked': false 80 | }, 81 | { 82 | 'name': 'mapPartitionsWithIndex', 83 | 'func' : true, 84 | 'checked': false 85 | }, 86 | { 87 | 'name': 'mapPartitionsWithSplit', 88 | 'func' : true, 89 | 'checked': false 90 | }, 91 | { 92 | 'name': 'mapValues', 93 | 'func' : true, 94 | 'checked': false 95 | }, 96 | { 97 | 'name': 'reduceByKey', 98 | 'func' : true, 99 | 'checked': false 100 | }, 101 | { 102 | 'name': 'reduceByKeyLocally', 103 | 'func' : true, 104 | 'checked': false 105 | }, 106 | { 107 | 'name': 'rightOuterJoin', 108 | 'func' : true, 109 | 'checked': false 110 | }, 111 | { 112 | 'name': 'sortBy', 113 | 'func' : true, 114 | 'checked': false 115 | }, 116 | { 117 | 'name': 'sortByKey', 118 | 'func' : true, 119 | 'checked': false 120 | }, 121 | { 122 | 'name': 'substract', 123 | 'func' : false, 124 | 'checked': false 125 | }, 126 | { 127 | 'name': 'union', 128 | 'func' : true, 129 | 'checked': false 130 | }, 131 | { 132 | 'name': 'values', 133 | 'func' : true, 134 | 'checked': false 135 | }, 136 | { 137 | 'name': 'zip', 138 | 'func' : true, 139 | 'checked': false 140 | }, 141 | { 142 | 'name': 'zipWithIndex', 143 | 'func' : true, 144 | 'checked': false 145 | }, 146 | { 147 | 'name': 'zipWithUniqueId', 148 | 'func' : true, 149 | 'checked': false 150 | }, 151 | 152 | 153 | ]; 154 | 155 | 156 | $scope.endMethods = [ 157 | { 158 | 'name': 'first', 159 | 'func' : false, 160 | 'checked': false 161 | }, 162 | { 163 | 'name': 'collect', 164 | 'func' : false, 165 | 'checked': false 166 | }, 167 | { 168 | 'name': 'cache', 169 | 'func' : false, 170 | 'checked': false 171 | }, 172 | { 173 | 'name': 'count', 174 | 'func' : false, 175 | 'checked': false 176 | }, 177 | { 178 | 'name': 'max', 179 | 'func' : false, 180 | 'checked': false 181 | }, 182 | { 183 | 'name': 'mean', 184 | 'func' : false, 185 | 'checked': false 186 | }, 187 | { 188 | 'name': 'min', 189 | 'func' : false, 190 | 'checked': false 191 | }, 192 | { 193 | 'name': 'name', 194 | 'func' : false, 195 | 'checked': false 196 | }, 197 | { 198 | 'name': 'sum', 199 | 'func' : false, 200 | 'checked': false 201 | }, 202 | { 203 | 'name': 'take', 204 | 'func' : false, 205 | 'checked': false 206 | }, 207 | { 208 | 'name': 'takeOrdered', 209 | 'func' : false, 210 | 'checked': false 211 | }, 212 | { 213 | 'name': 'top', 214 | 'func' : false, 215 | 'checked': false 216 | }, 217 | { 218 | 'name': 'variance', 219 | 'func' : false, 220 | 'checked': false 221 | }, 222 | 223 | 224 | ]; 225 | function chekedMethodOneList () { 226 | $scope.methodOneChecked=[]; 227 | $scope.funcList = []; 228 | for (var i = 0; i < $scope.methodTypeOne.length; i++) { 229 | if($scope.methodTypeOne[i].checked){ 230 | $scope.methodOneChecked.push($scope.methodTypeOne[i]) 231 | if($scope.methodTypeOne[i].func){ 232 | $scope.funcList.push($scope.methodTypeOne[i]); 233 | } 234 | } 235 | }; 236 | } 237 | $scope.checkMethodOne = function (id) { 238 | // $scope.methodTypeOne.checked=false 239 | chekedMethodOneList() 240 | }; 241 | $scope.SendData = function () { 242 | var data = []; 243 | data[0] = $scope.methodOneChecked; 244 | data[1] = $scope.endMethodChecked; 245 | data[3] = $scope.data; 246 | request.SendData(data).then(function (res) { 247 | $scope.output = res.success 248 | }) 249 | } 250 | }); 251 | -------------------------------------------------------------------------------- /dashboard_ui/app/scripts/controllers/main.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @ngdoc function 5 | * @name dashboardUiApp.controller:MainCtrl 6 | * @description 7 | * # MainCtrl 8 | * Controller of the dashboardUiApp 9 | */ 10 | angular.module('dashboardUiApp') 11 | .controller('MainCtrl', function ($scope) { 12 | $scope.awesomeThings = [ 13 | 'HTML5 Boilerplate', 14 | 'AngularJS', 15 | 'Karma' 16 | ]; 17 | }); 18 | -------------------------------------------------------------------------------- /dashboard_ui/app/scripts/services/request.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * @ngdoc service 5 | * @name dashboardUiApp.request 6 | * @description 7 | * # request 8 | * Factory in the dashboardUiApp. 9 | */ 10 | angular.module('dashboardUiApp') 11 | .factory('request', function ($http) { 12 | var services = { 13 | SendData: function (data) { 14 | return $http.post('http://localhost:8000/',data).then(function (response) { 15 | return response.data; 16 | }); 17 | } 18 | } 19 | 20 | // Public API here 21 | return services; 22 | 23 | }); 24 | -------------------------------------------------------------------------------- /dashboard_ui/app/styles/main.css: -------------------------------------------------------------------------------- 1 | /* Space out content a bit */ 2 | body { 3 | padding-top: 20px; 4 | padding-bottom: 20px; 5 | } 6 | 7 | /* Everything but the jumbotron gets side spacing for mobile first views */ 8 | .header, 9 | .marketing, 10 | .footer { 11 | padding-left: 15px; 12 | padding-right: 15px; 13 | } 14 | 15 | /* Custom page header */ 16 | .header { 17 | border-bottom: 1px solid #e5e5e5; 18 | } 19 | /* Make the masthead heading the same height as the navigation */ 20 | .header h3 { 21 | margin-top: 0; 22 | margin-bottom: 0; 23 | line-height: 40px; 24 | padding-bottom: 19px; 25 | } 26 | 27 | /* Custom page footer */ 28 | .footer { 29 | padding-top: 19px; 30 | color: #777; 31 | border-top: 1px solid #e5e5e5; 32 | } 33 | 34 | 35 | .container-narrow > hr { 36 | margin: 30px 0; 37 | } 38 | 39 | /* Main marketing message and sign up button */ 40 | .jumbotron { 41 | text-align: center; 42 | border-bottom: 1px solid #e5e5e5; 43 | } 44 | .jumbotron .btn { 45 | font-size: 21px; 46 | padding: 14px 24px; 47 | } 48 | 49 | /* Supporting marketing content */ 50 | .marketing { 51 | margin: 40px 0; 52 | } 53 | .marketing p + h4 { 54 | margin-top: 28px; 55 | } 56 | 57 | /* Responsive: Portrait tablets and up */ 58 | @media screen and (min-width: 768px) { 59 | /* Remove the padding we set earlier */ 60 | .header, 61 | .marketing, 62 | .footer { 63 | padding-left: 0; 64 | padding-right: 0; 65 | } 66 | /* Space out the masthead */ 67 | .header { 68 | margin-bottom: 30px; 69 | } 70 | /* Remove the bottom border on the jumbotron for visual effect */ 71 | .jumbotron { 72 | border-bottom: 0; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /dashboard_ui/app/views/about.html: -------------------------------------------------------------------------------- 1 |

This is the about view.

2 | -------------------------------------------------------------------------------- /dashboard_ui/app/views/dashboard.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Generic Function 4 |
5 |
6 |
7 | Lazy Evaluators 8 |
9 |
10 |
11 | lambda inputs 12 |
13 |
14 |
15 | Dataset 16 | 17 |

18 | 19 | Output 20 | Output from Spark Instance:
21 | {{output}} 22 |
23 |
24 | -------------------------------------------------------------------------------- /dashboard_ui/app/views/main.html: -------------------------------------------------------------------------------- 1 |
2 |

'Allo, 'Allo!

3 |

4 | I'm Yeoman
5 | Always a pleasure scaffolding your apps. 6 |

7 |

Splendid!

8 |
9 | 10 |
11 |

HTML5 Boilerplate

12 |

13 | HTML5 Boilerplate is a professional front-end template for building fast, robust, and adaptable web apps or sites. 14 |

15 | 16 |

Angular

17 |

18 | AngularJS is a toolset for building the framework most suited to your application development. 19 |

20 | 21 |

Karma

22 |

Spectacular Test Runner for JavaScript.

23 |
24 | -------------------------------------------------------------------------------- /dashboard_ui/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dashboard-ui", 3 | "version": "0.0.0", 4 | "dependencies": { 5 | "angular": "1.2.16", 6 | "json3": "~3.3.1", 7 | "es5-shim": "~3.1.0", 8 | "bootstrap": "~3.1.1", 9 | "angular-resource": "1.2.16", 10 | "angular-cookies": "1.2.16", 11 | "angular-sanitize": "1.2.16", 12 | "angular-animate": "1.2.16", 13 | "angular-touch": "1.2.16", 14 | "angular-route": "1.2.16" 15 | }, 16 | "devDependencies": { 17 | "angular-mocks": "1.2.16", 18 | "angular-scenario": "1.2.16" 19 | }, 20 | "appPath": "app" 21 | } 22 | -------------------------------------------------------------------------------- /dashboard_ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dashboardui", 3 | "version": "0.0.0", 4 | "dependencies": {}, 5 | "devDependencies": { 6 | "grunt": "^0.4.1", 7 | "grunt-autoprefixer": "^0.7.3", 8 | "grunt-concurrent": "^0.5.0", 9 | "grunt-contrib-clean": "^0.5.0", 10 | "grunt-contrib-concat": "^0.4.0", 11 | "grunt-contrib-connect": "^0.7.1", 12 | "grunt-contrib-copy": "^0.5.0", 13 | "grunt-contrib-cssmin": "^0.9.0", 14 | "grunt-contrib-htmlmin": "^0.3.0", 15 | "grunt-contrib-imagemin": "^0.7.0", 16 | "grunt-contrib-jshint": "^0.10.0", 17 | "grunt-contrib-uglify": "^0.4.0", 18 | "grunt-contrib-watch": "^0.6.1", 19 | "grunt-filerev": "^0.2.1", 20 | "grunt-google-cdn": "^0.4.0", 21 | "grunt-karma": "^0.12.0", 22 | "grunt-newer": "^0.7.0", 23 | "grunt-ngmin": "^0.0.3", 24 | "grunt-svgmin": "^0.4.0", 25 | "grunt-usemin": "^2.1.1", 26 | "grunt-wiredep": "1.8.0", 27 | "jasmine-core": "^2.3.4", 28 | "jshint-stylish": "^0.2.0", 29 | "karma": "^0.13.9", 30 | "karma-jasmine": "^0.3.6", 31 | "karma-phantomjs-launcher": "^0.2.1", 32 | "load-grunt-tasks": "^0.4.0", 33 | "phantomjs": "^1.9.18", 34 | "time-grunt": "^0.3.1" 35 | }, 36 | "engines": { 37 | "node": ">=0.10.0" 38 | }, 39 | "scripts": { 40 | "test": "grunt test" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dashboard_ui/test/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "browser": true, 4 | "esnext": true, 5 | "bitwise": true, 6 | "camelcase": true, 7 | "curly": true, 8 | "eqeqeq": true, 9 | "immed": true, 10 | "indent": 2, 11 | "latedef": true, 12 | "newcap": true, 13 | "noarg": true, 14 | "quotmark": "single", 15 | "regexp": true, 16 | "undef": true, 17 | "unused": true, 18 | "strict": true, 19 | "trailing": true, 20 | "smarttabs": true, 21 | "globals": { 22 | "after": false, 23 | "afterEach": false, 24 | "angular": false, 25 | "before": false, 26 | "beforeEach": false, 27 | "browser": false, 28 | "describe": false, 29 | "expect": false, 30 | "inject": false, 31 | "it": false, 32 | "jasmine": false, 33 | "spyOn": false 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /dashboard_ui/test/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration 2 | // http://karma-runner.github.io/0.12/config/configuration-file.html 3 | // Generated on 2015-08-23 using 4 | // generator-karma 0.8.3 5 | 6 | module.exports = function(config) { 7 | 'use strict'; 8 | 9 | config.set({ 10 | // enable / disable watching file and executing tests whenever any file changes 11 | autoWatch: true, 12 | 13 | // base path, that will be used to resolve files and exclude 14 | basePath: '../', 15 | 16 | // testing framework to use (jasmine/mocha/qunit/...) 17 | frameworks: ['jasmine'], 18 | 19 | // list of files / patterns to load in the browser 20 | files: [ 21 | 'bower_components/angular/angular.js', 22 | 'bower_components/angular-mocks/angular-mocks.js', 23 | 'bower_components/angular-animate/angular-animate.js', 24 | 'bower_components/angular-cookies/angular-cookies.js', 25 | 'bower_components/angular-resource/angular-resource.js', 26 | 'bower_components/angular-route/angular-route.js', 27 | 'bower_components/angular-sanitize/angular-sanitize.js', 28 | 'bower_components/angular-touch/angular-touch.js', 29 | 'app/scripts/**/*.js', 30 | 'test/mock/**/*.js', 31 | 'test/spec/**/*.js' 32 | ], 33 | 34 | // list of files / patterns to exclude 35 | exclude: [], 36 | 37 | // web server port 38 | port: 8080, 39 | 40 | // Start these browsers, currently available: 41 | // - Chrome 42 | // - ChromeCanary 43 | // - Firefox 44 | // - Opera 45 | // - Safari (only Mac) 46 | // - PhantomJS 47 | // - IE (only Windows) 48 | browsers: [ 49 | 'PhantomJS' 50 | ], 51 | 52 | // Which plugins to enable 53 | plugins: [ 54 | 'karma-phantomjs-launcher', 55 | 'karma-jasmine' 56 | ], 57 | 58 | // Continuous Integration mode 59 | // if true, it capture browsers, run tests and exit 60 | singleRun: false, 61 | 62 | colors: true, 63 | 64 | // level of logging 65 | // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG 66 | logLevel: config.LOG_INFO, 67 | 68 | // Uncomment the following lines if you are using grunt's server to run the tests 69 | // proxies: { 70 | // '/': 'http://localhost:9000/' 71 | // }, 72 | // URL root prevent conflicts with the site root 73 | // urlRoot: '_karma_' 74 | }); 75 | }; 76 | -------------------------------------------------------------------------------- /dashboard_ui/test/spec/controllers/about.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: AboutCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('dashboardUiApp')); 7 | 8 | var AboutCtrl, 9 | scope; 10 | 11 | // Initialize the controller and a mock scope 12 | beforeEach(inject(function ($controller, $rootScope) { 13 | scope = $rootScope.$new(); 14 | AboutCtrl = $controller('AboutCtrl', { 15 | $scope: scope 16 | }); 17 | })); 18 | 19 | it('should attach a list of awesomeThings to the scope', function () { 20 | expect(scope.awesomeThings.length).toBe(3); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /dashboard_ui/test/spec/controllers/dashboard.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: DashboardCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('dashboardUiApp')); 7 | 8 | var DashboardCtrl, 9 | scope; 10 | 11 | // Initialize the controller and a mock scope 12 | beforeEach(inject(function ($controller, $rootScope) { 13 | scope = $rootScope.$new(); 14 | DashboardCtrl = $controller('DashboardCtrl', { 15 | $scope: scope 16 | }); 17 | })); 18 | 19 | it('should attach a list of awesomeThings to the scope', function () { 20 | expect(scope.awesomeThings.length).toBe(3); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /dashboard_ui/test/spec/controllers/main.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: MainCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('dashboardUiApp')); 7 | 8 | var MainCtrl, 9 | scope; 10 | 11 | // Initialize the controller and a mock scope 12 | beforeEach(inject(function ($controller, $rootScope) { 13 | scope = $rootScope.$new(); 14 | MainCtrl = $controller('MainCtrl', { 15 | $scope: scope 16 | }); 17 | })); 18 | 19 | it('should attach a list of awesomeThings to the scope', function () { 20 | expect(scope.awesomeThings.length).toBe(3); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /dashboard_ui/test/spec/services/request.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Service: request', function () { 4 | 5 | // load the service's module 6 | beforeEach(module('dashboardUiApp')); 7 | 8 | // instantiate service 9 | var request; 10 | beforeEach(inject(function (_request_) { 11 | request = _request_; 12 | })); 13 | 14 | it('should do something', function () { 15 | expect(!!request).toBe(true); 16 | }); 17 | 18 | }); 19 | -------------------------------------------------------------------------------- /dump/foodRevPy-dev/PaginationColl.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dump/foodRevPy-dev/PaginationColl.bson -------------------------------------------------------------------------------- /dump/foodRevPy-dev/PaginationColl.metadata.json: -------------------------------------------------------------------------------- 1 | {"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"foodRevPy-dev.PaginationColl"}]} -------------------------------------------------------------------------------- /dump/foodRevPy-dev/ReviewColl.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/dump/foodRevPy-dev/ReviewColl.bson -------------------------------------------------------------------------------- /dump/foodRevPy-dev/ReviewColl.metadata.json: -------------------------------------------------------------------------------- 1 | {"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"foodRevPy-dev.ReviewColl"}]} -------------------------------------------------------------------------------- /dump/foodRevPy-dev/system.indexes.bson: -------------------------------------------------------------------------------- 1 | Ovkey_idname_id_nsfoodRevPy-dev.ReviewCollSvkey_idname_id_nsfoodRevPy-dev.PaginationColl -------------------------------------------------------------------------------- /examples/Foodista/Foodista/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/Foodista/__init__.py -------------------------------------------------------------------------------- /examples/Foodista/Foodista/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for Foodista project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.8.4. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.8/ref/settings/ 11 | """ 12 | 13 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 14 | import os 15 | 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'q_r#=0cptxorckxafmrb+r9io9#+i86g5ig*ltp^+e)!lyoj9%' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = ( 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | #Include the LibSpark 41 | 'libspark', 42 | #Review App 43 | 'review', 44 | ) 45 | 46 | MIDDLEWARE_CLASSES = ( 47 | 'django.contrib.sessions.middleware.SessionMiddleware', 48 | 'django.middleware.common.CommonMiddleware', 49 | 'django.middleware.csrf.CsrfViewMiddleware', 50 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 51 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 52 | 'django.contrib.messages.middleware.MessageMiddleware', 53 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 54 | 'django.middleware.security.SecurityMiddleware', 55 | ) 56 | 57 | ROOT_URLCONF = 'Foodista.urls' 58 | 59 | TEMPLATES = [ 60 | { 61 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 62 | 'DIRS': [], 63 | 'APP_DIRS': True, 64 | 'OPTIONS': { 65 | 'context_processors': [ 66 | 'django.template.context_processors.debug', 67 | 'django.template.context_processors.request', 68 | 'django.contrib.auth.context_processors.auth', 69 | 'django.contrib.messages.context_processors.messages', 70 | ], 71 | }, 72 | }, 73 | ] 74 | 75 | WSGI_APPLICATION = 'Foodista.wsgi.application' 76 | 77 | 78 | # Database 79 | # https://docs.djangoproject.com/en/1.8/ref/settings/#databases 80 | 81 | DATABASES = { 82 | 'default': { 83 | 'ENGINE': 'django.db.backends.sqlite3', 84 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 85 | }, 86 | } 87 | 88 | 89 | # Internationalization 90 | # https://docs.djangoproject.com/en/1.8/topics/i18n/ 91 | 92 | LANGUAGE_CODE = 'en-us' 93 | 94 | TIME_ZONE = 'UTC' 95 | 96 | USE_I18N = True 97 | 98 | USE_L10N = True 99 | 100 | USE_TZ = True 101 | 102 | 103 | # Static files (CSS, JavaScript, Images) 104 | # https://docs.djangoproject.com/en/1.8/howto/static-files/ 105 | 106 | STATIC_URL = '/static/' 107 | -------------------------------------------------------------------------------- /examples/Foodista/Foodista/urls.py: -------------------------------------------------------------------------------- 1 | """Foodista URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.8/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add an import: from blog import urls as blog_urls 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) 15 | """ 16 | from django.conf.urls import include, url 17 | from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | url(r'^admin/', include(admin.site.urls)), 21 | url(r'^findReviews','review.views.find_all_review'), 22 | url(r'^FUW','review.views.frequently_used_words') 23 | ] 24 | -------------------------------------------------------------------------------- /examples/Foodista/Foodista/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for Foodista project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Foodista.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /examples/Foodista/libspark/SparkRDD.py: -------------------------------------------------------------------------------- 1 | # Configure the necessary Spark environment 2 | import os 3 | import sys 4 | 5 | # Spark home 6 | spark_home = os.environ.get("SPARK_HOME") 7 | 8 | # If Spark V1.4.x is detected, then add ' pyspark-shell' to 9 | # the end of the 'PYSPARK_SUBMIT_ARGS' environment variable 10 | spark_release_file = spark_home + "/RELEASE" 11 | if os.path.exists(spark_release_file) and "Spark 1.4" in open(spark_release_file).read(): 12 | pyspark_submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "") 13 | if not "pyspark-shell" in pyspark_submit_args: pyspark_submit_args += " pyspark-shell" 14 | os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args 15 | 16 | # Add the spark python sub-directory to the path 17 | sys.path.insert(0, spark_home + "/python") 18 | 19 | # Add the py4j to the path. 20 | # You may need to change the version number to match your install 21 | sys.path.insert(0, os.path.join(spark_home, "python/lib/py4j-0.8.2.1-src.zip")) 22 | # Initialize PySpark to predefine the SparkContext variable 'sc' 23 | execfile(os.path.join(spark_home, "python/pyspark/shell.py")) 24 | 25 | class SparkRDD(): 26 | 27 | # When the SparkRDD is initialized - data and function 28 | # can be passed as arguments 29 | def __init__(self, data=[], **kwargs): 30 | self.func = kwargs.get("func",None) 31 | self.data = data 32 | self.is_cached = False 33 | try: 34 | self.dataRDD = sc.parallelize (data) 35 | except TypeError: 36 | self.dataRDD = data 37 | 38 | def map(self,func): 39 | mappedData = self.dataRDD.map(func) 40 | tempObj = SparkRDD(data = mappedData) 41 | return tempObj 42 | 43 | def filter(self,func): 44 | filteredData = self.dataRDD.filter(func) 45 | tempObj = SparkRDD(data = filteredData) 46 | return tempObj 47 | 48 | def collect(self): 49 | return self.dataRDD.collect() 50 | 51 | def cache(self): 52 | self.dataRDD.cache() 53 | self.is_cached = True 54 | return self 55 | 56 | def count(self): 57 | return self.dataRDD.count() 58 | 59 | def distinct(self): 60 | distinctRDD = self.dataRDD.distinct() 61 | distinctObj = SparkRDD(data=distinctRDD) 62 | return distinctObj 63 | 64 | def first(self): 65 | return self.dataRDD.first() 66 | 67 | def flatMap(self,func): 68 | flatMapRDD = self.dataRDD.flatMap(func) 69 | tempObj = SparkRDD(data=flatMapRDD) 70 | return tempObj 71 | 72 | def flatMapValues(self,func): 73 | flatMapRDD = self.dataRDD.flatMapValues(func) 74 | tempObj = SparkRDD(data=flatMapRDD) 75 | return tempObj 76 | 77 | def groupBy(self,func): 78 | groupByRDD = self.dataRDD.groupBy(func) 79 | tempObj = SparkRDD(data=groupByRDD) 80 | return tempObj 81 | 82 | def groupByKey(self): 83 | groupByRDD = self.dataRDD.groupByKey() 84 | tempObj = SparkRDD(data=groupByRDD) 85 | return tempObj 86 | 87 | def intersection(self,toBeIntersected): 88 | if isinstance(toBeIntersected,SparkRDD): 89 | intersected = self.dataRDD.intersection(toBeIntersected.dataRDD) 90 | tempObj = SparkRDD(data=intersected) 91 | else: 92 | #TODO: rasie err 93 | pass 94 | return tempObj 95 | 96 | def join(self,toBeJoined): 97 | if isinstance(toBeJoined,SparkRDD): 98 | joined = self.dataRDD.join(toBeJoined.dataRDD) 99 | tempObj = SparkRDD(data=joined) 100 | else: 101 | #TODO: rasie err 102 | pass 103 | return tempObj 104 | 105 | def keyBy(self,func): 106 | keyByRDD = self.dataRDD.keyBy(func) 107 | tempObj = SparkRDD(data=keyByRDD) 108 | return tempObj 109 | 110 | def keys(self): 111 | keysRDD = self.dataRDD.keys() 112 | tempObj = SparkRDD(data=keysRDD) 113 | return tempObj 114 | 115 | def leftOuterJoin(self,toBeJoined): 116 | if isinstance(toBeJoined,SparkRDD): 117 | leftJoined = self.dataRDD.join(toBeJoined.dataRDD) 118 | tempObj = SparkRDD(data=leftJoined) 119 | else: 120 | #TODO: rasie err 121 | pass 122 | return tempObj 123 | 124 | 125 | def lookup(self,key): 126 | valuesList = self.dataRDD.lookup(key) 127 | return valuesList 128 | 129 | def mapPartitions(self,func): 130 | mapPartitionsRDD = self.dataRDD.mapPartitions(func) 131 | tempObj = SparkRDD(data=mapPartitionsRDD) 132 | return tempObj 133 | 134 | def mapPartitionsWithIndex(self,func): 135 | mapPartitionsWithIndexRDD = self.dataRDD.mapPartitionsWithIndex(func) 136 | tempObj = SparkRDD(data=mapPartitionsWithIndexRDD) 137 | return tempObj 138 | 139 | def mapPartitionsWithSplit(self,func): 140 | mapPartitionsWithSplitRDD = self.dataRDD.mapPartitionsWithSplit(func) 141 | tempObj = SparkRDD(data=mapPartitionsWithSplitRDD) 142 | return tempObj 143 | 144 | def mapValues(self,func): 145 | mapValuesRDD = self.dataRDD.mapValues(func) 146 | tempObj = SparkRDD(data=mapValuesRDD) 147 | return tempObj 148 | 149 | def max(self,func=None): 150 | if key in None: 151 | maxRDD = self.dataRDD.max() 152 | else: 153 | maxRDD = self.dataRDD.max(func) 154 | tempObj = SparkRDD(data=maxRDD) 155 | return tempObj 156 | 157 | def mean(self): 158 | meanRDD = self.dataRDD.mean() 159 | tempObj = SparkRDD(data=meanRDD) 160 | return tempObj 161 | 162 | def min(self,func=None): 163 | if func is None: 164 | minRDD = self.dataRDD.min() 165 | else: 166 | minRDD = self.dataRDD.min(func) 167 | tempObj = SparkRDD(data=minRDD) 168 | return tempObj 169 | 170 | def name(self): 171 | return self.dataRDD.name 172 | 173 | def reduceByKey(self,func=None): 174 | if func is None: 175 | reduceByKeyRDD = self.dataRDD.reduceByKey() 176 | else: 177 | reduceByKeyRDD = self.dataRDD.reduceByKey(func) 178 | tempObj = SparkRDD(data=reduceByKeyRDD) 179 | return tempObj 180 | 181 | def reduceByKeyLocally(self,func=None): 182 | if key is None: 183 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally() 184 | else: 185 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally(func) 186 | tempObj = SparkRDD(data=reduceByKeyLocallyRDD) 187 | return tempObj 188 | 189 | 190 | def rightOuterJoin(self,toBeJoined): 191 | if isinstance(toBeJoined,SparkRDD): 192 | rightOuterJoinRDD = self.dataRDD.join(toBeJoined.dataRDD) 193 | tempObj = SparkRDD(data=rightOuterJoinRDD) 194 | else: 195 | #TODO: rasie err 196 | pass 197 | return tempObj 198 | 199 | def sortBy(self,func,ascending=True): 200 | sortedBy = self.dataRDD.sortBy(func) 201 | sortedObj = SparkRDD(data=sortedBy) 202 | return sortedObj 203 | 204 | def sortByKey(self,func,ascending=True): 205 | sortedByKey = self.dataRDD.sortByKey(func) 206 | sortedObj = SparkRDD(data=sortedByKey) 207 | return sortedObj 208 | 209 | def substract(self,toBeSubtracted): 210 | if isinstance(toBeJoined,SparkRDD): 211 | subtracted = self.dataRDD.subtract(toBeSubtracted.dataRDD) 212 | subtractedObj = SparkRDD(data=subtracted) 213 | else: 214 | #TODO: raise err 215 | pass 216 | return subtractedObj 217 | 218 | def sum(self): 219 | summed = self.dataRDD.sum() 220 | sumObj = SparkRDD(data=summed) 221 | return sumObj 222 | 223 | def take(self,num): 224 | took = self.dataRDD.take(num) 225 | tookObj = SparkRDD(data=took) 226 | return tookObj 227 | 228 | def takeOrdered(self,num,func=None): 229 | if func is None: 230 | tookOredered = self.dataRDD.takeOrdered(num) 231 | else: 232 | tookOredered = self.dataRDD.takeOrdered(num,key=func) 233 | return tookOredered 234 | 235 | def top(self,num,func): 236 | if func is None: 237 | topped = self.dataRDD.top(num) 238 | else: 239 | topped = self.dataRDD.top(num,key=func) 240 | return topped 241 | 242 | def union(self): 243 | unioned = self.dataRDD.union() 244 | unionObj = SparkRDD(data=unioned) 245 | return unionObj 246 | 247 | def values(self): 248 | valued = self.dataRDD.values() 249 | valueObj = SparkRDD(data=valued) 250 | return valueObj 251 | 252 | def variance(self): 253 | return self.dataRDD.variance() 254 | 255 | 256 | def zip(self,toBeZipped): 257 | if isinstance(toBeZipped,SparkRDD): 258 | zipped = self.dataRDD.zip(toBeZipped) 259 | else: 260 | #TODO: Raise err 261 | pass 262 | zipObj = SparkRDD(data=zipped) 263 | return zipObj 264 | 265 | def zipWithIndex(self): 266 | zipped = self.dataRDD.zipWithIndex() 267 | zipObj = SparkRDD(data=zipped) 268 | return zipObj 269 | 270 | def zipWithUniqueId(self): 271 | zipped = self.dataRDD.zipWithUniqueId() 272 | zippedObj = SparkRDD(data = zipped) 273 | return zipObj 274 | -------------------------------------------------------------------------------- /examples/Foodista/libspark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/libspark/__init__.py -------------------------------------------------------------------------------- /examples/Foodista/libspark/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /examples/Foodista/libspark/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/libspark/migrations/__init__.py -------------------------------------------------------------------------------- /examples/Foodista/libspark/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /examples/Foodista/libspark/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /examples/Foodista/libspark/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/libspark/views.py -------------------------------------------------------------------------------- /examples/Foodista/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Foodista.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /examples/Foodista/review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/review/__init__.py -------------------------------------------------------------------------------- /examples/Foodista/review/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /examples/Foodista/review/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/examples/Foodista/review/migrations/__init__.py -------------------------------------------------------------------------------- /examples/Foodista/review/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from mongokit import * 3 | import datetime 4 | import string 5 | import json 6 | 7 | connection = Connection() 8 | 9 | @connection.register 10 | class Review(Document): 11 | __collection__ = 'ReviewColl' 12 | __database__ = 'foodRevPy-dev' 13 | structure ={ 14 | 'message': unicode, 15 | 'fbId' : unicode 16 | } 17 | indexes = [ 18 | { 19 | 'fields':['fbId'], 20 | 'unique':True, 21 | } 22 | ] 23 | 24 | @connection.register 25 | class Pagination(Document): 26 | __collection__ = 'PaginationColl' 27 | __database__ = 'foodRevPy-dev' 28 | structure ={ 29 | 'next': unicode, 30 | 'create_time': datetime.datetime, 31 | } 32 | -------------------------------------------------------------------------------- /examples/Foodista/review/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /examples/Foodista/review/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from models import * 3 | from libspark.SparkRDD import SparkRDD 4 | from django.http import JsonResponse 5 | from nltk.corpus import stopwords 6 | cachedStopWords = stopwords.words("english") 7 | rawReviewRDD = None 8 | reviewsWithMessages = None 9 | tokenizedMessages = None 10 | def find(): 11 | reviews = [] 12 | for review in connection.Review.find(): 13 | newReview = { 14 | 'message': review['message'], 15 | 'fbId': review['fbId'] 16 | } 17 | reviews.append(newReview) 18 | global rawReviewRDD 19 | rawReviewRDD= SparkRDD(data=reviews).cache() 20 | return reviews 21 | 22 | def find_all_review(request): 23 | reviews = find() 24 | return JsonResponse(reviews,safe=False) 25 | 26 | def removePunctuations(message): 27 | message = message.encode('utf-8') 28 | return message.translate(None, string.punctuation) 29 | 30 | def find_reviews_with_messages(): 31 | find() 32 | global reviewsWithMessages 33 | global tokenizedMessages 34 | reviewsWithMessages = rawReviewRDD.filter(lambda x: x['message'] != None).map(lambda x: (x['fbId'],x['message'].lower())) 35 | tokenizedMessages = reviewsWithMessages.map(lambda (k,v): (k,removePunctuations(v))).map(lambda (k,v): (k,v.split(' '))).cache() 36 | 37 | def frequently_used_words(request): 38 | find() 39 | reviewsWithMessages = rawReviewRDD.filter(lambda x: x['message'] != None).map(lambda x: (x['fbId'],x['message'].lower())) 40 | tokenizedMessages = reviewsWithMessages.map(lambda (k,v): (k,removePunctuations(v))).map(lambda (k,v): (k,v.split(' '))).cache() 41 | def removeStopWords(tokens): 42 | newTokens = [] 43 | for word in tokens: 44 | if word.lower() not in cachedStopWords: 45 | newTokens.append(word) 46 | return newTokens 47 | 48 | def removeEmpty(tokens): 49 | return [token for token in tokens if token != ''] 50 | 51 | withoutStopWordsRDD = (tokenizedMessages.map(lambda (k,v): (k,removeStopWords(v))) 52 | .map(lambda (k,v): (k,removeEmpty(v))) 53 | .cache()) 54 | messageTokenMap = (withoutStopWordsRDD 55 | .flatMap(lambda (k,v): v) 56 | .map(lambda v: (v,1)) 57 | .reduceByKey(lambda a,b:a+b)) 58 | 59 | topWords = messageTokenMap.takeOrdered(100,lambda (k,v): -v) 60 | frequentWords = [] 61 | for wordCount in topWords: 62 | print '{0:30}{1}' . format(wordCount[0],wordCount[1]) 63 | fw = { 64 | wordCount[0]:wordCount[1] 65 | } 66 | frequentWords.append(fw) 67 | return JsonResponse(frequentWords,safe=False) 68 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | We have built an example application `Foodista` which uses the Django-LibSpark. Foodista mines data of restaurant and food review using Apache Spark and gives 3 | out the frequently used words in the database. 4 | 5 | # Dependencies 6 | The dependencies for building the example project are: 7 | * [MongoDB](https://www.mongodb.org) 8 | * [MongoKit](http://namlook.github.io/mongokit/) 9 | * [NLTK Library](http://www.nltk.org) 10 | 11 | # Build Instructions 12 | 1. The first step is to setup the MongoDB Server on your local machine. 13 | MongoDB site provides installation instructions for all major OS. 14 | 2. You will need to populate the MongoDB with the data we have collected. The dump of the data is provided in the root folder of the `Django-LibSpark` library. 15 | 3. Run the following code to push the data dump to your instance of MongoDB: 16 | ```bash 17 | $ mongorestor --verbose path-to-dump 18 | ``` 19 | 4. Next you will need to setup NLTK. Install NLTK using pip - 20 | ```bash 21 | $ pip install nltk 22 | ``` 23 | 24 | Once NLTK is installed, fire up your python interpreter and run the following 25 | code- 26 | ```python 27 | >>> import nltk 28 | >>> nltk.download() 29 | ``` 30 | A window will pop open. Go on the All Packages tab and search for `stopwords` and download it. 31 | 5. Your development depdency is done. Now is the time to fire up the Django Development Server - 32 | ```bash 33 | $ python manage.py runserver 34 | ``` 35 | and point your browser to http://localhost:8000/FUW 36 | -------------------------------------------------------------------------------- /libspark/README.md: -------------------------------------------------------------------------------- 1 | Django Lib Spark 2 | ================ 3 | 4 | This is the core Django Lib Spark. This library is built on top of [PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) available for [Apache Spark](http://spark.apache.org/). 5 | 6 | What makes Spark Lib easy to use? 7 | ================================= 8 | 9 | Lib Spark makes communication to a Spark instacnce a cake walk. Lib Spark takes care of all the connection related initialisation scripts. It also supports powerfull method chaining mechanism along with lazy evaluation. 10 | 11 | Packages 12 | ======== 13 | 14 | 1. [pyspark.RDD](http://spark.apache.org/docs/latest/api/python/pyspark.html#pyspark.RDD) 15 | 16 | Functions Implemented 17 | ===================== 18 | 19 | * map 20 | * filter 21 | * collect 22 | * cache 23 | * count 24 | * distinct 25 | * first 26 | * flatMap 27 | * flatMapValues 28 | * groupBy 29 | * groupByKey 30 | * intersection 31 | * join 32 | * keyBy 33 | * keys 34 | * leftOuterJoin 35 | * lookup 36 | * mapPartitions 37 | * mapPartitionsWithIndex 38 | * mapPartitionsWithSplit 39 | * mapValues 40 | * max 41 | * mean 42 | * min 43 | * name 44 | * reduceByKey 45 | * reduceByKeyLocally 46 | * rightOuterJoin 47 | * sortBy 48 | * sortByKey 49 | * substract 50 | * sum 51 | * take 52 | * takeOrdered 53 | * top 54 | * union 55 | * values 56 | * variance 57 | * zip 58 | * zipWithIndex 59 | * zipWithUniqueId 60 | -------------------------------------------------------------------------------- /libspark/SparkRDD.py: -------------------------------------------------------------------------------- 1 | # Configure the necessary Spark environment 2 | import os 3 | import sys 4 | 5 | # Spark home 6 | spark_home = os.environ.get("SPARK_HOME") 7 | 8 | # If Spark V1.4.x is detected, then add ' pyspark-shell' to 9 | # the end of the 'PYSPARK_SUBMIT_ARGS' environment variable 10 | spark_release_file = spark_home + "/RELEASE" 11 | if os.path.exists(spark_release_file) and "Spark 1.4" in open(spark_release_file).read(): 12 | pyspark_submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "") 13 | if not "pyspark-shell" in pyspark_submit_args: pyspark_submit_args += " pyspark-shell" 14 | os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args 15 | 16 | # Add the spark python sub-directory to the path 17 | sys.path.insert(0, spark_home + "/python") 18 | 19 | # Add the py4j to the path. 20 | # You may need to change the version number to match your install 21 | sys.path.insert(0, os.path.join(spark_home, "python/lib/py4j-0.8.2.1-src.zip")) 22 | # Initialize PySpark to predefine the SparkContext variable 'sc' 23 | execfile(os.path.join(spark_home, "python/pyspark/shell.py")) 24 | 25 | class SparkRDD(): 26 | # When the SparkRDD is initialized - data and function 27 | # can be passed as arguments 28 | def __init__(self, data=[], **kwargs): 29 | self.func = kwargs.get("func",None) 30 | self.data = data 31 | self.is_cached = False 32 | try: 33 | self.dataRDD = sc.parallelize (data) 34 | except TypeError: 35 | self.dataRDD = data 36 | 37 | def map(self,func): 38 | mappedData = self.dataRDD.map(func) 39 | tempObj = SparkRDD(data = mappedData) 40 | return tempObj 41 | 42 | def filter(self,func): 43 | filteredData = self.dataRDD.filter(func) 44 | tempObj = SparkRDD(data = filteredData) 45 | return tempObj 46 | 47 | def collect(self): 48 | return self.dataRDD.collect() 49 | 50 | def cache(self): 51 | self.dataRDD.cache() 52 | self.is_cached = True 53 | return self 54 | 55 | def count(self): 56 | return self.dataRDD.count() 57 | 58 | def distinct(self): 59 | distinctRDD = self.dataRDD.distinct() 60 | distinctObj = SparkRDD(data=distinctRDD) 61 | return distinctObj 62 | 63 | def first(self): 64 | return self.dataRDD.first() 65 | 66 | def flatMap(self,func): 67 | flatMapRDD = self.dataRDD.flatMap(func) 68 | tempObj = SparkRDD(data=flatMapRDD) 69 | return tempObj 70 | 71 | def flatMapValues(self,func): 72 | flatMapRDD = self.dataRDD.flatMapValues(func) 73 | tempObj = SparkRDD(data=flatMapRDD) 74 | return tempObj 75 | 76 | def groupBy(self,func): 77 | groupByRDD = self.dataRDD.groupBy(func) 78 | tempObj = SparkRDD(data=groupByRDD) 79 | return tempObj 80 | 81 | def groupByKey(self): 82 | groupByRDD = self.dataRDD.groupByKey() 83 | tempObj = SparkRDD(data=groupByRDD) 84 | return tempObj 85 | 86 | def intersection(self,toBeIntersected): 87 | if isinstance(toBeIntersected,SparkRDD): 88 | intersected = self.dataRDD.intersection(toBeIntersected.dataRDD) 89 | tempObj = SparkRDD(data=intersected) 90 | else: 91 | #TODO: rasie err 92 | pass 93 | return tempObj 94 | 95 | def join(self,toBeJoined): 96 | if isinstance(toBeJoined,SparkRDD): 97 | joined = self.dataRDD.join(toBeJoined.dataRDD) 98 | tempObj = SparkRDD(data=joined) 99 | else: 100 | #TODO: rasie err 101 | pass 102 | return tempObj 103 | 104 | def keyBy(self,func): 105 | keyByRDD = self.dataRDD.keyBy(func) 106 | tempObj = SparkRDD(data=keyByRDD) 107 | return tempObj 108 | 109 | def keys(self): 110 | keysRDD = self.dataRDD.keys() 111 | tempObj = SparkRDD(data=keysRDD) 112 | return tempObj 113 | 114 | def leftOuterJoin(self,toBeJoined): 115 | if isinstance(toBeJoined,SparkRDD): 116 | leftJoined = self.dataRDD.join(toBeJoined.dataRDD) 117 | tempObj = SparkRDD(data=leftJoined) 118 | else: 119 | #TODO: rasie err 120 | pass 121 | return tempObj 122 | 123 | 124 | def lookup(self,key): 125 | valuesList = self.dataRDD.lookup(key) 126 | return valuesList 127 | 128 | def mapPartitions(self,func): 129 | mapPartitionsRDD = self.dataRDD.mapPartitions(func) 130 | tempObj = SparkRDD(data=mapPartitionsRDD) 131 | return tempObj 132 | 133 | def mapPartitionsWithIndex(self,func): 134 | mapPartitionsWithIndexRDD = self.dataRDD.mapPartitionsWithIndex(func) 135 | tempObj = SparkRDD(data=mapPartitionsWithIndexRDD) 136 | return tempObj 137 | 138 | def mapPartitionsWithSplit(self,func): 139 | mapPartitionsWithSplitRDD = self.dataRDD.mapPartitionsWithSplit(func) 140 | tempObj = SparkRDD(data=mapPartitionsWithSplitRDD) 141 | return tempObj 142 | 143 | def mapValues(self,func): 144 | mapValuesRDD = self.dataRDD.mapValues(func) 145 | tempObj = SparkRDD(data=mapValuesRDD) 146 | return tempObj 147 | 148 | def max(self,func=None): 149 | if func is None: 150 | maxed = self.dataRDD.max() 151 | else: 152 | maxed = self.dataRDD.max(func) 153 | return maxed 154 | 155 | def mean(self): 156 | meanRDD = self.dataRDD.mean() 157 | tempObj = SparkRDD(data=meanRDD) 158 | return tempObj 159 | 160 | def min(self,func=None): 161 | if func is None: 162 | mined = self.dataRDD.min() 163 | else: 164 | mined = self.dataRDD.min(func) 165 | return mined 166 | 167 | def name(self): 168 | return self.dataRDD.name 169 | 170 | def reduceByKey(self,func=None): 171 | if func is None: 172 | reduceByKeyRDD = self.dataRDD.reduceByKey() 173 | else: 174 | reduceByKeyRDD = self.dataRDD.reduceByKey(func) 175 | tempObj = SparkRDD(data=reduceByKeyRDD) 176 | return tempObj 177 | 178 | def reduceByKeyLocally(self,func=None): 179 | if key is None: 180 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally() 181 | else: 182 | reduceByKeyLocallyRDD = self.dataRDD.reduceByKeyLocally(func) 183 | tempObj = SparkRDD(data=reduceByKeyLocallyRDD) 184 | return tempObj 185 | 186 | 187 | def rightOuterJoin(self,toBeJoined): 188 | if isinstance(toBeJoined,SparkRDD): 189 | rightOuterJoinRDD = self.dataRDD.join(toBeJoined.dataRDD) 190 | tempObj = SparkRDD(data=rightOuterJoinRDD) 191 | else: 192 | #TODO: rasie err 193 | pass 194 | return tempObj 195 | 196 | def sortBy(self,func,ascending=True): 197 | sortedBy = self.dataRDD.sortBy(func) 198 | sortedObj = SparkRDD(data=sortedBy) 199 | return sortedObj 200 | 201 | def sortByKey(self,func,ascending=True): 202 | sortedByKey = self.dataRDD.sortByKey(func) 203 | sortedObj = SparkRDD(data=sortedByKey) 204 | return sortedObj 205 | 206 | def substract(self,toBeSubtracted): 207 | if isinstance(toBeJoined,SparkRDD): 208 | subtracted = self.dataRDD.subtract(toBeSubtracted.dataRDD) 209 | subtractedObj = SparkRDD(data=subtracted) 210 | else: 211 | #TODO: raise err 212 | pass 213 | return subtractedObj 214 | 215 | def sum(self): 216 | summed = self.dataRDD.sum() 217 | sumObj = SparkRDD(data=summed) 218 | return sumObj 219 | 220 | def take(self,num): 221 | took = self.dataRDD.take(num) 222 | tookObj = SparkRDD(data=took) 223 | return tookObj 224 | 225 | def takeOrdered(self,num,func=None): 226 | if func is None: 227 | tookOredered = self.dataRDD.takeOrdered(num) 228 | else: 229 | tookOredered = self.dataRDD.takeOrdered(num,key=func) 230 | return tookOredered 231 | 232 | def top(self,num,func): 233 | if func is None: 234 | topped = self.dataRDD.top(num) 235 | else: 236 | topped = self.dataRDD.top(num,key=func) 237 | return topped 238 | 239 | def union(self): 240 | unioned = self.dataRDD.union() 241 | unionObj = SparkRDD(data=unioned) 242 | return unionObj 243 | 244 | def values(self): 245 | valued = self.dataRDD.values() 246 | valueObj = SparkRDD(data=valued) 247 | return valueObj 248 | 249 | def variance(self): 250 | return self.dataRDD.variance() 251 | 252 | 253 | def zip(self,toBeZipped): 254 | if isinstance(toBeZipped,SparkRDD): 255 | zipped = self.dataRDD.zip(toBeZipped) 256 | else: 257 | #TODO: Raise err 258 | pass 259 | zipObj = SparkRDD(data=zipped) 260 | return zipObj 261 | 262 | def zipWithIndex(self): 263 | zipped = self.dataRDD.zipWithIndex() 264 | zipObj = SparkRDD(data=zipped) 265 | return zipObj 266 | 267 | def zipWithUniqueId(self): 268 | zipped = self.dataRDD.zipWithUniqueId() 269 | zippedObj = SparkRDD(data = zipped) 270 | return zipObj 271 | -------------------------------------------------------------------------------- /libspark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/libspark/__init__.py -------------------------------------------------------------------------------- /libspark/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /libspark/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/libspark/migrations/__init__.py -------------------------------------------------------------------------------- /libspark/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /libspark/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /libspark/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/libspark/views.py -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "spark_lib.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /spark_lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellowmessenger/django-libspark/7b626526b672f706a4d3accd68f131fb395b674f/spark_lib/__init__.py -------------------------------------------------------------------------------- /spark_lib/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for spark_lib project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.7/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.7/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = 'wu(zp0&cs2+liyh#!e(jb#3hgpk!8)ko*#o6^0r08y)!kux1dx' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 40 | 'libspark', 41 | ) 42 | 43 | MIDDLEWARE_CLASSES = ( 44 | 'django.contrib.sessions.middleware.SessionMiddleware', 45 | 'django.middleware.common.CommonMiddleware', 46 | 'django.middleware.csrf.CsrfViewMiddleware', 47 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 48 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | ) 52 | 53 | ROOT_URLCONF = 'spark_lib.urls' 54 | 55 | WSGI_APPLICATION = 'spark_lib.wsgi.application' 56 | 57 | 58 | # Database 59 | # https://docs.djangoproject.com/en/1.7/ref/settings/#databases 60 | 61 | DATABASES = { 62 | 'default': { 63 | 'ENGINE': 'django.db.backends.sqlite3', 64 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 65 | } 66 | } 67 | 68 | # Internationalization 69 | # https://docs.djangoproject.com/en/1.7/topics/i18n/ 70 | 71 | LANGUAGE_CODE = 'en-us' 72 | 73 | TIME_ZONE = 'UTC' 74 | 75 | USE_I18N = True 76 | 77 | USE_L10N = True 78 | 79 | USE_TZ = True 80 | 81 | 82 | # Static files (CSS, JavaScript, Images) 83 | # https://docs.djangoproject.com/en/1.7/howto/static-files/ 84 | 85 | STATIC_URL = '/static/' 86 | -------------------------------------------------------------------------------- /spark_lib/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | from django.contrib import admin 3 | 4 | 5 | urlpatterns = patterns('', 6 | # Examples: 7 | # url(r'^$', 'spark_lib.views.home', name='home'), 8 | # url(r'^blog/', include('blog.urls')), 9 | 10 | url(r'^admin/', include(admin.site.urls)), 11 | url(r'^run/?$', 'ml.views.run'), 12 | ) 13 | -------------------------------------------------------------------------------- /spark_lib/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for spark_lib project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "spark_lib.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | --------------------------------------------------------------------------------