├── .github └── workflows │ └── master.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Large_Dataset_ServerSide_Processing ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py ├── README.md ├── db.sqlite3 ├── deploy.sh ├── deployment ├── clear-junk ├── db_utils.sh ├── entrypoint └── start-app ├── docker-compose.yml ├── env.example ├── ldsp ├── __init__.py ├── admin.py ├── apps.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── templates │ └── ldsp │ │ └── index.html ├── tests.py ├── urls.py └── views.py ├── manage.py └── requirements.txt /.github/workflows/master.yml: -------------------------------------------------------------------------------- 1 | name: 🚀 LDSP [MASTER] 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | app_deployment: 10 | name: 🎉 Deployment 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: 🚚 Install SSH Key 14 | uses: shimataro/ssh-key-action@v2 15 | with: 16 | key: ${{ secrets.SSH_PRIVATE_KEY }} 17 | known_hosts: 'lsdp' 18 | 19 | - name: 🚚 Adding Known Hosts 20 | run: ssh-keyscan -p ${{ secrets.SSH_PORT }} -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts 21 | 22 | - name: 🚀 Run Deployment Script 23 | uses: appleboy/ssh-action@master 24 | with: 25 | host: ${{ secrets.SSH_HOST }} 26 | username: ${{ secrets.SSH_USER }} 27 | key: ${{ secrets.SSH_PRIVATE_KEY }} 28 | port: ${{ secrets.SSH_PORT }} 29 | command_timeout: 30m 30 | script: | 31 | cd /home/docker_projects/Large_Dataset_ServerSide_Processing 32 | git stash 33 | git pull origin master 34 | chmod +x deployment/* 35 | chmod +x deploy.sh 36 | sh deploy.sh 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .env.local 107 | .env.development.local 108 | .env.test.local 109 | .env.production.local 110 | .env.schema.json 111 | .venv 112 | venv/ 113 | ENV/ 114 | env/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | 136 | # pytype static type analyzer 137 | .pytype/ 138 | 139 | # Cython debug symbols 140 | cython_debug/ 141 | 142 | # VS Code settings 143 | .vscode/ 144 | 145 | # Node.js 146 | # Dependency directories 147 | node_modules/ 148 | # Optional npm cache directory 149 | .npm 150 | # Optional eslint cache 151 | .eslintcache 152 | # Optional REPL history 153 | .node_repl_history 154 | # Output of 'npm pack' 155 | *.tgz 156 | # Yarn Integrity file 157 | .yarn-integrity 158 | # dotenv environment variables file 159 | .env.local 160 | .env.development.local 161 | .env.test.local 162 | .env.production.local 163 | .env.test 164 | .env.production 165 | 166 | # Operating System Files 167 | .DS_Store 168 | .DS_Store? 169 | ._* 170 | .Spotlight-V100 171 | .Trashes 172 | ehthumbs.db 173 | Thumbs.db 174 | .idea -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.12 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | ENV PYTHONDONTWRITEBYTECODE 1 5 | 6 | WORKDIR /app 7 | 8 | COPY requirements.txt /app/requirements.txt 9 | RUN pip3 install -r requirements.txt 10 | 11 | RUN pip3 install mysql-connector-python 12 | RUN pip3 install hypercorn 13 | 14 | COPY deployment/* /app/deployment/* 15 | RUN sed -i 's/\r$//g' /app/deployment/* 16 | RUN chmod +x /app/deployment/* 17 | 18 | ENTRYPOINT ["/app/deployment/entrypoint"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Md. Rafat Hossain 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafathossain/Large_Dataset_ServerSide_Processing/95dded749694b2e00904582a3888196faee0990c/Large_Dataset_ServerSide_Processing/__init__.py -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for Large_Dataset_ServerSide_Processing project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Large_Dataset_ServerSide_Processing.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for Large_Dataset_ServerSide_Processing project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.1.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.1/ref/settings/ 11 | """ 12 | import os 13 | import environ 14 | from pathlib import Path 15 | 16 | env = environ.Env( 17 | # set casting, default value 18 | DEBUG=(bool, False) 19 | ) 20 | 21 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 22 | BASE_DIR = Path(__file__).resolve().parent.parent 23 | 24 | # Take environment variables from .env file 25 | environ.Env.read_env(os.path.join(BASE_DIR, '.env')) 26 | 27 | # Quick-start development settings - unsuitable for production 28 | # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ 29 | 30 | # SECURITY WARNING: keep the secret key used in production secret! 31 | SECRET_KEY = env('SECRET_KEY') 32 | 33 | # SECURITY WARNING: don't run with debug turned on in production! 34 | DEBUG = env('DEBUG') 35 | 36 | ALLOWED_HOSTS = ['*'] 37 | 38 | # Application definition 39 | INSTALLED_APPS = [ 40 | 'django.contrib.auth', 41 | 'django.contrib.contenttypes', 42 | 'django.contrib.sessions', 43 | 'django.contrib.messages', 44 | 'django.contrib.staticfiles', 45 | 'ldsp' 46 | ] 47 | 48 | MIDDLEWARE = [ 49 | 'django.middleware.security.SecurityMiddleware', 50 | 'django.contrib.sessions.middleware.SessionMiddleware', 51 | 'django.middleware.common.CommonMiddleware', 52 | 'django.middleware.csrf.CsrfViewMiddleware', 53 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 54 | 'django.contrib.messages.middleware.MessageMiddleware', 55 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 56 | ] 57 | 58 | ROOT_URLCONF = 'Large_Dataset_ServerSide_Processing.urls' 59 | 60 | TEMPLATES = [ 61 | { 62 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 63 | 'DIRS': [BASE_DIR / 'templates'], 64 | 'APP_DIRS': True, 65 | 'OPTIONS': { 66 | 'context_processors': [ 67 | 'django.template.context_processors.debug', 68 | 'django.template.context_processors.request', 69 | 'django.contrib.auth.context_processors.auth', 70 | 'django.contrib.messages.context_processors.messages', 71 | ], 72 | }, 73 | }, 74 | ] 75 | 76 | WSGI_APPLICATION = 'Large_Dataset_ServerSide_Processing.wsgi.application' 77 | 78 | # Database 79 | # https://docs.djangoproject.com/en/3.1/ref/settings/#databases 80 | 81 | DATABASES = { 82 | 'default': { 83 | 'ENGINE': 'django.db.backends.mysql', 84 | 'NAME': env('DB_NAME'), 85 | 'HOST': env('DB_HOST'), 86 | 'PORT': env('DB_PORT'), 87 | 'USER': env('DB_USER'), 88 | 'PASSWORD': env('DB_PASSWORD') 89 | } 90 | } 91 | 92 | # Password validation 93 | # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators 94 | 95 | AUTH_PASSWORD_VALIDATORS = [ 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 98 | }, 99 | { 100 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 101 | }, 102 | { 103 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 104 | }, 105 | { 106 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 107 | }, 108 | ] 109 | 110 | # Internationalization 111 | # https://docs.djangoproject.com/en/3.1/topics/i18n/ 112 | 113 | LANGUAGE_CODE = 'en-us' 114 | 115 | TIME_ZONE = 'Asia/Dhaka' 116 | 117 | USE_I18N = True 118 | 119 | USE_L10N = True 120 | 121 | USE_TZ = True 122 | 123 | # Static files (CSS, JavaScript, Images) 124 | # https://docs.djangoproject.com/en/3.1/howto/static-files/ 125 | 126 | STATIC_URL = 'static/' 127 | MEDIA_URL = 'media/' 128 | 129 | MEDIA_ROOT = os.path.join(BASE_DIR, "media") 130 | STATIC_ROOT = os.path.join(BASE_DIR, "static") 131 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/urls.py: -------------------------------------------------------------------------------- 1 | """Large_Dataset_ServerSide_Processing URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.1/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.urls import path, include 17 | 18 | urlpatterns = [ 19 | path('', include('ldsp.urls')) 20 | ] 21 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for Large_Dataset_ServerSide_Processing project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Large_Dataset_ServerSide_Processing.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Large Dataset Server-side Processing 2 | 3 | Here is a Demonstration of how you can easily handle extremely large dataset. Client-side processing is slow and consumes a lot of resources. Server-side processing if extremely fast and suitable for large dataset. 4 | 5 |

Live Demo: LDSP (ldsp.rafat.me)

6 | 7 | ## Required commands to run the project 8 | 9 | ``` 10 | pip install -r requirements.txt 11 | 12 | python manage.py makemigrations 13 | 14 | python manage.py migrate 15 | 16 | python manage.py runserver 17 | ``` 18 | 19 | If you have any queries, send me an email or send message on facebook. 20 | -------------------------------------------------------------------------------- /db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafathossain/Large_Dataset_ServerSide_Processing/95dded749694b2e00904582a3888196faee0990c/db.sqlite3 -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Deploying LDSP" 4 | echo "=========================================" 5 | echo "Prepared by Md. Rafat Hossain" 6 | echo "Date: 11 July 2024" 7 | echo "Version: v1.0" 8 | echo "=========================================" 9 | 10 | SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) 11 | 12 | echo -n "Bringing up docker container..." 13 | 14 | docker-compose -f "$SCRIPT_DIR/docker-compose.yml" down 15 | 16 | sh "$SCRIPT_DIR/deployment/clear-junk" 17 | 18 | sleep 1 19 | 20 | docker-compose -f "$SCRIPT_DIR/docker-compose.yml" up -d 21 | 22 | echo -n "Cleaning up junk..." 23 | 24 | sh "$SCRIPT_DIR/deployment/clear-junk" 25 | 26 | echo "COMPLETE" 27 | 28 | echo "Deployment completed successfully." 29 | echo "=========================================" 30 | -------------------------------------------------------------------------------- /deployment/clear-junk: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker system prune -a -f >/dev/null 2>&1 4 | docker system prune --all -f --volumes >/dev/null 2>&1 5 | docker volume prune -f >/dev/null 2>&1 6 | docker builder prune -a -f >/dev/null 2>&1 7 | docker volume rm $(docker volume ls -qf dangling=true) >/dev/null 2>&1 -------------------------------------------------------------------------------- /deployment/db_utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mariadb_ready() { 4 | python <&2 'Waiting for MariaDB to become available...' 14 | sleep 2 15 | done 16 | echo >&2 'MariaDB is available' 17 | 18 | exec "$@" 19 | -------------------------------------------------------------------------------- /deployment/start-app: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # if any of the commands in your code fails for any reason, the entire script fails 4 | set -o errexit 5 | # fail exit if one of your pipe command fails 6 | set -o pipefail 7 | # exits if any of your variables is not set 8 | set -o nounset 9 | 10 | echo "Collect static files" 11 | python manage.py collectstatic --noinput 12 | 13 | # Apply database migrations 14 | echo "Apply database migrations" 15 | python manage.py migrate --noinput 16 | 17 | #echo "Seed initial data" 18 | #python manage.py seed 19 | 20 | #Remove unused permissions and contenttypes 21 | python manage.py remove_stale_contenttypes --include-stale-apps --no-input 22 | 23 | # Start server 24 | echo "Starting server" 25 | 26 | #python manage.py runserver 0.0.0.0:8000 27 | hypercorn Large_Dataset_ServerSide_Processing.wsgi:application --bind 0.0.0.0:8001 28 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | networks: 4 | ldsp_tier: 5 | driver: bridge 6 | 7 | services: 8 | ldsp_web: 9 | container_name: ldsp 10 | build: 11 | context: . 12 | dockerfile: Dockerfile 13 | command: ./deployment/start-app 14 | restart: always 15 | env_file: 16 | - .env 17 | volumes: 18 | - .:/app 19 | ports: 20 | - "127.0.0.1:20007:8001" 21 | networks: 22 | - ldsp_tier 23 | -------------------------------------------------------------------------------- /env.example: -------------------------------------------------------------------------------- 1 | # Note: Don't use any single or double quote in env 2 | 3 | # SECURITY WARNING: don't run with debug turned on in production! 4 | DEBUG=False 5 | 6 | # SECURITY WARNING: keep the secret key used in production secret! 7 | SECRET_KEY= 8 | 9 | # Database Configuration 10 | DB_HOST=127.0.0.1 11 | DB_PORT=3306 12 | DB_NAME= 13 | DB_USER= 14 | DB_PASS= -------------------------------------------------------------------------------- /ldsp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafathossain/Large_Dataset_ServerSide_Processing/95dded749694b2e00904582a3888196faee0990c/ldsp/__init__.py -------------------------------------------------------------------------------- /ldsp/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /ldsp/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class LdspConfig(AppConfig): 5 | name = 'ldsp' 6 | -------------------------------------------------------------------------------- /ldsp/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1.2 on 2020-10-30 18:07 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='DataSet', 16 | fields=[ 17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('text', models.TextField()), 19 | ('random', models.IntegerField()), 20 | ('created_at', models.DateTimeField(auto_now_add=True)), 21 | ], 22 | options={ 23 | 'db_table': 'dataset', 24 | }, 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /ldsp/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafathossain/Large_Dataset_ServerSide_Processing/95dded749694b2e00904582a3888196faee0990c/ldsp/migrations/__init__.py -------------------------------------------------------------------------------- /ldsp/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class DataSet(models.Model): 5 | text = models.TextField() 6 | random = models.IntegerField() 7 | created_at = models.DateTimeField(auto_now_add=True) 8 | 9 | class Meta: 10 | db_table = 'dataset' 11 | -------------------------------------------------------------------------------- /ldsp/templates/ldsp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 11 | 12 | 15 | 17 | 18 | 19 | 21 | 22 | 23 | 25 | 27 | 28 | LDSP - Large Dataset Server-side Processing 29 | 30 | 32 | 33 | 51 | 52 | 53 |
54 |

Large Dataset Server-side Processing

55 |
56 | Here is a demonstration of how you can easily handle extremely large dataset. Client-side processing is 57 | slow and consumes a lot of resources. Server-side processing if extremely fast and suitable for large 58 | dataset. 59 |
60 |
61 |

62 | 64 |
65 | View Project on Github 67 |

68 |
69 |
70 |
71 |
72 |
73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 |
#TextRandomCreated At
85 |
86 |
87 |
88 |
89 |
90 |

Developed By Md. Rafat Hossain 91 |
92 | Founder & Managing Director, MetroVPS, A Brand of Flarezen Ltd. 93 |

94 |

95 | 98 | 101 | 105 | 109 |

110 |
111 | 112 | 113 | 115 | 118 | 121 | 122 | 123 | 125 | 127 | 129 | 146 | 147 | -------------------------------------------------------------------------------- /ldsp/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /ldsp/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path, include 2 | from .views import * 3 | 4 | urlpatterns = [ 5 | # Table Page URL 6 | path('', ldsp, name='ldsp.home'), 7 | 8 | # Data Endpoint 9 | path('data/', ldspData, name='ldsp.data'), 10 | 11 | # Data Seed 12 | path('seed/', ldspSeed, name='ldsp.seed') 13 | ] 14 | -------------------------------------------------------------------------------- /ldsp/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from django.http import JsonResponse, HttpResponse 3 | from .models import * 4 | import random 5 | from django.db.models import Q 6 | 7 | 8 | # Load Table Page 9 | def ldsp(request): 10 | return render(request, 'ldsp/index.html') 11 | 12 | 13 | # Load Table Data 14 | def ldspData(request): 15 | # Get the data from request 16 | search_value = request.GET['search[value]'].strip() 17 | startLimit = int(request.GET['start']) 18 | endLimit = startLimit + int(request.GET['length']) 19 | data_array = [] 20 | 21 | # Count the total length 22 | totalLength = DataSet.objects.count() 23 | 24 | # if search parameter is passed 25 | if search_value != '': 26 | # Querying dataset 27 | dataList = DataSet.objects.filter(Q(text__contains=search_value) | Q(random__contains=search_value)).order_by( 28 | 'id') 29 | 30 | # Filtering dataset 31 | dataFilter = dataList[startLimit:endLimit] 32 | 33 | # Get the filter length 34 | filterLength = dataList.count() 35 | else: 36 | # Querying dataset 37 | dataList = DataSet.objects.all().order_by('id') 38 | 39 | # Filtering dataset 40 | dataFilter = dataList[startLimit:endLimit] 41 | 42 | # Get the filter length 43 | filterLength = totalLength 44 | 45 | # Processing the data for table 46 | for key, item in enumerate(dataFilter): 47 | row_array = [str(key + 1), item.text, item.random, item.created_at] 48 | data_array.append(row_array) 49 | 50 | # Preparing the response 51 | response = { 52 | "draw": request.GET['draw'], 53 | "recordsTotal": totalLength, 54 | "recordsFiltered": filterLength, 55 | "data": data_array 56 | } 57 | 58 | # Returning json response 59 | return JsonResponse(response) 60 | 61 | 62 | def ldspSeed(request): 63 | # 1 Lakh Data Seed 64 | for i in range(1, 100000): 65 | # Creating dataset object 66 | DataSet.objects.create( 67 | text='This is text %s' % str(i), 68 | random=random.randint(100000, 999999) 69 | ) 70 | return HttpResponse("Done") 71 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Large_Dataset_ServerSide_Processing.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asgiref==3.2.10 2 | Django==3.1.2 3 | django-environ==0.8.1 4 | mysqlclient==2.1.0 5 | pytz==2020.1 6 | sqlparse==0.4.1 7 | --------------------------------------------------------------------------------