├── .github └── workflows │ └── master.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Large_Dataset_ServerSide_Processing ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py ├── README.md ├── db.sqlite3 ├── deploy.sh ├── deployment ├── clear-junk ├── db_utils.sh ├── entrypoint └── start-app ├── docker-compose.yml ├── env.example ├── ldsp ├── __init__.py ├── admin.py ├── apps.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── templates │ └── ldsp │ │ └── index.html ├── tests.py ├── urls.py └── views.py ├── manage.py └── requirements.txt /.github/workflows/master.yml: -------------------------------------------------------------------------------- 1 | name: 🚀 LDSP [MASTER] 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | app_deployment: 10 | name: 🎉 Deployment 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: 🚚 Install SSH Key 14 | uses: shimataro/ssh-key-action@v2 15 | with: 16 | key: ${{ secrets.SSH_PRIVATE_KEY }} 17 | known_hosts: 'lsdp' 18 | 19 | - name: 🚚 Adding Known Hosts 20 | run: ssh-keyscan -p ${{ secrets.SSH_PORT }} -H ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts 21 | 22 | - name: 🚀 Run Deployment Script 23 | uses: appleboy/ssh-action@master 24 | with: 25 | host: ${{ secrets.SSH_HOST }} 26 | username: ${{ secrets.SSH_USER }} 27 | key: ${{ secrets.SSH_PRIVATE_KEY }} 28 | port: ${{ secrets.SSH_PORT }} 29 | command_timeout: 30m 30 | script: | 31 | cd /home/docker_projects/Large_Dataset_ServerSide_Processing 32 | git stash 33 | git pull origin master 34 | chmod +x deployment/* 35 | chmod +x deploy.sh 36 | sh deploy.sh 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .env.local 107 | .env.development.local 108 | .env.test.local 109 | .env.production.local 110 | .env.schema.json 111 | .venv 112 | venv/ 113 | ENV/ 114 | env/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | 136 | # pytype static type analyzer 137 | .pytype/ 138 | 139 | # Cython debug symbols 140 | cython_debug/ 141 | 142 | # VS Code settings 143 | .vscode/ 144 | 145 | # Node.js 146 | # Dependency directories 147 | node_modules/ 148 | # Optional npm cache directory 149 | .npm 150 | # Optional eslint cache 151 | .eslintcache 152 | # Optional REPL history 153 | .node_repl_history 154 | # Output of 'npm pack' 155 | *.tgz 156 | # Yarn Integrity file 157 | .yarn-integrity 158 | # dotenv environment variables file 159 | .env.local 160 | .env.development.local 161 | .env.test.local 162 | .env.production.local 163 | .env.test 164 | .env.production 165 | 166 | # Operating System Files 167 | .DS_Store 168 | .DS_Store? 169 | ._* 170 | .Spotlight-V100 171 | .Trashes 172 | ehthumbs.db 173 | Thumbs.db 174 | .idea -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.12 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | ENV PYTHONDONTWRITEBYTECODE 1 5 | 6 | WORKDIR /app 7 | 8 | COPY requirements.txt /app/requirements.txt 9 | RUN pip3 install -r requirements.txt 10 | 11 | RUN pip3 install mysql-connector-python 12 | RUN pip3 install hypercorn 13 | 14 | COPY deployment/* /app/deployment/* 15 | RUN sed -i 's/\r$//g' /app/deployment/* 16 | RUN chmod +x /app/deployment/* 17 | 18 | ENTRYPOINT ["/app/deployment/entrypoint"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Md. Rafat Hossain 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafathossain/Large_Dataset_ServerSide_Processing/95dded749694b2e00904582a3888196faee0990c/Large_Dataset_ServerSide_Processing/__init__.py -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for Large_Dataset_ServerSide_Processing project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Large_Dataset_ServerSide_Processing.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for Large_Dataset_ServerSide_Processing project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.1.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.1/ref/settings/ 11 | """ 12 | import os 13 | import environ 14 | from pathlib import Path 15 | 16 | env = environ.Env( 17 | # set casting, default value 18 | DEBUG=(bool, False) 19 | ) 20 | 21 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 22 | BASE_DIR = Path(__file__).resolve().parent.parent 23 | 24 | # Take environment variables from .env file 25 | environ.Env.read_env(os.path.join(BASE_DIR, '.env')) 26 | 27 | # Quick-start development settings - unsuitable for production 28 | # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ 29 | 30 | # SECURITY WARNING: keep the secret key used in production secret! 31 | SECRET_KEY = env('SECRET_KEY') 32 | 33 | # SECURITY WARNING: don't run with debug turned on in production! 34 | DEBUG = env('DEBUG') 35 | 36 | ALLOWED_HOSTS = ['*'] 37 | 38 | # Application definition 39 | INSTALLED_APPS = [ 40 | 'django.contrib.auth', 41 | 'django.contrib.contenttypes', 42 | 'django.contrib.sessions', 43 | 'django.contrib.messages', 44 | 'django.contrib.staticfiles', 45 | 'ldsp' 46 | ] 47 | 48 | MIDDLEWARE = [ 49 | 'django.middleware.security.SecurityMiddleware', 50 | 'django.contrib.sessions.middleware.SessionMiddleware', 51 | 'django.middleware.common.CommonMiddleware', 52 | 'django.middleware.csrf.CsrfViewMiddleware', 53 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 54 | 'django.contrib.messages.middleware.MessageMiddleware', 55 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 56 | ] 57 | 58 | ROOT_URLCONF = 'Large_Dataset_ServerSide_Processing.urls' 59 | 60 | TEMPLATES = [ 61 | { 62 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 63 | 'DIRS': [BASE_DIR / 'templates'], 64 | 'APP_DIRS': True, 65 | 'OPTIONS': { 66 | 'context_processors': [ 67 | 'django.template.context_processors.debug', 68 | 'django.template.context_processors.request', 69 | 'django.contrib.auth.context_processors.auth', 70 | 'django.contrib.messages.context_processors.messages', 71 | ], 72 | }, 73 | }, 74 | ] 75 | 76 | WSGI_APPLICATION = 'Large_Dataset_ServerSide_Processing.wsgi.application' 77 | 78 | # Database 79 | # https://docs.djangoproject.com/en/3.1/ref/settings/#databases 80 | 81 | DATABASES = { 82 | 'default': { 83 | 'ENGINE': 'django.db.backends.mysql', 84 | 'NAME': env('DB_NAME'), 85 | 'HOST': env('DB_HOST'), 86 | 'PORT': env('DB_PORT'), 87 | 'USER': env('DB_USER'), 88 | 'PASSWORD': env('DB_PASSWORD') 89 | } 90 | } 91 | 92 | # Password validation 93 | # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators 94 | 95 | AUTH_PASSWORD_VALIDATORS = [ 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 98 | }, 99 | { 100 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 101 | }, 102 | { 103 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 104 | }, 105 | { 106 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 107 | }, 108 | ] 109 | 110 | # Internationalization 111 | # https://docs.djangoproject.com/en/3.1/topics/i18n/ 112 | 113 | LANGUAGE_CODE = 'en-us' 114 | 115 | TIME_ZONE = 'Asia/Dhaka' 116 | 117 | USE_I18N = True 118 | 119 | USE_L10N = True 120 | 121 | USE_TZ = True 122 | 123 | # Static files (CSS, JavaScript, Images) 124 | # https://docs.djangoproject.com/en/3.1/howto/static-files/ 125 | 126 | STATIC_URL = 'static/' 127 | MEDIA_URL = 'media/' 128 | 129 | MEDIA_ROOT = os.path.join(BASE_DIR, "media") 130 | STATIC_ROOT = os.path.join(BASE_DIR, "static") 131 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/urls.py: -------------------------------------------------------------------------------- 1 | """Large_Dataset_ServerSide_Processing URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.1/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.urls import path, include 17 | 18 | urlpatterns = [ 19 | path('', include('ldsp.urls')) 20 | ] 21 | -------------------------------------------------------------------------------- /Large_Dataset_ServerSide_Processing/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for Large_Dataset_ServerSide_Processing project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Large_Dataset_ServerSide_Processing.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Large Dataset Server-side Processing 2 | 3 | Here is a Demonstration of how you can easily handle extremely large dataset. Client-side processing is slow and consumes a lot of resources. Server-side processing if extremely fast and suitable for large dataset. 4 | 5 |
62 |
64 |
65 | View Project on Github
67 |
# | 77 |Text | 78 |Random | 79 |Created At | 80 |
---|
Developed By Md. Rafat Hossain
91 |
92 | Founder & Managing Director, MetroVPS, A Brand of Flarezen Ltd.
93 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
104 |
105 |
106 |
108 |
109 |