├── web ├── web │ ├── __init__.py │ ├── __pycache__ │ │ ├── urls.cpython-37.pyc │ │ ├── wsgi.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ └── settings.cpython-37.pyc │ ├── wsgi.py │ ├── urls.py │ └── settings.py ├── questionFilter │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── 0001_initial.cpython-37.pyc │ │ └── 0001_initial.py │ ├── templatetags │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── custom_tags.cpython-37.pyc │ │ └── custom_tags.py │ ├── tests.py │ ├── apps.py │ ├── __pycache__ │ │ ├── admin.cpython-37.pyc │ │ ├── forms.cpython-37.pyc │ │ ├── models.cpython-37.pyc │ │ ├── views.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ ├── forms.py │ ├── admin.py │ ├── models.py │ └── views.py ├── .ignore ├── db.sqlite3 ├── manage.py ├── templets │ └── questionFilter │ │ ├── question_detail.html │ │ └── homepage.html └── crawler.py └── README.md /web/web/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/questionFilter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/questionFilter/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/questionFilter/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/.ignore: -------------------------------------------------------------------------------- 1 | *.log 2 | local_settings.py 3 | db.sqlite3 4 | db.sqlite3-journal -------------------------------------------------------------------------------- /web/db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/db.sqlite3 -------------------------------------------------------------------------------- /web/questionFilter/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /web/web/__pycache__/urls.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/web/__pycache__/urls.cpython-37.pyc -------------------------------------------------------------------------------- /web/web/__pycache__/wsgi.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/web/__pycache__/wsgi.cpython-37.pyc -------------------------------------------------------------------------------- /web/web/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/web/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /web/web/__pycache__/settings.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/web/__pycache__/settings.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class QuestionfilterConfig(AppConfig): 5 | name = 'questionFilter' 6 | -------------------------------------------------------------------------------- /web/questionFilter/__pycache__/admin.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/__pycache__/admin.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/__pycache__/forms.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/__pycache__/forms.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/__pycache__/views.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/__pycache__/views.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/migrations/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/migrations/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/templatetags/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/templatetags/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/migrations/__pycache__/0001_initial.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/migrations/__pycache__/0001_initial.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/templatetags/__pycache__/custom_tags.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lkw222/LeetcodeHelper/HEAD/web/questionFilter/templatetags/__pycache__/custom_tags.cpython-37.pyc -------------------------------------------------------------------------------- /web/questionFilter/templatetags/custom_tags.py: -------------------------------------------------------------------------------- 1 | from django import template 2 | 3 | register = template.Library() 4 | 5 | @register.filter 6 | def get_key(value, arg): 7 | return value.get(arg, None) 8 | -------------------------------------------------------------------------------- /web/questionFilter/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | class Search(forms.Form): 4 | google = forms.BooleanField(label="Google test") 5 | amazon = forms.BooleanField() 6 | facebook = forms.BooleanField() 7 | -------------------------------------------------------------------------------- /web/questionFilter/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from questionFilter.models import * 3 | # Register your models here. 4 | admin.site.register(Question) 5 | admin.site.register(Company) 6 | admin.site.register(CompanyTag) 7 | admin.site.register(Algorithm) 8 | admin.site.register(AlgorithmTag) 9 | admin.site.register(Similar) 10 | -------------------------------------------------------------------------------- /web/web/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for web project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'web.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /web/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'web.settings') 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError as exc: 12 | raise ImportError( 13 | "Couldn't import Django. Are you sure it's installed and " 14 | "available on your PYTHONPATH environment variable? Did you " 15 | "forget to activate a virtual environment?" 16 | ) from exc 17 | execute_from_command_line(sys.argv) 18 | 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LeetcodeHelper 2 | 3 | This project built a website which provides an advanced multiple filter for users to get more extensive use of leetcode. 4 | 5 | • Utilized Requests library and GraphQL API to crawl the question details from leetcode website. 6 | 7 | • Used multiprocessing to improve performance 200%. 8 | 9 | • Designed the relational database by using Django object-relational mapper (ORM) to store the data fetched. 10 | 11 | • Successfully created the website back-end by using Django MTV framework. 12 | 13 | • Built an interactive UI based on jQuery and CSS, embellished the interface of the website with Bootstrap. 14 | 15 | • Delivered the functional multiple filter, search and ranking by using Django model query. 16 | 17 | • Deployed the website on PythonAnywhere and got more than 5k visits within first week after launch. 18 | -------------------------------------------------------------------------------- /web/templets/questionFilter/question_detail.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 13 | 14 |
15 |

{{detail.front_id}}.{{detail.name}}

16 | 17 | 18 | 19 | 20 | 21 | {% autoescape off %} 22 | {{detail.content}} 23 | {% endautoescape %} 24 | 25 |
26 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /web/web/urls.py: -------------------------------------------------------------------------------- 1 | """web URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.2/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path, re_path 18 | from questionFilter import views 19 | 20 | urlpatterns = [ 21 | path('admin/', admin.site.urls), 22 | path('', views.home), 23 | re_path(r'question_detail/(.+)/$', views.question_detail), 24 | re_path(r'company_filter/(.+)/$', views.company_filter), 25 | re_path(r'algorithm_filter/(.+)/$', views.algorithm_filter), 26 | re_path(r'difficulty_filter/(.+)/$', views.difficulty_filter), 27 | re_path(r'question_sort/(.+)/$', views.question_sort), 28 | re_path(r'like_filter/(.+)/$', views.like_filter) 29 | 30 | ] 31 | -------------------------------------------------------------------------------- /web/questionFilter/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | 5 | class Question(models.Model): 6 | question_slug = models.SlugField(primary_key=True, unique=True) 7 | front_id = models.IntegerField() 8 | content = models.TextField() 9 | name = models.CharField(max_length=100) 10 | difficulty = models.CharField(max_length=10) 11 | total_accept = models.IntegerField() 12 | total_submission = models.IntegerField() 13 | accept_rate = models.FloatField() 14 | paid_only = models.BooleanField() 15 | like = models.IntegerField() 16 | dislike = models.IntegerField() 17 | frequency = models.FloatField() 18 | 19 | def __str__(self): 20 | return self.name 21 | 22 | 23 | class Company(models.Model): 24 | company_slug = models.SlugField(primary_key=True, unique=True) 25 | name = models.CharField(max_length=100) 26 | hire_link = models.URLField(null=True) 27 | 28 | def __str__(self): 29 | return self.name 30 | 31 | 32 | class CompanyTag(models.Model): 33 | company_slug = models.ForeignKey(Company, on_delete=True) 34 | question_slug = models.ForeignKey(Question, on_delete=True) 35 | vote_count = models.IntegerField() 36 | 37 | def __str__(self): 38 | return self.company_slug.name + ': ' + self.question_slug.name 39 | 40 | class Algorithm(models.Model): 41 | algorithm_slug = models.SlugField(primary_key=True, unique=True) 42 | name = models.CharField(max_length=100) 43 | topic_link = models.URLField(null=True) 44 | 45 | def __str__(self): 46 | return self.name 47 | 48 | 49 | class AlgorithmTag(models.Model): 50 | algorithm_slug = models.ForeignKey(Algorithm, on_delete=True) 51 | question_slug = models.ForeignKey(Question, on_delete=True) 52 | 53 | def __str__(self): 54 | return self.algorithm_slug.name + ': ' + self.question_slug.name 55 | 56 | class Similar(models.Model): 57 | cur_question_slug = models.ForeignKey(Question, on_delete=True, related_name='%(class)s_requests_created') 58 | tar_question_slug = models.ForeignKey(Question, on_delete=True) 59 | tar_difficulty = models.CharField(max_length=10) 60 | 61 | def __str__(self): 62 | return self.cur_question_slug.name + ' -> ' + self.tar_question_slug.name 63 | -------------------------------------------------------------------------------- /web/questionFilter/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.1 on 2019-08-26 20:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Algorithm', 16 | fields=[ 17 | ('algorithm_slug', models.SlugField(primary_key=True, serialize=False, unique=True)), 18 | ('name', models.CharField(max_length=100)), 19 | ('topic_link', models.URLField(null=True)), 20 | ], 21 | ), 22 | migrations.CreateModel( 23 | name='Company', 24 | fields=[ 25 | ('company_slug', models.SlugField(primary_key=True, serialize=False, unique=True)), 26 | ('name', models.CharField(max_length=100)), 27 | ('hire_link', models.URLField(null=True)), 28 | ], 29 | ), 30 | migrations.CreateModel( 31 | name='Question', 32 | fields=[ 33 | ('question_slug', models.SlugField(primary_key=True, serialize=False, unique=True)), 34 | ('front_id', models.IntegerField()), 35 | ('content', models.TextField()), 36 | ('name', models.CharField(max_length=100)), 37 | ('difficulty', models.CharField(max_length=10)), 38 | ('total_accept', models.IntegerField()), 39 | ('total_submission', models.IntegerField()), 40 | ('accept_rate', models.FloatField()), 41 | ('paid_only', models.BooleanField()), 42 | ('like', models.IntegerField()), 43 | ('dislike', models.IntegerField()), 44 | ('frequency', models.FloatField()), 45 | ], 46 | ), 47 | migrations.CreateModel( 48 | name='Similar', 49 | fields=[ 50 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 51 | ('tar_difficulty', models.CharField(max_length=10)), 52 | ('cur_question_slug', models.ForeignKey(on_delete=True, related_name='similar_requests_created', to='questionFilter.Question')), 53 | ('tar_question_slug', models.ForeignKey(on_delete=True, to='questionFilter.Question')), 54 | ], 55 | ), 56 | migrations.CreateModel( 57 | name='CompanyTag', 58 | fields=[ 59 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 60 | ('vote_count', models.IntegerField()), 61 | ('company_slug', models.ForeignKey(on_delete=True, to='questionFilter.Company')), 62 | ('question_slug', models.ForeignKey(on_delete=True, to='questionFilter.Question')), 63 | ], 64 | ), 65 | migrations.CreateModel( 66 | name='AlgorithmTag', 67 | fields=[ 68 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 69 | ('algorithm_slug', models.ForeignKey(on_delete=True, to='questionFilter.Algorithm')), 70 | ('question_slug', models.ForeignKey(on_delete=True, to='questionFilter.Question')), 71 | ], 72 | ), 73 | ] 74 | -------------------------------------------------------------------------------- /web/web/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for web project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.2.1. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.2/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | TEMPLATES_DIR = os.path.join(BASE_DIR, 'templets') 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'eolzocyv6dg-9_n&u)71)efff29yynhwaip=3!(ski^aijz5q7' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'questionFilter', 41 | ] 42 | 43 | MIDDLEWARE = [ 44 | 'django.middleware.security.SecurityMiddleware', 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | ] 52 | 53 | ROOT_URLCONF = 'web.urls' 54 | 55 | TEMPLATES = [ 56 | { 57 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 58 | 'DIRS': [TEMPLATES_DIR], 59 | 'APP_DIRS': True, 60 | 'OPTIONS': { 61 | 'context_processors': [ 62 | 'django.template.context_processors.debug', 63 | 'django.template.context_processors.request', 64 | 'django.contrib.auth.context_processors.auth', 65 | 'django.contrib.messages.context_processors.messages', 66 | ], 67 | }, 68 | }, 69 | ] 70 | 71 | WSGI_APPLICATION = 'web.wsgi.application' 72 | 73 | 74 | # Database 75 | # https://docs.djangoproject.com/en/2.2/ref/settings/#databases 76 | 77 | DATABASES = { 78 | 'default': { 79 | 'ENGINE': 'django.db.backends.sqlite3', 80 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 81 | } 82 | } 83 | 84 | 85 | # Password validation 86 | # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators 87 | 88 | AUTH_PASSWORD_VALIDATORS = [ 89 | { 90 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 91 | }, 92 | { 93 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 94 | }, 95 | { 96 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 97 | }, 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 100 | }, 101 | ] 102 | 103 | 104 | # Internationalization 105 | # https://docs.djangoproject.com/en/2.2/topics/i18n/ 106 | 107 | LANGUAGE_CODE = 'en-us' 108 | 109 | TIME_ZONE = 'UTC' 110 | 111 | USE_I18N = True 112 | 113 | USE_L10N = True 114 | 115 | USE_TZ = True 116 | 117 | 118 | # Static files (CSS, JavaScript, Images) 119 | # https://docs.djangoproject.com/en/2.2/howto/static-files/ 120 | 121 | STATIC_URL = '/static/' 122 | -------------------------------------------------------------------------------- /web/questionFilter/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render, redirect 2 | from questionFilter.models import * 3 | from questionFilter.forms import * 4 | from django.core.paginator import Paginator 5 | from django.http import HttpResponseRedirect 6 | from django.urls import reverse 7 | from django.db.models import Q 8 | from django.db.models import F 9 | 10 | 11 | # Create your views here. 12 | def home(request): 13 | data = {} 14 | data['companys'] = Company.objects.all().order_by('company_slug') 15 | data['algorithms'] = Algorithm.objects.all().order_by('algorithm_slug') 16 | company_selected = request.session.get('company_selected', {}) 17 | data['company_selected'] = company_selected 18 | algorithm_selected = request.session.get('algorithm_selected', {}) 19 | data['algorithm_selected'] = algorithm_selected 20 | difficulty_selected = request.session.get('difficulty_selected', {}) 21 | data['difficulty_selected'] = difficulty_selected 22 | like_ratio = request.session.get('like_ratio', 'all') 23 | data['like_ratio'] = like_ratio 24 | questions = Question.objects.all() 25 | 26 | # Company 27 | if len(company_selected) > 0: 28 | for cur_company in company_selected: 29 | questions = questions.filter(companytag__company_slug__company_slug__exact=cur_company) 30 | 31 | # Tag 32 | if len(algorithm_selected) > 0: 33 | for cur_algorithm in algorithm_selected: 34 | questions = questions.filter(algorithmtag__algorithm_slug__algorithm_slug__exact=cur_algorithm) 35 | 36 | # Difficulty 37 | if 0 < len(difficulty_selected) < 3: 38 | questions = questions.filter(difficulty__in=difficulty_selected) 39 | 40 | # Like Ratio 41 | if like_ratio == "Best": 42 | questions = questions.filter(like__gte=F('dislike')*10) 43 | elif like_ratio == "Excellent": 44 | questions = questions.filter(Q(like__lt=F('dislike')*10) & Q(like__gte=F('dislike')*1)) 45 | elif like_ratio == "Normal": 46 | questions = questions.filter(Q(like__lt=F('dislike')*1) & Q(like__gte=F('dislike')*0.5)) 47 | elif like_ratio == "CRAZE": 48 | questions = questions.filter(Q(like__lt=F('dislike')*0.5) & Q(like__gte=F('dislike')*0.1)) 49 | elif like_ratio == "SHIT": 50 | questions = questions.filter(like__lte=F('dislike')*0.1) 51 | else: 52 | pass 53 | 54 | 55 | # Search 56 | if request.method == 'POST': 57 | content = request.POST['search_content'] 58 | ids = [] 59 | names = [] 60 | for cur in content.split(): 61 | try: 62 | ids.append(int(cur)) 63 | except: 64 | names.append(cur) 65 | if names != [] and ids != []: 66 | questions = questions.filter(Q(name__icontains=' '.join(names)) & Q(front_id__in=ids)) 67 | elif ids != []: 68 | questions = questions.filter(Q(front_id__in=ids)) 69 | elif names != []: 70 | questions = questions.filter(Q(name__icontains=' '.join(names))) 71 | else: 72 | pass 73 | # Sort 74 | sort_type = request.session.get('sort_type', 'id') 75 | if sort_type == 'id': 76 | questions = questions.order_by('front_id') 77 | elif sort_type == 'frequency': 78 | questions = questions.order_by('-frequency') 79 | elif sort_type == 'AC': 80 | questions = questions.order_by('-accept_rate') 81 | elif sort_type == 'name': 82 | questions = questions.order_by('question_slug') 83 | data['type'] = sort_type 84 | 85 | for question in questions: 86 | question.frequency /= 0.05 87 | 88 | #Pagination 89 | limit = 50 90 | paginator = Paginator(questions, limit) 91 | page = request.GET.get('page') 92 | contacts = paginator.get_page(page) 93 | page_range = get_pane_range(paginator, 5) 94 | 95 | data['questions'] = contacts 96 | data['page_range'] = page_range 97 | return render(request, 'questionFilter/homepage.html', data) 98 | 99 | 100 | def company_filter(request, company_slug): 101 | company_selected = request.session.get('company_selected', {}) 102 | if company_slug in company_selected: 103 | del company_selected[company_slug] 104 | else: 105 | company_selected[company_slug] = None 106 | 107 | data = {'company_selected':company_selected} 108 | request.session['company_selected'] = company_selected 109 | return HttpResponseRedirect("/") 110 | 111 | 112 | def algorithm_filter(request, algorithm_slug): 113 | algorithm_selected = request.session.get('algorithm_selected', {}) 114 | if algorithm_slug in algorithm_selected: 115 | del algorithm_selected[algorithm_slug] 116 | else: 117 | algorithm_selected[algorithm_slug] = None 118 | 119 | data = {'algorithm_selected':algorithm_selected} 120 | request.session['algorithm_selected'] = algorithm_selected 121 | return HttpResponseRedirect("/") 122 | 123 | 124 | def difficulty_filter(request, difficulty): 125 | difficulty_selected = request.session.get('difficulty_selected', {}) 126 | if difficulty in difficulty_selected: 127 | del difficulty_selected[difficulty] 128 | else: 129 | difficulty_selected[difficulty] = None 130 | 131 | data = {'difficulty_selected':difficulty_selected} 132 | request.session['difficulty_selected'] = difficulty_selected 133 | return HttpResponseRedirect("/") 134 | 135 | def question_sort(request, sort_type): 136 | request.session['sort_type'] = sort_type 137 | return HttpResponseRedirect("/") 138 | 139 | def get_pane_range(paginator, display_num): 140 | max_page = paginator.num_pages 141 | page_range = {} 142 | page_range[1] = range(1, min(max_page, display_num)+1) 143 | page_range[2] = range(1, min(max_page, display_num)+1) 144 | page_range[max_page-1] = range(max(1, max_page+1-display_num), max_page) 145 | page_range[max_page] = range(max(1, max_page+1-display_num), max_page) 146 | return page_range 147 | 148 | 149 | def like_filter(request, like_ratio): 150 | request.session['like_ratio'] = like_ratio 151 | return HttpResponseRedirect("/") 152 | 153 | 154 | def question_detail(request, question_slug): 155 | detail = Question.objects.get(question_slug=question_slug) 156 | data = {'detail':detail} 157 | return render(request, 'questionFilter/question_detail.html', data) 158 | -------------------------------------------------------------------------------- /web/templets/questionFilter/homepage.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | LeetcodeHelper 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {% load custom_tags %} 14 |
15 |
16 |

Hi, Welcome to LeetcodeHelper


17 |

This website provides an advanced multiple filter for users to get more extensive use of leetcode. You can select 18 | questions based on whatever condition you want. For example, you can choose questions encountered in 19 | interviews of companies like Google Amazon, and tags of algorithms used (dp,dfs). You can also sort these questions by conditions like frequency... 20 | Beyond that, you can also search for some key words or certain ID.

21 |
22 |
23 | 24 | 25 |
26 | Easy 31 | Medium 38 | Hard 43 |
44 | 45 | 46 |
47 |

48 | 51 |

52 |
53 |
54 | {% for company in companys %} 55 | {% if company_selected and company.company_slug in company_selected %} 56 | {{company.name}} 57 | {% else %} 58 | {{company.name}} 59 | {% endif %} 60 | {% endfor %} 61 |
62 |
63 |
64 | 65 | 66 | 67 |
68 |

69 | 72 |

73 |
74 |
75 | {% for algorithm in algorithms %} 76 | {% if algorithm_selected and algorithm.algorithm_slug in algorithm_selected %} 77 | {{algorithm.name}} 78 | {% else %} 79 | {{algorithm.name}} 80 | {% endif %} 81 | {% endfor %} 82 |
83 |
84 |
85 | 86 | 87 | 88 |
89 |
90 | 93 |
94 | 95 | 96 | 97 | 98 | 99 | 100 |
101 |
102 |
103 | 104 | 105 | 106 |
107 |
108 | 109 |
110 | 111 |
112 | 113 | {% csrf_token %} 114 |
115 |
116 | 117 |
118 |
119 | 120 | 121 | 122 | 123 | 124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | {% for question in questions %} 137 | 138 | 139 | 145 | 154 | 155 | 170 | 171 | {% endfor %} 172 | 173 |
# Name Difficulty AC Rate Frequency
{{question.front_id}} 140 | {{question.name}} 141 | {% if question.paid_only %} 142 | 143 | {% endif %} 144 | 146 | {% if question.difficulty == 'Easy' %} 147 | Easy 148 | {% elif question.difficulty == 'Medium' %} 149 | Medium 150 | {% else %} 151 | Hard 152 | {% endif %} 153 | {{question.accept_rate}}% 156 | {% if question.frequency <= 20 %} 157 |
158 |
159 |
160 | {% elif 20 < question.frequency and question.frequency < 60 %} 161 |
162 |
163 |
164 | {% else %} 165 |
166 |
167 |
168 | {% endif %} 169 |
174 | 175 | 176 | 223 | 224 |
225 | 226 | 227 | -------------------------------------------------------------------------------- /web/crawler.py: -------------------------------------------------------------------------------- 1 | # from bs4 import BeautifulSoup 2 | # from urllib.request import urlopen 3 | import os 4 | import django 5 | import requests 6 | import json 7 | # import csv 8 | import multiprocessing 9 | import time 10 | import re 11 | 12 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'web.settings') 13 | django.setup() 14 | 15 | from questionFilter.models import * 16 | 17 | 18 | class LeetCodeCrawler(object): 19 | def __init__(self, user, password): 20 | self.user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36' 21 | self.session = requests.Session() 22 | self.base_url = 'https://leetcode.com/' 23 | self.is_premium = False 24 | self.headers = {'User-Agent': self.user_agent} 25 | r = self.session.get(self.base_url, headers=self.headers) 26 | csrftoken = re.search(r'(?<=csrftoken=).*?(?=;)', r.headers['Set-Cookie']).group(0) 27 | self.headers['x-csrftoken'] = csrftoken 28 | self.max_retry_time = 10 29 | self.is_login = self.login(user, password) 30 | if self.is_login: 31 | print('login successfully!') 32 | else: 33 | print('login failed!') 34 | 35 | def login(self, user, password): 36 | current_headers = self.headers.copy() 37 | login_url = 'https://leetcode.com/accounts/login/' 38 | login_info = {'csrfmiddlewaretoken':current_headers['x-csrftoken'], 39 | 'login':'lionpowder', 40 | 'password':'199488lkw@@', 41 | # 'login':'xiyu@oregonstate.edu', 42 | # 'password':'960615', 43 | 'next':'/problems' 44 | } 45 | current_headers['referer'] = 'https://leetcode.com/accounts/login/' 46 | 47 | query = {'operationName': "globalData", 48 | 'query': '''query globalData { 49 | isCurrentUserAuthenticated 50 | isPremium 51 | }''' 52 | } 53 | verify = None 54 | retry_time = 0 55 | while verify is None and retry_time < self.max_retry_time: 56 | try: 57 | retry_time += 1 58 | print('retry: ',retry_time) 59 | log = self.session.post(login_url, headers = current_headers, timeout=2, data = login_info, allow_redirects = False) 60 | current_headers['content-type'] = 'application/json' 61 | verify = self.session.post('https://leetcode.com/graphql', headers = current_headers, timeout=2, data=json.dumps(query)).json() 62 | except: 63 | pass 64 | if verify is None: 65 | print("Something Wrong") 66 | return False 67 | else: 68 | self.is_premium = verify['data']['isPremium'] 69 | return verify['data']['isCurrentUserAuthenticated'] 70 | 71 | def get_questions_list(self): 72 | all_questions_url = 'https://leetcode.com/api/problems/all/' 73 | all_questions = self.session.get(all_questions_url, headers=self.headers) 74 | return all_questions.json() 75 | 76 | def save_to_db(self, detail, paid_only, frequency): 77 | detail = detail['data'] 78 | question = detail['question'] 79 | 80 | question_slug = question['questionTitleSlug'] 81 | front_id = question['questionFrontendId'] 82 | content = question['content'] 83 | name = question['questionTitle'] 84 | difficulty = question['difficulty'] 85 | stats = eval(question['stats']) 86 | total_accept = stats['totalAcceptedRaw'] 87 | total_submission = stats['totalSubmissionRaw'] 88 | accept_rate = stats['acRate'][:-1] 89 | like = question['likes'] 90 | dislike = question['dislikes'] 91 | 92 | # slimilars = question['similarQuestions'] 93 | # company_tags = question['companyTags'] 94 | # company_stats = question['companyTagStats'] 95 | db_question = Question.objects.get_or_create(question_slug = question_slug, 96 | front_id=front_id, 97 | content=content, 98 | name=name, 99 | difficulty=difficulty, 100 | total_accept=total_accept, 101 | total_submission=total_submission, 102 | accept_rate=accept_rate, 103 | paid_only=paid_only, 104 | like=like, 105 | dislike=dislike, 106 | frequency=frequency) 107 | 108 | 109 | if db_question[1]: 110 | db_question[0].save() 111 | print(db_question) 112 | # db_companies = question['companyTags'] 113 | company_tags = json.loads(question['companyTagStats']).values() 114 | # if len(db_companies) > 0: 115 | # for company in db_companies: 116 | # company_slug = company['slug'] 117 | # name = company['name'] 118 | # db_company = Company.objects.get_or_create(company_slug=company_slug, 119 | # name=name) 120 | # # db_company.save() 121 | # 122 | for block in company_tags: 123 | for company_tag in block: 124 | company_slug = company_tag['slug'] 125 | name = company_tag['name'] 126 | db_company = Company.objects.get_or_create(company_slug=company_slug, 127 | name=name) 128 | if db_company[1]: 129 | db_company[0].save() 130 | vote_count = company_tag['timesEncountered'] 131 | db_company_tag = CompanyTag.objects.get_or_create(company_slug=db_company[0], 132 | question_slug=db_question[0], 133 | vote_count=vote_count) 134 | if db_company_tag[1]: 135 | db_company_tag[0].save() 136 | algorithm_tags = question['topicTags'] 137 | for algorithm_tag in algorithm_tags: 138 | algorithm_slug = algorithm_tag['slug'] 139 | name = algorithm_tag['name'] 140 | db_algorithm = Algorithm.objects.get_or_create(algorithm_slug=algorithm_slug, 141 | name=name) 142 | if db_algorithm[1]: 143 | db_algorithm[0].save() 144 | db_algorithm_tag = AlgorithmTag.objects.get_or_create(algorithm_slug=db_algorithm[0], 145 | question_slug=db_question[0]) 146 | if db_algorithm_tag[1]: 147 | db_algorithm_tag[0].save() 148 | 149 | similars = json.loads(question['similarQuestions']) 150 | for similar in similars: 151 | tar_question_slug = similar['titleSlug'] 152 | tar_difficulty = similar['difficulty'] 153 | try: 154 | db_tar_question = Question.objects.get(question_slug=tar_question_slug) 155 | db_similar1 = Similar.objects.get_or_create(cur_question_slug=db_question[0], 156 | tar_question_slug=db_tar_question, 157 | tar_difficulty=tar_difficulty) 158 | if db_similar1[1]: 159 | db_similar1[0].save() 160 | db_similar2 = Similar.objects.get_or_create(cur_question_slug=db_tar_question, 161 | tar_question_slug=db_question[0], 162 | tar_difficulty=difficulty) 163 | if db_similar2[1]: 164 | db_similar2[0].save() 165 | except Question.DoesNotExist: 166 | continue 167 | 168 | 169 | def get_detail(self, slug, paid_only, frequency): 170 | query_url = 'https://leetcode.com/graphql' 171 | current_headers = self.headers.copy() 172 | current_headers['referer'] = self.base_url + '/problems/' + slug 173 | current_headers['content-type'] = 'application/json' 174 | query = {'operationName': "getQuestionDetail", 175 | 'variables': {'titleSlug': slug}, 176 | 'query': '''query getQuestionDetail($titleSlug: String!) { 177 | question(titleSlug: $titleSlug) { 178 | questionId 179 | questionFrontendId 180 | questionTitle 181 | questionTitleSlug 182 | content 183 | difficulty 184 | stats 185 | similarQuestions 186 | categoryTitle 187 | topicTags { 188 | name 189 | slug 190 | } 191 | # companyTags { 192 | # name 193 | # slug 194 | # } 195 | companyTagStats 196 | likes 197 | dislikes 198 | } 199 | }''' 200 | } 201 | 202 | detail = None 203 | retry_time = -1 204 | while detail is None and retry_time < self.max_retry_time: 205 | try: 206 | retry_time += 1 207 | if retry_time > 0: 208 | print(retry_time) 209 | detail = self.session.get(query_url, data=json.dumps(query), headers=current_headers).json() 210 | temp = detail['data']['question'] 211 | 212 | except: 213 | pass 214 | self.save_to_db(detail, paid_only, frequency) 215 | print(temp['questionFrontendId'], temp['questionTitleSlug']) 216 | 217 | return detail 218 | 219 | # Multithreading version 220 | def get_all_details1(self): 221 | questions_list = self.get_questions_list()['stat_status_pairs'] 222 | start = time.time() 223 | pool = multiprocessing.Pool() 224 | for question in questions_list: 225 | print(question['stat']['frontend_question_id']) 226 | paid_only = question['paid_only'] 227 | frequency = question['frequency'] 228 | if self.is_premium or not paid_only: 229 | pool.apply_async(self.get_detail, (question['stat']['question__title_slug'], paid_only, frequency)) 230 | # self.getDetail(question['stat']['question__title_slug']) 231 | pool.close() 232 | pool.join() 233 | print("Multiple Processing Version: ", time.time() - start) 234 | 235 | # Sequential version 236 | def get_all_details2(self): 237 | questions_list = self.get_questions_list()['stat_status_pairs'] 238 | start = time.time() 239 | for question in questions_list: 240 | paid_only = question['paid_only'] 241 | frequency = question['frequency'] 242 | if self.is_premium or not paid_only: 243 | self.get_detail(question['stat']['question__title_slug'], paid_only, frequency) 244 | 245 | print("Sequential Version: ", time.time() - start) 246 | 247 | 248 | if __name__ == '__main__': 249 | crawler = LeetCodeCrawler(1,2) 250 | # l = crawler.get_questions_list() 251 | # print(l) 252 | # crawler.login(1,2) 253 | # detail = crawler.get_detail('two-sum', False, 0) 254 | # detail = crawler.get_detail('3sum', False) 255 | # detail = crawler.get_detail('reordered-power-of-2', False) 256 | 257 | # print(detail['data']['question']) 258 | crawler.get_all_details1() 259 | # print(l['stat_status_pairs'][0]) 260 | --------------------------------------------------------------------------------