├── .gitignore ├── Dockerfile ├── api ├── __init__.py ├── admin.py ├── apps.py ├── cron.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── serializers.py ├── tests.py └── views.py ├── db.sqlite3 ├── docker-compose.yml ├── manage.py ├── readme.md ├── requirements.txt ├── screenshot ├── dashboard.png └── filters.png └── youtube_fetch_api ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | env/ 3 | venv/ 4 | .env 5 | .vscode 6 | **/.DS_Store 7 | 8 | cron_jobs.log 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | # create root directory for our project in the container 4 | RUN mkdir /youtube_fetch_api 5 | 6 | # Set the working directory to /youtube_fetch_api 7 | WORKDIR /youtube_fetch_api 8 | 9 | # Copy the current directory contents into the container at /youtube_fetch_api 10 | ADD . /youtube_fetch_api/ 11 | 12 | # Install any needed packages specified in requirements.txt 13 | RUN pip install -r requirements.txt -------------------------------------------------------------------------------- /api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/api/__init__.py -------------------------------------------------------------------------------- /api/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from . import models 3 | 4 | admin.site.register(models.Videos) 5 | -------------------------------------------------------------------------------- /api/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ApiConfig(AppConfig): 5 | name = 'api' 6 | -------------------------------------------------------------------------------- /api/cron.py: -------------------------------------------------------------------------------- 1 | import os 2 | # Cron Job 3 | from django_cron import CronJobBase, Schedule 4 | 5 | # Google API 6 | from apiclient.discovery import build 7 | import apiclient 8 | 9 | from .models import * 10 | from youtube_fetch_api import settings 11 | from datetime import datetime, timedelta 12 | 13 | 14 | class CallYoutubeApi(CronJobBase): 15 | RUN_EVERY_MINS = 5 # runs after every 5 minutes 16 | 17 | schedule = Schedule(run_every_mins=RUN_EVERY_MINS) 18 | code = 'api.call_youtube_api' # a unique code 19 | 20 | def do(self): 21 | apiKeys = settings.GOOGLE_API_KEYS 22 | time_now = datetime.now() 23 | last_request_time = time_now - timedelta(minutes=5) 24 | 25 | ##################################################### 26 | 27 | # A variable to check if a valid api key exists or not 28 | valid = False 29 | 30 | for apiKey in apiKeys: 31 | try: 32 | youtube = build("youtube", "v3", developerKey=apiKey) 33 | req = youtube.search().list(q="cricket", part="snippet", order="date", maxResults=50, 34 | publishedAfter=(last_request_time.replace(microsecond=0).isoformat()+'Z')) 35 | res = req.execute() 36 | valid = True 37 | except apiclient.errors.HttpError as err: 38 | code = err.resp.status 39 | if not(code == 400 or code == 403): 40 | break 41 | 42 | if valid: 43 | break 44 | ##################################################### 45 | 46 | 47 | if valid: 48 | 49 | ### CREATES AN OBJECT IN THE DB IF THERE IS A VALID API KEY AVAILABLE ### 50 | 51 | for item in res['items']: 52 | video_id = item['id']['videoId'] 53 | publishedDateTime = item['snippet']['publishedAt'] 54 | title = item['snippet']['title'] 55 | description = item['snippet']['description'] 56 | thumbnailsUrls = item['snippet']['thumbnails']['default']['url'] 57 | channel_id = item['snippet']['channelId'] 58 | channel_title = item['snippet']['channelTitle'] 59 | print(title) 60 | Videos.objects.create( 61 | video_id=video_id, 62 | title=title, 63 | description=description, 64 | channel_id=channel_id, 65 | channel_title=channel_title, 66 | publishedDateTime=publishedDateTime, 67 | thumbnailsUrls=thumbnailsUrls, 68 | ) 69 | -------------------------------------------------------------------------------- /api/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.10 on 2020-05-01 06:08 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Videos', 16 | fields=[ 17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('video_id', models.CharField(max_length=200)), 19 | ('title', models.CharField(blank=True, max_length=500, null=True)), 20 | ('description', models.CharField(blank=True, max_length=5000, null=True)), 21 | ('publishedDateTime', models.DateTimeField()), 22 | ('thumbnailsUrls', models.URLField()), 23 | ('channel_id', models.CharField(max_length=500)), 24 | ('channel_title', models.CharField(blank=True, max_length=500, null=True)), 25 | ('created', models.DateTimeField(auto_now_add=True, null=True)), 26 | ], 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /api/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/api/migrations/__init__.py -------------------------------------------------------------------------------- /api/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class Videos(models.Model): 4 | video_id = models.CharField( 5 | null=False, 6 | blank=False, 7 | max_length=200 8 | ) 9 | 10 | title = models.CharField( 11 | null=True, 12 | blank=True, 13 | max_length=500 14 | ) 15 | 16 | description = models.CharField( 17 | null=True, 18 | blank=True, 19 | max_length=5000 20 | ) 21 | 22 | publishedDateTime = models.DateTimeField() 23 | 24 | thumbnailsUrls = models.URLField() 25 | 26 | channel_id = models.CharField( 27 | null=False, 28 | blank=False, 29 | max_length=500 30 | ) 31 | 32 | channel_title = models.CharField( 33 | null=True, 34 | blank=True, 35 | max_length=500 36 | ) 37 | 38 | created = models.DateTimeField( 39 | auto_now_add=True, 40 | null=True, 41 | blank=True, 42 | ) 43 | -------------------------------------------------------------------------------- /api/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from .models import * 3 | 4 | class VideosSerializer(serializers.ModelSerializer): 5 | 6 | class Meta: 7 | model = Videos 8 | fields = "__all__" 9 | -------------------------------------------------------------------------------- /api/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /api/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from rest_framework import filters 3 | from django_filters.rest_framework import DjangoFilterBackend 4 | 5 | from .models import * 6 | from .serializers import * 7 | 8 | # Rest FrameWork 9 | from rest_framework import generics 10 | from rest_framework.pagination import CursorPagination 11 | 12 | class ResultsPagination(CursorPagination): 13 | page_size = 25 14 | page_size_query_param = 'page_size' 15 | max_page_size = 100 16 | 17 | # Searching is implemented using DRF Filters 18 | # DRF filter by default uses [icontains] and thus the search by default supports partial searches 19 | 20 | class YoutubeItems(generics.ListAPIView): 21 | search_fields = ['title', 'description'] 22 | filter_backends = (filters.SearchFilter,DjangoFilterBackend,filters.OrderingFilter) 23 | filterset_fields = ['channel_id','channel_title'] 24 | ordering = ('-publishedDateTime') 25 | queryset = Videos.objects.all() 26 | serializer_class = VideosSerializer 27 | pagination_class = ResultsPagination 28 | -------------------------------------------------------------------------------- /db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/db.sqlite3 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | web: 5 | build: . 6 | command: bash -c "python manage.py makemigrations && python manage.py migrate && python manage.py runserver 0.0.0.0:8000" 7 | container_name: youtube_fetch_api 8 | volumes: 9 | - .:/youtube_fetch_api 10 | ports: 11 | - "8000:8000" -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 9 | 'youtube_fetch_api.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Youtube Fetch Api 2 | An API to fetch latest videos from youtube sorted in reverse chronological order of their publishing date-time from YouTube for a given tag/search query in a paginated response. 3 | 4 | The server fetches latest videos async after every 10 minutes and saves it to the db. 5 | 6 | This project is completely based on Django. 7 | 8 | ## Method Used 9 | 10 | Used Cron Jobs [django_cron](https://django-cron.readthedocs.io/en/latest/introduction.html) to fetch videos after every 10 minutes using [Youtube Data Api](https://developers.google.com/youtube/v3/docs/search/list) and save it to the db 11 | 12 | ## Setup Guide 13 | - Clone the project 14 | - As this project is based on Django, your system need to have proper python setup, refer [this](https://www.python.org/downloads/) 15 | - Go the project through the terminal and install all dependencies by using typing `pip install -r requirements.txt` in the terminal 16 | - Inside the `setting.py` file, fill the variable `GOOGLE_API_KEYS` with all the API Keys available,the list should be filled as `['API_KEY_1','API_KEY_2',...]` 17 | - For getting an API key follow [this](https://developers.google.com/youtube/v3/getting-started) 18 | - Setup crontab to run Job, Follow [this](https://django-cron.readthedocs.io/en/latest/installation.html) 19 | - Run the server using `python mange.py runserver` 20 | 21 | ## Screenshots 22 | 23 | For visualization through a dashboard one may directly run the app locally and the get the UI provided through django-rest-framework 24 | 25 | ### Dashboard 26 | ![Dashboard](screenshot/dashboard.png) 27 | 28 | ### Filters 29 | ![Filters](screenshot/filters.png) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools==3.1.1 2 | certifi==2019.9.11 3 | chardet==3.0.4 4 | environ==1.0 5 | Django==2.2.10 6 | django-common-helpers==0.9.2 7 | django-cron==0.5.1 8 | django-crontab==0.7.1 9 | django-environ==0.4.5 10 | django-filter==2.2.0 11 | djangorestframework==3.10.3 12 | environ==1.0 13 | google-api-python-client==1.7.11 14 | google-auth==1.7.0 15 | google-auth-httplib2==0.0.3 16 | google-auth-oauthlib==0.4.1 17 | httplib2==0.14.0 18 | idna==2.8 19 | Markdown==3.1.1 20 | oauthlib==3.1.0 21 | pyasn1==0.4.7 22 | pyasn1-modules==0.2.7 23 | pytz==2019.3 24 | requests==2.22.0 25 | requests-oauthlib==1.2.0 26 | rsa==4.0 27 | six==1.13.0 28 | sqlparse==0.3.0 29 | uritemplate==3.0.0 30 | urllib3==1.25.6 31 | -------------------------------------------------------------------------------- /screenshot/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/screenshot/dashboard.png -------------------------------------------------------------------------------- /screenshot/filters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/screenshot/filters.png -------------------------------------------------------------------------------- /youtube_fetch_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavraj23/youtube_fetch_api/b8821eab98326f0eb63e8e8fb37d8da0dd987ef1/youtube_fetch_api/__init__.py -------------------------------------------------------------------------------- /youtube_fetch_api/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for youtube_fetch_api project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'youtube_fetch_api.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /youtube_fetch_api/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for youtube_fetch_api project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.0.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | import environ 15 | environ.Env.read_env() 16 | env = environ.Env( 17 | # set casting, default value 18 | DEBUG=(bool, False) 19 | ) 20 | 21 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 22 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 23 | 24 | 25 | # Quick-start development settings - unsuitable for production 26 | # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ 27 | 28 | # SECURITY WARNING: keep the secret key used in production secret! 29 | SECRET_KEY = 'w==ivz3++6_94s&rftfmnqua#2r7bm_eqjeek7=tsa9&h2wflm' 30 | 31 | # SECURITY WARNING: don't run with debug turned on in production! 32 | DEBUG = True 33 | 34 | ALLOWED_HOSTS = [] 35 | 36 | 37 | # Application definition 38 | 39 | INSTALLED_APPS = [ 40 | 'django.contrib.admin', 41 | 'django.contrib.auth', 42 | 'django.contrib.contenttypes', 43 | 'django.contrib.sessions', 44 | 'django.contrib.messages', 45 | 'django.contrib.staticfiles', 46 | 'django_cron', 47 | 'django_filters', 48 | 'django_crontab', 49 | 'rest_framework', 50 | 'api' 51 | ] 52 | 53 | MIDDLEWARE = [ 54 | 'django.middleware.security.SecurityMiddleware', 55 | 'django.contrib.sessions.middleware.SessionMiddleware', 56 | 'django.middleware.common.CommonMiddleware', 57 | 'django.middleware.csrf.CsrfViewMiddleware', 58 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 59 | 'django.contrib.messages.middleware.MessageMiddleware', 60 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 61 | ] 62 | 63 | 64 | CRON_CLASSES = [ 65 | "api.cron.CallYoutubeApi", 66 | ] 67 | 68 | # CRONJOBS = [ 69 | # ('*/5 * * * *', 'youtube_fetch_api.api.cron.CallYoutubeApi','>> ~/work/fam_pay_task/youtube_fetch_api/cron_job.log') 70 | # ] 71 | 72 | ROOT_URLCONF = 'youtube_fetch_api.urls' 73 | 74 | TEMPLATES = [ 75 | { 76 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 77 | 'DIRS': [], 78 | 'APP_DIRS': True, 79 | 'OPTIONS': { 80 | 'context_processors': [ 81 | 'django.template.context_processors.debug', 82 | 'django.template.context_processors.request', 83 | 'django.contrib.auth.context_processors.auth', 84 | 'django.contrib.messages.context_processors.messages', 85 | ], 86 | }, 87 | }, 88 | ] 89 | 90 | WSGI_APPLICATION = 'youtube_fetch_api.wsgi.application' 91 | 92 | 93 | # Database 94 | # https://docs.djangoproject.com/en/3.0/ref/settings/#databases 95 | 96 | DATABASES = { 97 | 'default': { 98 | 'ENGINE': 'django.db.backends.sqlite3', 99 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 100 | } 101 | } 102 | 103 | 104 | # Password validation 105 | # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators 106 | 107 | AUTH_PASSWORD_VALIDATORS = [ 108 | { 109 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 110 | }, 111 | { 112 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 113 | }, 114 | { 115 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 116 | }, 117 | { 118 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 119 | }, 120 | ] 121 | 122 | 123 | # Internationalization 124 | # https://docs.djangoproject.com/en/3.0/topics/i18n/ 125 | 126 | LANGUAGE_CODE = 'en-us' 127 | 128 | TIME_ZONE = 'UTC' 129 | 130 | USE_I18N = True 131 | 132 | USE_L10N = True 133 | 134 | USE_TZ = True 135 | 136 | GOOGLE_API_KEYS = ['AIzaSyA_BflqTg7p40wGKo0O-P4bW8zh3lSM6v8','AIzaSyDed2NGHComh2pbxw_QzjJmAD-rWXt7C2A'] 137 | 138 | REST_FRAMEWORK = { 139 | 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', 140 | 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], 141 | 'PAGE_SIZE': 10 142 | } 143 | 144 | 145 | # Static files (CSS, JavaScript, Images) 146 | # https://docs.djangoproject.com/en/3.0/howto/static-files/ 147 | 148 | STATIC_URL = '/static/' 149 | -------------------------------------------------------------------------------- /youtube_fetch_api/urls.py: -------------------------------------------------------------------------------- 1 | """youtube_fetch_api URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path 18 | 19 | from api import views 20 | 21 | urlpatterns = [ 22 | path('admin/', admin.site.urls), 23 | path('', views.YoutubeItems.as_view()) 24 | ] 25 | -------------------------------------------------------------------------------- /youtube_fetch_api/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for youtube_fetch_api project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'youtube_fetch_api.settings') 15 | 16 | application = get_wsgi_application() 17 | --------------------------------------------------------------------------------