├── .dockerignore ├── .gitignore ├── searchtube ├── __init__.py ├── db.py ├── search.py ├── str_find.py ├── download_subtitle.py ├── utils.py └── proccess_channel.py ├── requirements.txt ├── web ├── static │ ├── images │ │ └── loading.png │ ├── js │ │ ├── jquery.nice-select.min.js │ │ ├── index.js │ │ └── jquery-3.6.0.min.js │ └── css │ │ ├── index.css │ │ └── nice-select.css ├── searchtube.wsgi ├── server.py └── templates │ └── index.html ├── setup.sh ├── tests ├── test_add.py └── test_search.py ├── .example.env ├── bin ├── reprocess_channels ├── remove_channel ├── add_channel ├── proccess_channels ├── entrypiont.sh └── apache-setup.sh ├── Dockerfile ├── Makefile ├── README.md ├── LICENSE └── docker-compose.yml /.dockerignore: -------------------------------------------------------------------------------- 1 | .data 2 | .git 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .data 2 | *__pycache__* 3 | .env 4 | notes.txt 5 | etc/ -------------------------------------------------------------------------------- /searchtube/__init__.py: -------------------------------------------------------------------------------- 1 | from . import db, search, proccess_channel, utils 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | pymongo 3 | python-dotenv 4 | youtube_dl 5 | requests 6 | webvtt-py 7 | scrapetube 8 | -------------------------------------------------------------------------------- /web/static/images/loading.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dermasmid/searchtube/HEAD/web/static/images/loading.png -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sudo apt update 4 | sudo apt install make python3-pip docker.io -y 5 | pip3 install docker-compose 6 | -------------------------------------------------------------------------------- /tests/test_add.py: -------------------------------------------------------------------------------- 1 | from searchtube import proccess_channel 2 | 3 | 4 | proccess_channel.proccess('UCXv-co3EYHF7aOH4A93qAHQ') 5 | -------------------------------------------------------------------------------- /.example.env: -------------------------------------------------------------------------------- 1 | PYTHONUNBUFFERED=1 2 | SERVER_NAME=localhost 3 | SERVER_ADMIN=cheskeltwersky@gmail.com 4 | DB_USERNAME=admin 5 | DB_PASSWORD=test 6 | SLEEP_AFTER_DOWNLOAD=1 # 0 for fasle 7 | -------------------------------------------------------------------------------- /bin/reprocess_channels: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | from searchtube import proccess_channel 3 | import sys 4 | 5 | 6 | if __name__ == '__main__': 7 | proccess_channel.reprocess_channels() 8 | -------------------------------------------------------------------------------- /bin/remove_channel: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | from searchtube import utils 3 | import sys 4 | 5 | 6 | if __name__ == '__main__': 7 | # 1 = channel_id 8 | utils.remove_channel(sys.argv[1]) 9 | -------------------------------------------------------------------------------- /web/searchtube.wsgi: -------------------------------------------------------------------------------- 1 | #!/bin/python3 2 | import sys 3 | sys.path.append('/var/www/searchtube/web') 4 | from dotenv import load_dotenv 5 | load_dotenv() 6 | 7 | from server import app as application 8 | -------------------------------------------------------------------------------- /bin/add_channel: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | from searchtube import utils 3 | import sys 4 | 5 | 6 | if __name__ == '__main__': 7 | # 1 = channel_id, 2 = channel_name 8 | utils.add_channel(sys.argv[1], sys.argv[2]) 9 | -------------------------------------------------------------------------------- /searchtube/db.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | import os 3 | 4 | def get_client(): 5 | client = MongoClient('searchtube_mongo', 27017, username=os.environ['DB_USERNAME'], password=os.environ['DB_PASSWORD']) 6 | return client 7 | -------------------------------------------------------------------------------- /tests/test_search.py: -------------------------------------------------------------------------------- 1 | from searchtube import search 2 | import datetime 3 | q = input("enter q: ") 4 | 5 | start = datetime.datetime.now() 6 | x= search.search('UCXv-co3EYHF7aOH4A93qAHQ', q) 7 | 8 | print(x) 9 | print(len(x)) 10 | print(datetime.datetime.now() - start) -------------------------------------------------------------------------------- /bin/proccess_channels: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | from searchtube import proccess_channel, utils 4 | 5 | 6 | 7 | for channel in utils.get_channels(): 8 | channel_id = channel['channel_id'] 9 | print(f'Now proccessing {channel_id}') 10 | proccess_channel.process(channel_id) 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-buster 2 | 3 | 4 | RUN apt-get update && apt-get install -y \ 5 | apt-utils \ 6 | apache2 \ 7 | libapache2-mod-wsgi-py3 \ 8 | python3-certbot-apache 9 | 10 | COPY requirements.txt .env / 11 | COPY bin/* /usr/local/sbin/ 12 | 13 | 14 | RUN pip3 install -r requirements.txt 15 | 16 | 17 | 18 | CMD ["entrypiont.sh"] 19 | -------------------------------------------------------------------------------- /bin/entrypiont.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | if [ ! -f /etc/apache2/sites-available/searchtube.conf ]; then 5 | apache-setup.sh 6 | fi 7 | 8 | a2enmod rewrite 9 | a2enmod ssl 10 | a2enmod socache_shmcb 11 | a2ensite searchtube 12 | a2ensite searchtube-le-ssl 13 | 14 | service apache2 start 15 | 16 | 17 | while : 18 | do 19 | echo "Proccessing channels" 20 | proccess_channels 21 | echo "Slepping" 22 | sleep 6h 23 | done 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: up 2 | 3 | up: 4 | docker-compose up -d --remove-orphans 5 | 6 | shell: 7 | docker exec -ti -e COLUMNS=$(shell tput cols) -e LINES=$(shell tput lines) searchtube_wsgi bash 8 | 9 | logs: 10 | docker logs searchtube_wsgi -f 11 | 12 | stop: 13 | docker-compose stop 14 | 15 | down: 16 | docker-compose down 17 | 18 | build: 19 | docker-compose up -d --build 20 | 21 | ps: 22 | docker-compose ps 23 | 24 | restart: 25 | docker-compose restart 26 | -------------------------------------------------------------------------------- /bin/apache-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source .env 4 | 5 | cat << EOF > /etc/apache2/sites-available/searchtube.conf 6 | 7 | ServerName $SERVER_NAME 8 | ServerAdmin $SERVER_ADMIN 9 | WSGIScriptAlias / /var/www/searchtube/web/searchtube.wsgi 10 | 11 | Require all granted 12 | 13 | 14 | Alias /static /var/www/searchtube/web/static 15 | 16 | Require all granted 17 | 18 | 19 | ErrorLog /var/log/apache2/error.log 20 | LogLevel warn 21 | CustomLog /var/log/apache2/access.log combined 22 | SetEnv DB_USERNAME $DB_USERNAME 23 | SetEnv DB_PASSWORD $DB_PASSWORD 24 | 25 | EOF 26 | -------------------------------------------------------------------------------- /searchtube/search.py: -------------------------------------------------------------------------------- 1 | from . import db, str_find 2 | 3 | 4 | def search(channel_id: str, q: str, limit = 0): 5 | results = [] 6 | client = db.get_client() 7 | database = client.get_database(channel_id) 8 | full_coll = database.get_collection('full_text') 9 | vague_results = full_coll.find({'data': {'$regex': rf'\b{q}\s'}}).sort('date', -1).allow_disk_use(True).limit(limit) 10 | for video in vague_results: 11 | video_id = video['video_id'] 12 | video_coll = database.get_collection(video_id) 13 | points = str_find.split_find(video['data'], q) 14 | for result in points: 15 | data = list(video_coll.find({'indexes': {"$in": result}})) 16 | start = data[0]['start'] 17 | end = data[-1]['end'] 18 | results.append(f"https://www.youtube.com/embed/{video_id}?start={start}&end={end}") 19 | 20 | 21 | return results 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | You can use this project to make any youtube channel searchable. 3 | Follow the instuctions bellow, and if you bump in to any issue, please open an issue. 4 | 5 | You can see the site live [here](https://searchtube.site) 6 | 7 | I know that UI is not pretty, I"m not soo good at those stuff, so if you are - please consider contibuting, thanks. 8 | 9 | # Installation 10 | 11 | ```bash 12 | git clone https://github.com/dermasmid/searchtube.git && cd searchtube 13 | ./setup.sh 14 | cp .example.env .env 15 | nano .env 16 | make build 17 | ``` 18 | 19 | To add ssl run inside the container (make shell): 20 | `certbot --apache -d yourdomain.com` 21 | 22 | # Adding a channel 23 | 24 | ```bash 25 | make shell 26 | add_channel UCXv-co3EYHF7aOH4A93qAHQ "Lew later" 27 | ``` 28 | 29 | Replace the first argument with the channel id, and the second with the name of the channel - how you want it to be displayed on the site. 30 | 31 | -------------------------------------------------------------------------------- /web/server.py: -------------------------------------------------------------------------------- 1 | #!/bin/python3 2 | from flask import Flask, request, render_template 3 | import searchtube 4 | 5 | app = Flask(__name__) 6 | 7 | 8 | @app.route('/') 9 | def hello(): 10 | channels = searchtube.utils.get_channels() 11 | return render_template('index.html', channels= channels) 12 | 13 | 14 | @app.route('/search') 15 | def search(): 16 | q = request.args.get('q') 17 | channel_id = request.args.get('channel_id') 18 | limit = int(request.args.get('limit', 0)) 19 | 20 | if channel_id == '0': 21 | try: 22 | channel_id = searchtube.utils.get_channels()[0]['channel_id'] 23 | except IndexError: 24 | # there's no channels added 25 | channel_id = '' 26 | if searchtube.utils.channel_is_in_db(channel_id): 27 | try: 28 | results = searchtube.search.search(channel_id, q, limit) 29 | except: 30 | results = [] 31 | return {'data': results} 32 | else: 33 | return {'data': []} 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Cheskel Twersky 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /searchtube/str_find.py: -------------------------------------------------------------------------------- 1 | def list_find(heystack: list, needle: list): 2 | heystack_index = 0 3 | needle_index = 0 4 | results = [] 5 | for i in heystack: 6 | heystack_index +=1 7 | if i == needle[needle_index]: 8 | needle_index += 1 9 | else: 10 | needle_index = 0 11 | if needle_index == len(needle): 12 | results.append(list(range(heystack_index - needle_index, heystack_index))) 13 | needle_index = 0 14 | 15 | return results 16 | 17 | 18 | 19 | def split_find(heystack: str, needle: str): 20 | results = [] 21 | global_index = 0 22 | needle_space_len = needle.count(' ') 23 | needle_word_len = len(needle.split(' ')) 24 | data = heystack.split(needle) 25 | data_len = len(data) 26 | for i, result in enumerate(data): 27 | global_index += result.count(' ') 28 | if (not result.endswith(' ') and result) or (not data_len == i + 1 and not data[i + 1].startswith(' ')): 29 | continue 30 | results.append(list(range(global_index, global_index + needle_word_len))) 31 | global_index += needle_space_len 32 | return results 33 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | 4 | services: 5 | wsgi: 6 | build: ./ 7 | container_name: "searchtube_wsgi" 8 | volumes: 9 | - ./web:/var/www/searchtube/web 10 | - ./.data/channels:/var/www/searchtube/data 11 | - ./searchtube:/usr/local/lib/python3.7/site-packages/searchtube 12 | - /var/log/apache2/:/var/log/apache2/ 13 | - ./etc/apache2/sites-available/:/etc/apache2/sites-available/ 14 | - ./etc/letsencrypt/:/etc/letsencrypt/ 15 | # uncomment to mount test scripts in container 16 | # - ./tests:/tests 17 | ports: 18 | - "80:80" 19 | - "443:443" 20 | env_file: 21 | - .env 22 | 23 | mongo: 24 | image: mongo 25 | container_name: "searchtube_mongo" 26 | environment: 27 | MONGO_INITDB_ROOT_USERNAME: $DB_USERNAME 28 | MONGO_INITDB_ROOT_PASSWORD: $DB_PASSWORD 29 | volumes: 30 | - ./.data/db:/data/db:delegated 31 | 32 | # uncomment if you want MongoDB to be accessible outside of docker 33 | # ports: 34 | # - "27017:27017" 35 | -------------------------------------------------------------------------------- /web/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Searchtube 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 | 22 | Star 23 | 24 | 27 |
28 | 39 |
40 | 41 |
42 | 43 |
44 |
45 | 46 | 47 | -------------------------------------------------------------------------------- /web/static/js/jquery.nice-select.min.js: -------------------------------------------------------------------------------- 1 | !function(e){e.fn.niceSelect=function(t){function s(t){t.after(e("
").addClass("nice-select").addClass(t.attr("class")||"").addClass(t.attr("disabled")?"disabled":"").attr("tabindex",t.attr("disabled")?null:"0").html(''));var s=t.next(),n=t.find("option"),i=t.find("option:selected");s.find(".current").html(i.data("display")||i.text()),n.each(function(t){var n=e(this),i=n.data("display");s.find("ul").append(e("
  • ").attr("data-value",n.val()).attr("data-display",i||null).addClass("option"+(n.is(":selected")?" selected":"")+(n.is(":disabled")?" disabled":"")).html(n.text()))})}if("string"==typeof t)return"update"==t?this.each(function(){var t=e(this),n=e(this).next(".nice-select"),i=n.hasClass("open");n.length&&(n.remove(),s(t),i&&t.next().trigger("click"))}):"destroy"==t?(this.each(function(){var t=e(this),s=e(this).next(".nice-select");s.length&&(s.remove(),t.css("display",""))}),0==e(".nice-select").length&&e(document).off(".nice_select")):console.log('Method "'+t+'" does not exist.'),this;this.hide(),this.each(function(){var t=e(this);t.next().hasClass("nice-select")||s(t)}),e(document).off(".nice_select"),e(document).on("click.nice_select",".nice-select",function(t){var s=e(this);e(".nice-select").not(s).removeClass("open"),s.toggleClass("open"),s.hasClass("open")?(s.find(".option"),s.find(".focus").removeClass("focus"),s.find(".selected").addClass("focus")):s.focus()}),e(document).on("click.nice_select",function(t){0===e(t.target).closest(".nice-select").length&&e(".nice-select").removeClass("open").find(".option")}),e(document).on("click.nice_select",".nice-select .option:not(.disabled)",function(t){var s=e(this),n=s.closest(".nice-select");n.find(".selected").removeClass("selected"),s.addClass("selected");var i=s.data("display")||s.text();n.find(".current").text(i),n.prev("select").val(s.data("value")).trigger("change")}),e(document).on("keydown.nice_select",".nice-select",function(t){var s=e(this),n=e(s.find(".focus")||s.find(".list .option.selected"));if(32==t.keyCode||13==t.keyCode)return s.hasClass("open")?n.trigger("click"):s.trigger("click"),!1;if(40==t.keyCode){if(s.hasClass("open")){var i=n.nextAll(".option:not(.disabled)").first();i.length>0&&(s.find(".focus").removeClass("focus"),i.addClass("focus"))}else s.trigger("click");return!1}if(38==t.keyCode){if(s.hasClass("open")){var l=n.prevAll(".option:not(.disabled)").first();l.length>0&&(s.find(".focus").removeClass("focus"),l.addClass("focus"))}else s.trigger("click");return!1}if(27==t.keyCode)s.hasClass("open")&&s.trigger("click");else if(9==t.keyCode&&s.hasClass("open"))return!1});var n=document.createElement("a").style;return n.cssText="pointer-events:auto","auto"!==n.pointerEvents&&e("html").addClass("no-csspointerevents"),this}}(jQuery); -------------------------------------------------------------------------------- /web/static/css/index.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --color-dark: #2f2f2f; 3 | } 4 | 5 | body { 6 | font-family: 'Courier Prime'; 7 | background-color: #2e2f32; 8 | } 9 | 10 | .search-form { 11 | display: flex; 12 | justify-content: center; 13 | align-items: center; 14 | flex-wrap: wrap; 15 | margin: auto; 16 | } 17 | 18 | .search-form > * { 19 | margin: 15px; 20 | } 21 | 22 | .search { 23 | font-family: 'Courier Prime'; 24 | outline: 0; 25 | background: white; 26 | padding: 0 1.6rem; 27 | border-radius: 5px; 28 | appearance: none; 29 | height: 60px; 30 | width: 300px; 31 | font-size: 30px; 32 | border: none; 33 | box-shadow: 4px 4px 3px; 34 | margin: 30px; 35 | } 36 | 37 | .search-submit { 38 | height: 60px; 39 | box-shadow: 4px 4px 3px; 40 | border-radius: 5px; 41 | border: none; 42 | font-family: 'Courier Prime'; 43 | outline: 0; 44 | font-size: 30px; 45 | padding: 15px; 46 | cursor: pointer; 47 | } 48 | 49 | .search-submit:hover { 50 | background-color: aquamarine; 51 | } 52 | 53 | .main-container .nice-select { 54 | line-height: 60px; 55 | font-size: 30px; 56 | height: 60px; 57 | box-shadow: 4px 4px 3px; 58 | border: none; 59 | } 60 | 61 | .main-container .nice-select .list { 62 | font-size: 14px; 63 | } 64 | 65 | .main-container .nice-select .list { 66 | margin-top: 6px; 67 | } 68 | 69 | .menu { 70 | display: none; 71 | max-width: 40%; 72 | background: var(--color-dark); 73 | margin-top: 20px; 74 | border: 5px solid deepskyblue; 75 | border-radius: 15px; 76 | width: auto; 77 | height: auto; 78 | text-align: center; 79 | margin-left: auto; 80 | margin-right: auto; 81 | } 82 | 83 | .number-of-results { 84 | color: deepskyblue; 85 | padding: 20px; 86 | font-size: 20; 87 | } 88 | 89 | 90 | .main-container { 91 | width: 70%; 92 | margin: auto; 93 | } 94 | 95 | .search-container { 96 | padding-top: 40px; 97 | margin: auto; 98 | } 99 | 100 | .iframe { 101 | display: block; 102 | margin-left: auto; 103 | margin-right: auto; 104 | margin-top: 50px; 105 | } 106 | 107 | .results-container { 108 | padding: 0; 109 | text-align: center; 110 | } 111 | 112 | 113 | .load-more-button { 114 | margin: 30px; 115 | padding: 20px; 116 | font-size: 20px; 117 | font-family: 'Courier Prime'; 118 | color: black; 119 | background-color: aqua; 120 | border-radius: 10px; 121 | } 122 | 123 | 124 | @keyframes rotation { 125 | from { 126 | transform: rotate(0deg); 127 | } 128 | to { 129 | transform: rotate(359deg); 130 | } 131 | } 132 | 133 | .loading-png { 134 | animation: rotation 1s infinite linear; 135 | } 136 | 137 | 138 | @media screen and (max-width: 1000px) { 139 | .menu { 140 | max-width: 60%; 141 | } 142 | .main-container { 143 | width: 90%; 144 | } 145 | form { 146 | width: 90%; 147 | } 148 | .me , .twitter{ 149 | font-size: inherit; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /searchtube/download_subtitle.py: -------------------------------------------------------------------------------- 1 | from typing import Generator 2 | from youtube_dl import std_headers, YoutubeDL 3 | import webvtt 4 | import random 5 | import os 6 | import requests 7 | import time 8 | import list_youtube_channel 9 | from . import utils 10 | 11 | 12 | 13 | def get_videos(channel_id: str, channel_is_new: bool) -> Generator: 14 | limit = 20 if not channel_is_new else None 15 | videos = list_youtube_channel.get_channel(channel_id, limit = limit) 16 | return videos 17 | 18 | 19 | def download(channel_id: str, video_id: str) -> dict: 20 | unwanted = False 21 | subtitle_path = f'/var/www/searchtube/data/{channel_id}/{video_id}.en.vtt' 22 | video_data = get_video_info(video_id) 23 | 24 | if not video_data: 25 | # Might be age restricted 26 | unwanted = True 27 | 28 | if not os.path.exists(subtitle_path) and video_data: 29 | date = utils.date_to_epoch(video_data['upload_date']) 30 | subtitle_data = get_english_subtitles(video_data) 31 | 32 | if subtitle_data: 33 | print('Downloading subtitles for ' + video_id) 34 | download_subtitle(subtitle_data, subtitle_path) 35 | if bool(int(os.environ['SLEEP_AFTER_DOWNLOAD'])): 36 | print('Sleeping') 37 | sleep_interval = random.uniform(30, 60) 38 | time.sleep(sleep_interval) 39 | return {'path': subtitle_path, 'date': date} 40 | 41 | elif utils.is_two_weeks_old(date): 42 | unwanted = True 43 | 44 | if unwanted: 45 | utils.add_to_ignore(channel_id, video_id) 46 | return None 47 | 48 | 49 | 50 | def get_english_subtitles(raw_video_info: dict) -> dict: 51 | subtitles = {} 52 | english_subtitle = list(i for i in raw_video_info.get('subtitles', []) if 'en' in i) 53 | 54 | if english_subtitle: 55 | subtitles = raw_video_info['subtitles'][english_subtitle[0]] 56 | elif raw_video_info.get('automatic_captions', {}).get('en'): 57 | subtitles = raw_video_info['automatic_captions']['en'] 58 | 59 | return subtitles 60 | 61 | 62 | def download_subtitle(subtitle_data: dict, output_path: str) -> str: 63 | url = list(i['url'] for i in subtitle_data if i['ext'] == 'vtt')[0] 64 | 65 | success = False 66 | 67 | while not success: 68 | with open(output_path, 'wb') as f: 69 | f.write(requests.get(url, headers= std_headers).content) 70 | 71 | try: 72 | webvtt.read(output_path) 73 | success = True 74 | except webvtt.errors.MalformedCaptionError: 75 | sleep_interval = random.uniform(30 * 60, 45 * 60) 76 | print(f'Got reject from youtube, going to sleep for {int(sleep_interval // 60)} minutes') 77 | time.sleep(sleep_interval) 78 | continue 79 | 80 | return output_path 81 | 82 | 83 | def get_video_info(video_id: str) -> dict: 84 | youtube_dl_options = { 85 | 'skip_download': True, 86 | 'ignoreerrors': True 87 | } 88 | with YoutubeDL(youtube_dl_options) as ydl: 89 | video_data = ydl.extract_info(video_id) 90 | return video_data 91 | -------------------------------------------------------------------------------- /searchtube/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import string 4 | from . import db 5 | 6 | 7 | def timestamp_to_secs(timestamp: str) -> int: 8 | if '.' in timestamp: 9 | timestamp = timestamp.split('.')[0] 10 | result = int(str((datetime.datetime.strptime(timestamp, '%H:%M:%S') - datetime.datetime(1900, 1, 1)).total_seconds()).split('.')[0]) 11 | return result 12 | 13 | 14 | def date_to_epoch(date: str) -> int: 15 | return int((datetime.datetime.strptime(date, '%Y%m%d') - datetime.datetime(1970,1,1)).total_seconds()) 16 | 17 | 18 | def is_two_weeks_old(timestamp: int) -> bool: 19 | two_weeks_ago = int((datetime.datetime.now() - datetime.timedelta(days=13)).timestamp()) 20 | return timestamp < two_weeks_ago 21 | 22 | 23 | def add_to_ignore(channel_id: str, video_id: str) -> None: 24 | client = db.get_client() 25 | database = client.get_database(channel_id) 26 | ignore_coll = database.get_collection('ignore') 27 | if not ignore_coll.find_one({"video_id": video_id}): 28 | ignore_coll.insert_one({"video_id": video_id}) 29 | 30 | 31 | def add_channel(channel_id: str, name: str) -> None: 32 | prefix = '/var/www/searchtube/data' 33 | if not os.path.exists(f'{prefix}/{channel_id}/'): 34 | os.mkdir(f'{prefix}/{channel_id}/') 35 | 36 | client = db.get_client() 37 | database = client.get_database('searchtube') 38 | channels_coll = database.get_collection('channels') 39 | if not channels_coll.find_one({"channel_id": channel_id}): 40 | channels_coll.insert_one({"channel_id": channel_id, "channel_name": name, "is_new": True}) 41 | 42 | 43 | def remove_channel(channel_id: str) -> None: 44 | client = db.get_client() 45 | channels_database = client.get_database('searchtube') 46 | channels_coll = channels_database.get_collection('channels') 47 | if channels_coll.find_one({"channel_id": channel_id}): 48 | # Delete from config db 49 | channels_coll.delete_one({"channel_id": channel_id}) 50 | # Delete videos data 51 | client.drop_database(channel_id) 52 | 53 | 54 | def channel_is_in_db(channel_id: str) -> bool: 55 | client = db.get_client() 56 | database = client.get_database('searchtube') 57 | channels_coll = database.get_collection('channels') 58 | return bool(channels_coll.find_one({"channel_id": channel_id})) 59 | 60 | 61 | def get_channels() -> list: 62 | client = db.get_client() 63 | database = client.get_database('searchtube') 64 | channels_coll = database.get_collection('channels') 65 | return list(channels_coll.find()) 66 | 67 | 68 | def clean_vtt(data) -> list: 69 | results = [] 70 | last_lines = [] 71 | for caption in data: 72 | if all(bool(x.strip()) for x in caption.text.split('\n')): 73 | text_lines = caption.text.split('\n') 74 | text_lines = list(filter(lambda x: x not in last_lines, text_lines)) 75 | text = ' '.join(text_lines) 76 | caption.text = text 77 | last_lines = text_lines 78 | results.append(caption) 79 | 80 | return results 81 | 82 | 83 | def clean_text(text: str) -> str: 84 | words = text.split() 85 | cleaned_text_list = [] 86 | for word in words: 87 | word = word.lower() 88 | word = word.strip(string.punctuation) 89 | if word: 90 | cleaned_text_list.append(word) 91 | cleaned_text = ' '.join(cleaned_text_list) 92 | return cleaned_text 93 | -------------------------------------------------------------------------------- /searchtube/proccess_channel.py: -------------------------------------------------------------------------------- 1 | from pymongo.results import UpdateResult 2 | import webvtt 3 | from . import db, utils, download_subtitle 4 | 5 | 6 | 7 | def process(channel_id: str) -> None: 8 | is_new = channel_is_new(channel_id) 9 | videos = download_subtitle.get_videos(channel_id, is_new) 10 | for video in videos: 11 | video_id = video['videoId'] 12 | if video_is_new(channel_id, video_id): 13 | data = download_subtitle.download(channel_id, video_id) 14 | if data: 15 | save(channel_id, data['path'], data['date'], video_id) 16 | if is_new: 17 | set_channel_to_old(channel_id) 18 | 19 | 20 | 21 | 22 | def save(channel_id: str, path: str, date: int, video_id: str) -> None: 23 | client = db.get_client() 24 | database = client.get_database(channel_id) 25 | full_coll = database.get_collection('full_text') 26 | video_coll = database.get_collection(video_id) 27 | lines = webvtt.read(path) 28 | all_words = '' 29 | word_index = 0 30 | for caption in utils.clean_vtt(lines): 31 | text = utils.clean_text(caption.text) 32 | start = utils.timestamp_to_secs(caption.start) - 1 33 | end = utils.timestamp_to_secs(caption.end) + 1 34 | all_words += " " + text 35 | words_len = len(text.split(' ')) 36 | indexes = list(range(word_index, word_index + words_len)) 37 | data = { 38 | 'start': start, 39 | 'end': end, 40 | 'indexes': indexes 41 | } 42 | video_coll.insert_one(data) 43 | word_index += words_len 44 | data = { 45 | 'video_id': video_id, 46 | 'data': all_words.strip(), 47 | 'date': date, 48 | } 49 | full_coll.insert_one(data) 50 | full_coll.create_index([('data', 'text')]) 51 | video_coll.create_index('indexes') 52 | 53 | 54 | 55 | def video_is_new(channel_id: str, video_id: str) -> bool: 56 | client = db.get_client() 57 | database = client.get_database(channel_id) 58 | ignore_coll = database.get_collection('ignore') 59 | ignore = bool(ignore_coll.find_one({"video_id": video_id})) 60 | return video_id not in list(database.list_collection_names()) and not ignore 61 | 62 | 63 | def channel_is_new(channel_id: str) -> bool: 64 | client = db.get_client() 65 | database = client.get_database('searchtube') 66 | channels_coll = database.get_collection('channels') 67 | return bool(channels_coll.find_one({"channel_id": channel_id, "is_new": True})) 68 | 69 | 70 | def set_channel_to_old(channel_id: str) -> UpdateResult: 71 | client = db.get_client() 72 | database = client.get_database('searchtube') 73 | channels_coll = database.get_collection('channels') 74 | return channels_coll.update_one({"channel_id": channel_id}, {"$set": {"is_new": False}}) 75 | 76 | 77 | def reprocess_channel(channel_id: str) -> None: 78 | client = db.get_client() 79 | database = client.get_database(channel_id) 80 | full_coll = database.get_collection('full_text') 81 | videos = list({'video_id': i['video_id'], 'date': i['date']} for i in full_coll.find()) 82 | for video in videos: 83 | video_id = video['video_id'] 84 | print(f'Now reprocessing video: {video_id}') 85 | date = video['date'] 86 | subtitle_path = f'/var/www/searchtube/data/{channel_id}/{video_id}.en.vtt' 87 | full_coll.delete_one({'video_id': video_id}) 88 | database.drop_collection(video_id) 89 | save(channel_id, subtitle_path, date, video_id) 90 | 91 | 92 | def reprocess_channels() -> None: 93 | for channel in utils.get_channels(): 94 | print(f'Now reprocessing channel: {channel["channel_id"]}') 95 | reprocess_channel(channel['channel_id']) 96 | -------------------------------------------------------------------------------- /web/static/js/index.js: -------------------------------------------------------------------------------- 1 | var index = 0 2 | var data = [] 3 | var numOfresults = 0 4 | 5 | 6 | 7 | function getData(url, params, callback) { 8 | onSearch() 9 | fetch(url).then((resp) => { 10 | resp.json().then((data) => { 11 | document.getElementById('loading-png').remove() 12 | callback(data, params) 13 | }) 14 | }) 15 | } 16 | 17 | function onSearch() { 18 | index = 0 19 | let resultsContainer = document.getElementById('results-container') 20 | resultsContainer.innerHTML = '' 21 | // loading 22 | let img = document.createElement('img') 23 | img.id = 'loading-png' 24 | img.className = 'loading-png' 25 | img.src = 'static/images/loading.png' 26 | resultsContainer.appendChild(img) 27 | const menu = document.getElementById('menu') 28 | menu.innerHTML = '' 29 | let results = document.createElement('div') 30 | results.id = 'results' 31 | resultsContainer.appendChild(results) 32 | let button = document.createElement('button') 33 | button.innerHTML = 'Load More' 34 | button.className = 'load-more-button' 35 | resultsContainer.appendChild(button) 36 | 37 | button.addEventListener('click', (e) => { 38 | addFive() 39 | }) 40 | } 41 | 42 | function displayInfo(searchTerm) { 43 | const menu = document.getElementById('menu') 44 | menu.style.display = 'block' 45 | const num = document.createElement('p') 46 | num.className = 'number-of-results' 47 | num.innerText = `Total results: ${data.length} | Query: ${searchTerm}` 48 | menu.appendChild(num) 49 | } 50 | 51 | 52 | function addFive() { 53 | for (let i = index + 5; index < i && index < numOfresults; index++) { 54 | results = document.getElementById('results') 55 | let iframe = document.createElement('iframe') 56 | iframe.className = 'iframe' 57 | if (screen.width > 750) { 58 | iframe.style.width = '495px' 59 | iframe.style.height = '280px' 60 | } else { 61 | iframe.style.width = '320px' 62 | iframe.style.height = '180px' 63 | } 64 | 65 | iframe.src = data[index] 66 | results.appendChild(iframe) 67 | } 68 | if (numOfresults == index) { 69 | let button = document.getElementsByClassName('load-more-button')[0] 70 | button.style.display = 'none' 71 | } 72 | } 73 | 74 | function handleData(rawData, params) { 75 | data = rawData.data 76 | numOfresults = data.length 77 | window.history.pushState({}, '', '/?q=' + params.term + '&channel_id=' + params.channelId); 78 | displayInfo(params.term) 79 | addFive() 80 | } 81 | 82 | 83 | function runInit() { 84 | let currentUrl = new URL(window.location.href) 85 | let channelId = currentUrl.searchParams.get('channel_id') 86 | searchTerm = currentUrl.searchParams.get('q') 87 | if (channelId) { 88 | $('select').val(channelId).niceSelect('update') 89 | } 90 | if (searchTerm) { 91 | params = getCurrentQuery() 92 | params.term = searchTerm 93 | let url = '/search?q=' + params.term + '&channel_id=' + params.channelId 94 | getData(url, params, handleData) 95 | } 96 | } 97 | 98 | 99 | function getCurrentQuery() { 100 | let q = document.getElementById('search') 101 | let searchTerm = q.value; 102 | q.value = '' 103 | let channel = document.getElementById('channel-select') 104 | let channelId 105 | if (channel) { 106 | channelId = channel.value 107 | } else { 108 | channelId = 0 109 | } 110 | params = { 111 | term: searchTerm.toLowerCase(), 112 | channelId: channelId 113 | } 114 | return params 115 | } 116 | 117 | $(document).ready(function() { 118 | $('select').niceSelect() 119 | document.getElementById('search-form').addEventListener("submit", (e) => { 120 | e.preventDefault(); 121 | params = getCurrentQuery() 122 | let url = '/search?q=' + params.term + '&channel_id=' + params.channelId 123 | getData(url, params, handleData) 124 | }); 125 | runInit() 126 | }); 127 | -------------------------------------------------------------------------------- /web/static/css/nice-select.css: -------------------------------------------------------------------------------- 1 | .nice-select { 2 | -webkit-tap-highlight-color: transparent; 3 | background-color: #fff; 4 | border-radius: 5px; 5 | border: solid 1px #e8e8e8; 6 | box-sizing: border-box; 7 | clear: both; 8 | cursor: pointer; 9 | display: block; 10 | float: left; 11 | font-family: inherit; 12 | font-size: 14px; 13 | font-weight: normal; 14 | height: 42px; 15 | line-height: 40px; 16 | outline: none; 17 | padding-left: 18px; 18 | padding-right: 30px; 19 | position: relative; 20 | text-align: left !important; 21 | -webkit-transition: all 0.2s ease-in-out; 22 | transition: all 0.2s ease-in-out; 23 | -webkit-user-select: none; 24 | -moz-user-select: none; 25 | -ms-user-select: none; 26 | user-select: none; 27 | white-space: nowrap; 28 | width: auto; } 29 | .nice-select:hover { 30 | border-color: #dbdbdb; } 31 | .nice-select:active, .nice-select.open, .nice-select:focus { 32 | border-color: #999; } 33 | .nice-select:after { 34 | border-bottom: 2px solid #999; 35 | border-right: 2px solid #999; 36 | content: ''; 37 | display: block; 38 | height: 5px; 39 | margin-top: -4px; 40 | pointer-events: none; 41 | position: absolute; 42 | right: 12px; 43 | top: 50%; 44 | -webkit-transform-origin: 66% 66%; 45 | -ms-transform-origin: 66% 66%; 46 | transform-origin: 66% 66%; 47 | -webkit-transform: rotate(45deg); 48 | -ms-transform: rotate(45deg); 49 | transform: rotate(45deg); 50 | -webkit-transition: all 0.15s ease-in-out; 51 | transition: all 0.15s ease-in-out; 52 | width: 5px; } 53 | .nice-select.open:after { 54 | -webkit-transform: rotate(-135deg); 55 | -ms-transform: rotate(-135deg); 56 | transform: rotate(-135deg); } 57 | .nice-select.open .list { 58 | opacity: 1; 59 | pointer-events: auto; 60 | -webkit-transform: scale(1) translateY(0); 61 | -ms-transform: scale(1) translateY(0); 62 | transform: scale(1) translateY(0); } 63 | .nice-select.disabled { 64 | border-color: #ededed; 65 | color: #999; 66 | pointer-events: none; } 67 | .nice-select.disabled:after { 68 | border-color: #cccccc; } 69 | .nice-select.wide { 70 | width: 100%; } 71 | .nice-select.wide .list { 72 | left: 0 !important; 73 | right: 0 !important; } 74 | .nice-select.right { 75 | float: right; } 76 | .nice-select.right .list { 77 | left: auto; 78 | right: 0; } 79 | .nice-select.small { 80 | font-size: 12px; 81 | height: 36px; 82 | line-height: 34px; } 83 | .nice-select.small:after { 84 | height: 4px; 85 | width: 4px; } 86 | .nice-select.small .option { 87 | line-height: 34px; 88 | min-height: 34px; } 89 | .nice-select .list { 90 | background-color: #fff; 91 | border-radius: 5px; 92 | box-shadow: 0 0 0 1px rgba(68, 68, 68, 0.11); 93 | box-sizing: border-box; 94 | margin-top: 4px; 95 | opacity: 0; 96 | overflow: hidden; 97 | padding: 0; 98 | pointer-events: none; 99 | position: absolute; 100 | top: 100%; 101 | left: 0; 102 | -webkit-transform-origin: 50% 0; 103 | -ms-transform-origin: 50% 0; 104 | transform-origin: 50% 0; 105 | -webkit-transform: scale(0.75) translateY(-21px); 106 | -ms-transform: scale(0.75) translateY(-21px); 107 | transform: scale(0.75) translateY(-21px); 108 | -webkit-transition: all 0.2s cubic-bezier(0.5, 0, 0, 1.25), opacity 0.15s ease-out; 109 | transition: all 0.2s cubic-bezier(0.5, 0, 0, 1.25), opacity 0.15s ease-out; 110 | z-index: 9; } 111 | .nice-select .list:hover .option:not(:hover) { 112 | background-color: transparent !important; } 113 | .nice-select .option { 114 | cursor: pointer; 115 | font-weight: 400; 116 | line-height: 40px; 117 | list-style: none; 118 | min-height: 40px; 119 | outline: none; 120 | padding-left: 18px; 121 | padding-right: 29px; 122 | text-align: left; 123 | -webkit-transition: all 0.2s; 124 | transition: all 0.2s; } 125 | .nice-select .option:hover, .nice-select .option.focus, .nice-select .option.selected.focus { 126 | background-color: #f6f6f6; } 127 | .nice-select .option.selected { 128 | font-weight: bold; } 129 | .nice-select .option.disabled { 130 | background-color: transparent; 131 | color: #999; 132 | cursor: default; } 133 | 134 | .no-csspointerevents .nice-select .list { 135 | display: none; } 136 | 137 | .no-csspointerevents .nice-select.open .list { 138 | display: block; } 139 | -------------------------------------------------------------------------------- /web/static/js/jquery-3.6.0.min.js: -------------------------------------------------------------------------------- 1 | /*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ 2 | !function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
    "],col:[2,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
    ",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0