├── .env ├── .env.test ├── .eslintrc.json ├── .gitignore ├── .travis.yml ├── Dockerfile ├── Dockerfile.test ├── Procfile ├── docker-compose.test.yml ├── docker-compose.yml ├── main.py ├── mypy.ini ├── readme.md ├── requirements.txt └── ytdl ├── __init__.py ├── app.py ├── download_api.py ├── flask_rq.py ├── models.py ├── paginator.py ├── settings.py ├── static ├── css │ ├── font-awesome.min.css │ └── foundation.min.css ├── fonts │ ├── FontAwesome.otf │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.svg │ ├── fontawesome-webfont.ttf │ └── fontawesome-webfont.woff ├── img │ ├── blank.gif │ ├── glyphicons-halflings-white.png │ ├── glyphicons-halflings.png │ └── spinner.gif ├── js │ ├── JSXTransformer-0.12.0.js │ ├── director.min.js │ ├── jquery-1.10.0.min.js │ ├── moment.min.js │ ├── mousetrap.min.js │ ├── react-with-addons-0.12.1.js │ └── react-with-addons-0.12.1.min.js ├── ytdl.jsx └── ytdl2.html ├── tasks.py ├── test_service_api.py ├── vimeo_api.py └── youtube_api.py /.env: -------------------------------------------------------------------------------- 1 | REDIS_HOST=redis 2 | REDIS_PORT=6379 3 | REDIS_URL=redis://redis:6379/0 4 | YTDL_DB_PATH=/data/db.sqlite3 5 | PYTHONUNBUFFERED=1 6 | -------------------------------------------------------------------------------- /.env.test: -------------------------------------------------------------------------------- 1 | REDIS_HOST=redis 2 | REDIS_PORT=6379 3 | REDIS_URL=redis://redis:6379/0 4 | YTDL_DB_PATH=/data/ytdl_test.sqlite3 5 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es6": true 5 | }, 6 | "extends": "eslint:recommended", 7 | "installedESLint": true, 8 | "parserOptions": { 9 | "ecmaFeatures": { 10 | "experimentalObjectRestSpread": true, 11 | "jsx": true 12 | } 13 | }, 14 | "plugins": [ 15 | "react" 16 | ], 17 | "rules": { 18 | "indent": [ 19 | "error", 20 | 4 21 | ], 22 | "linebreak-style": [ 23 | "error", 24 | "unix" 25 | ], 26 | "quotes": [ 27 | "error", 28 | "double" 29 | ], 30 | "semi": [ 31 | "error", 32 | "always" 33 | ], 34 | "no-unused-vars": [ 35 | 0 36 | ] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.sqlite3 4 | celeryev.pid 5 | node_modules/* 6 | /.mypy_cache 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: required 3 | services: 4 | - docker 5 | 6 | script: 7 | - docker-compose -f docker-compose.test.yml -p ci up --build --exit-code-from test 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.3-alpine3.4 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt /app/requirements.txt 6 | RUN pip install --no-cache-dir -r requirements.txt 7 | RUN pip install --no-cache-dir pytest 8 | 9 | RUN addgroup -g 1000 -S app && adduser -S -G app app -u 1000 10 | USER app 11 | 12 | COPY . /app 13 | 14 | EXPOSE 8008 15 | CMD ["env", "PYTHONUNBUFFERED=1", "honcho", "start", "-c", "taskdl=4"] 16 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM python:3.6.3-alpine3.4 2 | 3 | WORKDIR /app 4 | 5 | RUN set -ex \ 6 | && apk add --no-cache --virtual .build-deps \ 7 | gcc \ 8 | musl-dev \ 9 | python3-dev \ 10 | && python3 -m pip install --no-cache-dir mypy \ 11 | && apk del .build-deps 12 | 13 | COPY requirements.txt /app/requirements.txt 14 | RUN pip install --no-cache-dir -r requirements.txt 15 | RUN pip install --no-cache-dir pytest 16 | 17 | COPY . /app 18 | 19 | RUN mkdir /data 20 | CMD ["sh", "-c", "python main.py dbinit && mypy ytdl/ && pytest"] 21 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: python main.py server --host 0.0.0.0 --port 8008 2 | 3 | task: env PYTHONUNBUFFERED=1 rqworker ytdl-default --url $REDIS_URL 4 | taskdl: env PYTHONUNBUFFERED=1 rqworker ytdl-download --url $REDIS_URL 5 | 6 | scheduler: env PYTHONUNBUFFERED=1 python main.py scheduler 7 | -------------------------------------------------------------------------------- /docker-compose.test.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | app: 6 | env_file: .env.test 7 | build: . 8 | depends_on: 9 | - redis 10 | 11 | test: 12 | env_file: .env.test 13 | build: 14 | context: . 15 | dockerfile: Dockerfile.test 16 | links: 17 | - app 18 | 19 | redis: 20 | image: redis:3.2-alpine 21 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | app: 6 | restart: unless-stopped 7 | env_file: .env 8 | build: . 9 | depends_on: 10 | - redis 11 | ports: 12 | - "8008:8008" 13 | volumes: 14 | - /opt/ytdl/localdata:/data 15 | - .:/app 16 | restart: always 17 | logging: 18 | options: 19 | max-size: 50m 20 | 21 | redis: 22 | restart: unless-stopped 23 | image: redis:3.2-alpine 24 | volumes: 25 | - /opt/ytdl/redisdata:/data 26 | 27 | volumes: 28 | redis_data: 29 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import json 5 | import datetime 6 | 7 | 8 | def _scheduler_run(on_start): 9 | import ytdl.tasks 10 | minutes = 60 11 | 12 | first_run = True 13 | while True: 14 | if not on_start or not first_run: 15 | print("%s Sleeping %s minutes" % (datetime.datetime.now(), minutes)) 16 | time.sleep(60*minutes) 17 | first_run = False 18 | 19 | print("%s Refreshing!" % datetime.datetime.now()) 20 | ytdl.tasks.refresh_all_channels() 21 | 22 | 23 | def scheduler(on_start): 24 | try: 25 | return _scheduler_run(on_start=on_start) 26 | except KeyboardInterrupt: 27 | return 28 | 29 | 30 | def refresh(limit, all, filter): 31 | import ytdl.models 32 | channels = ytdl.models.Channel.select() 33 | for c in channels: 34 | if filter is not None and filter not in c.title.lower(): 35 | continue 36 | print("Force-refreshing %s" % c) 37 | if all: 38 | c.grab(limit=limit, stop_on_existing=False) 39 | else: 40 | c.grab(limit=limit) 41 | 42 | 43 | def dedupe(kill): 44 | import ytdl.models 45 | 46 | if not kill: 47 | print("Dry-run, specify --kill to delete dupes") 48 | 49 | seen = set() 50 | for video in ytdl.models.Video.select(): 51 | # TODO: maybe only check for dupes in channel? Utterly unlikely 52 | url = video.url 53 | if url in seen: 54 | if kill: 55 | print("Deleting dupe %s (%s on %s)" % (url, video.status, video.channel.title)) 56 | video.delete_instance() 57 | else: 58 | print("Dupe %s (%s on %s)" % (url, video.status, video.channel.title)) 59 | else: 60 | seen.add(url) 61 | 62 | def cleanup(): 63 | """Deletes videos where the associated channel no longer exists 64 | """ 65 | import ytdl.models 66 | for v in ytdl.models.Video.select(): 67 | try: 68 | assert v.channel 69 | except ytdl.models.Channel.DoesNotExist: 70 | print("Deleting orphaned video '%s'" % v.title) 71 | v.delete_instance() 72 | 73 | def backup(filename): 74 | if filename is None: 75 | f = sys.stdout 76 | else: 77 | f = open(filename + ".tmp", "w+") 78 | 79 | 80 | import ytdl.models 81 | 82 | channels = [] 83 | for c in ytdl.models.Channel.select(): 84 | chaninfo = {'chanid': c.chanid, 85 | 'service': c.service, 86 | 'videos': [] 87 | } 88 | for v in ytdl.models.Video.select().where(ytdl.models.Video.channel == c): 89 | chaninfo['videos'].append( 90 | {'title': v.title, 91 | 'url': v.url, 92 | 'videoid': v.videoid, 93 | 'status': v.status, 94 | 'publishdate': v.status, 95 | 'service': c.service, 96 | }) 97 | 98 | channels.append(chaninfo) 99 | 100 | json.dump(channels, f, 101 | sort_keys=True, indent=1, separators=(',', ': ')) 102 | f.write("\n") # Trailing new line in file (or stdout) 103 | 104 | if filename is not None: 105 | os.rename(filename + ".tmp", filename) 106 | 107 | 108 | def restore(filename): 109 | if filename is None: 110 | f = sys.stdin 111 | else: 112 | f = open(filename) 113 | 114 | import ytdl.models 115 | 116 | with ytdl.models.database.transaction(): 117 | all_chan = json.load(f) 118 | for channel in all_chan: 119 | db_chan = ytdl.models.Channel.get(chanid=channel['chanid']) 120 | if db_chan is None: 121 | print("Creating %s (service %s)" % (channel['chanid'], channel['service'])) 122 | db_chan = ytdl.models.Channel(chanid = channel['chanid'], service=channel['service']) 123 | db_chan.save() 124 | 125 | # Get videos form channel 126 | print("Getting videos for %s" % (db_chan)) 127 | db_chan.grab() 128 | 129 | # Restore statuses 130 | print("Restore statuses") 131 | for video in channel['videos']: 132 | v = ytdl.models.Video.get(videoid=video['videoid']) 133 | if v is None: 134 | print("%s does not exist (title: %s)" % (video['videoid'], video['title'])) 135 | continue # Next video 136 | v.status = video['status'] 137 | v.save() 138 | 139 | 140 | 141 | def server(port, host): 142 | from ytdl.app import app 143 | app.debug=True 144 | app.run(host=host, port=port) 145 | 146 | 147 | def dbinit(): 148 | import ytdl.models 149 | ytdl.models.database.connect() 150 | ytdl.models.Channel.create_table() 151 | ytdl.models.Video.create_table() 152 | 153 | 154 | if __name__ == '__main__': 155 | import argparse 156 | p_main = argparse.ArgumentParser() 157 | subparsers = p_main.add_subparsers() 158 | 159 | p_server = subparsers.add_parser('server') 160 | p_server.add_argument('-o', '--host', default='0.0.0.0') 161 | p_server.add_argument('-p', '--port', default=8008, type=int) 162 | p_server.set_defaults(func=server) 163 | 164 | p_server = subparsers.add_parser('dbinit') 165 | p_server.set_defaults(func=dbinit) 166 | 167 | p_refresh = subparsers.add_parser('refresh') 168 | p_refresh.set_defaults(func=refresh) 169 | p_refresh.add_argument("-a", "--all", action="store_true", help="don't stop because a video exists (check for older videos)") 170 | p_refresh.add_argument("--limit", type=int, default=1000, help="maximum number of videos to try and grab (default %(default)s)") 171 | p_refresh.add_argument("-f", "--filter", default=None, help="only refresh channels matching this (simple case-insensetive string-matching on channel title)") 172 | 173 | 174 | p_dedupe = subparsers.add_parser('dedupe') 175 | p_dedupe.set_defaults(func=dedupe) 176 | p_dedupe.add_argument('--kill', default=False, action="store_true") 177 | 178 | p_scheduler = subparsers.add_parser('scheduler') 179 | p_scheduler.set_defaults(func=scheduler) 180 | p_scheduler.add_argument('--on-start', action="store_true", help="perform refresh immediately (instead of after delay)") 181 | 182 | p_backup = subparsers.add_parser('backup') 183 | p_backup.set_defaults(func=backup) 184 | p_backup.add_argument("-f", "--filename", default=None) 185 | 186 | p_restore = subparsers.add_parser('restore') 187 | p_restore.set_defaults(func=restore) 188 | p_restore.add_argument("-f", "--filename", default=None) 189 | 190 | p_cleanup = subparsers.add_parser('cleanup') 191 | p_cleanup.set_defaults(func=cleanup) 192 | 193 | args = p_main.parse_args() 194 | func = args.func 195 | funcargs = {k:v for k, v in vars(args).items() if k != 'func'} 196 | func(**funcargs) 197 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | ignore_missing_imports = True 3 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # `ytdl` 2 | 3 | [![Build Status](https://travis-ci.org/dbr/webtools.png?branch=master)](https://travis-ci.org/dbr/webtools) [![Requirements Status](https://requires.io/github/dbr/webtools/requirements.png?branch=master)](https://requires.io/github/dbr/webtools/requirements/?branch=master) 4 | 5 | A web interface where you can add any number of Youtube or Vimeo channels. Videos from these channels are then cleanly listed, and can be downloaded with a single click (using [`youtube-dl`][youtube-dl]). 6 | 7 | Multiple downloads will be queued up (using the [python-rq][python-rq] task queue), and it keeps track of the state of videos (new, downloaded, ignored) 8 | 9 | So you can quickly see what new videos have been released on various channels, click the "download" button on the interesting ones, and end up with a folder of `.mp4` files to watch later (or, transfer onto an iPad to something like AVPlayerHD etc etc). 10 | 11 | Very little noise (no Youtube comments, no annotations, no pre-roll and overlayed-banner ads), no buffering. 12 | 13 | Some probably-outdated screenshots: 14 | 15 | * [List of all channels](http://i.imgur.com/1v5WVW8.png) 16 | * [Viewing a channel](http://i.imgur.com/1RPHbuM.png) 17 | 18 | ## Installation 19 | 20 | Docker is the preferred means of installation. 21 | 22 | 1. Adjust paths in `docker-compose.yml` as necessary. 23 | 24 | In the `app` container, `/data` contains the SQLite database and downloaded files. `/app` contains the code. In the `redis` container, there is a `/data` container used for the download queue data. 25 | 26 | 2. Start via `docker-compose`: 27 | 28 | docker-compose up --build --detatch 29 | 30 | This will run in the background (because `--detatch`) 31 | 32 | 3. Initialize database (only necessary on first run) 33 | 34 | docker-compose exec app python3 main.py dbinit 35 | 36 | 4. To stop: 37 | 38 | docker-compose down 39 | 40 | ## Running tests 41 | 42 | Tests are run via docker-compose similarly to the main application. 43 | 44 | docker-compose -f docker-compose.test.yml up --build --exit-code-from test 45 | 46 | ## Random notes: 47 | 48 | * Currently will always download the highest quality video possible, 49 | and the files are potentially quite large (some 30-40 minute videos 50 | are ~1GB). The `--max-quality` flag to `youtube-dl` might be worth 51 | using 52 | * When adding a Vimeo user, only the first "3 pages" (about 60 videos) 53 | will be listed currently, due to using the 54 | ["simple API"](http://developer.vimeo.com/apis/simple). This 55 | restriction could be removed by registering for an API key and using 56 | the full API (which requires OAuth) 57 | * The web interface does what I needed and no more. It's not the 58 | fanciest thing ever, but functional. 59 | 60 | 61 | [youtube-dl]: http://rg3.github.io/youtube-dl/ 62 | [python-rq]: http://python-rq.org/ 63 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.1.1 2 | Jinja2==2.11.1 3 | MarkupSafe==1.1.1 4 | Werkzeug==1.0.0 5 | honcho==1.0.1 6 | itsdangerous==1.1.0 7 | peewee==3.13.1 8 | py==1.8.1 9 | pytest==5.3.5 10 | python-dateutil==2.8.1 11 | pytz==2019.3 12 | redis==3.4.1 13 | requests==2.22.0 14 | rq-dashboard==0.6.1 15 | rq==1.2.2 16 | simplejson==3.17.0 17 | six==1.14.0 18 | times==0.7 19 | youtube-dl==2020.02.16 20 | mypy_extensions==0.4.3 -------------------------------------------------------------------------------- /ytdl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/__init__.py -------------------------------------------------------------------------------- /ytdl/app.py: -------------------------------------------------------------------------------- 1 | import json 2 | from flask import Flask 3 | from flask import abort, redirect, url_for, send_file 4 | from ytdl.paginator import Paginator, PageNotAnInteger, EmptyPage 5 | 6 | import ytdl.settings 7 | import ytdl.tasks 8 | from ytdl.models import Video, Channel 9 | 10 | import rq_dashboard 11 | 12 | 13 | app = Flask(__name__) 14 | 15 | app.config['RQ_DEFAULT_HOST'] = ytdl.settings.REDIS_HOST 16 | app.config['RQ_DEFAULT_PORT'] = ytdl.settings.REDIS_PORT 17 | app.config['RQ_DEFAULT_DB'] = 1 18 | 19 | app.config['REDIS_HOST'] = ytdl.settings.REDIS_HOST 20 | app.config['REDIS_PORT'] = ytdl.settings.REDIS_PORT 21 | 22 | 23 | # Setup rq dashboard 24 | app.config.from_object(rq_dashboard.default_settings) 25 | app.register_blueprint(rq_dashboard.blueprint, url_prefix="/rq") 26 | 27 | 28 | # Web page 29 | @app.route("/") 30 | def index(): 31 | return redirect("/youtube/", code=302) 32 | 33 | 34 | @app.route("/youtube/") 35 | def page2(): 36 | return send_file("static/ytdl2.html") 37 | 38 | 39 | # API 40 | 41 | def _channel_info_dict(c): 42 | return { 43 | 'id': c.id, 44 | 'title': c.title, 45 | 'service': c.service, 46 | 'chanid': c.chanid, 47 | 'icon': c.icon_url, 48 | 'last_refresh': str(c.last_refresh), 49 | 'last_content_update': str(c.last_update_content), 50 | 'last_meta_update': str(c.last_update_meta), 51 | } 52 | 53 | 54 | @app.route("/youtube/api/1/refresh") 55 | def refresh(): 56 | chanid = request.args.get("channel") 57 | if chanid == "_all": 58 | ytdl.tasks.refresh_all_channels() 59 | return json.dumps({"message": "refreshing all channels"}) 60 | else: 61 | chan = Channel.get(id=chanid) 62 | if chan is None: 63 | return json.dumps({"error": "so such channel"}), 404 64 | ytdl.tasks.refresh_channel.delay(id=chan.id) 65 | return json.dumps({"message": "refreshing channel %s (%s)" % (chan.id, chan.title)}) 66 | 67 | from flask import request 68 | @app.route("/youtube/api/1/channels") 69 | def list_channels(): 70 | page = int(request.args.get("page", "1")) 71 | count = request.args.get("count") 72 | 73 | def offset(sliceable, page, count): 74 | start = (page - 1) * count 75 | end = page * count 76 | return sliceable[start:end] 77 | 78 | query = Channel.select().order_by(Channel.title.asc()) 79 | if count is not None: 80 | count = int(count) 81 | query = offset(query, page, count) 82 | 83 | channels = [] 84 | for c in query: 85 | channels.append(_channel_info_dict(c)) 86 | return json.dumps({'channels': channels, 'total': Channel.select().count()}) 87 | 88 | 89 | @app.route("/youtube/api/1/channels/") 90 | def channel_details(chanid): 91 | if chanid == "_all": 92 | query = Video.select() 93 | else: 94 | chan = Channel.get(id=chanid) 95 | query = Video.select().filter(channel = chan) 96 | 97 | query = query.order_by(Video.publishdate.desc()) 98 | search = request.args.get('search', "") 99 | 100 | if len(search) > 0: 101 | query = query.where(Video.title.contains(search)) 102 | 103 | # Query based on status 104 | status = request.args.get('status', "") 105 | if len(status) > 0: 106 | status = status.strip().split(",") 107 | x = Video.status == status[0] 108 | for st in status[1:]: 109 | x = x | (Video.status == st) 110 | query = query.where(x) 111 | 112 | # 25 videos per page, with no less than 5 per page 113 | paginator = Paginator(query, per_page=25, orphans=5) 114 | 115 | # Get page parameter 116 | page_num = request.args.get('page', '1') 117 | if int(page_num) < 1: 118 | page_num = 1 119 | 120 | try: 121 | page = paginator.page(page_num) 122 | except PageNotAnInteger: 123 | page = paginator.page(1) 124 | except EmptyPage: 125 | page = paginator.page(paginator.num_pages) 126 | 127 | out_videos = [] 128 | for v in page: 129 | out_videos.append({ 130 | 'id': v.id, 131 | 'title': v.title, 132 | 'imgs': v.img, 133 | 'url': v.url, 134 | 'description': v.description, 135 | 'publishdate': str(v.publishdate), 136 | 'status': v.status, 137 | # FIXME: Data duplication, only used for "all" channel view 138 | 'channel': _channel_info_dict(v.channel), 139 | }) 140 | 141 | if chanid == '_all': 142 | channel = None 143 | else: 144 | channel = _channel_info_dict(chan) 145 | 146 | page_info = { 147 | 'total': paginator.num_pages, 148 | 'current': page.number, 149 | 'has_next': page.has_next(), 150 | 'has_previous': page.has_previous(), 151 | } 152 | 153 | return json.dumps( 154 | {'channel': channel, 155 | 'videos': out_videos, 156 | 'pagination': page_info}) 157 | 158 | 159 | 160 | @app.route("/youtube/api/1/video//grab") 161 | def grab(videoid): 162 | """For a given video ID, enqueue the task to download it 163 | """ 164 | 165 | video = Video.get(id=videoid) 166 | if video is None: 167 | abort(404) 168 | 169 | force = request.args.get("force", "false").lower() == "true" 170 | 171 | grabbable = video.status in [Video.STATE_NEW, Video.STATE_GRAB_ERROR, Video.STATE_IGNORE] 172 | if not grabbable and not force: 173 | ret = {"error": "Already grabbed (status %s)" % (video.status)} 174 | return json.dumps(ret), 500 175 | 176 | ytdl.tasks.grab_video.delay(video.id, force=force) 177 | 178 | video.status = Video.STATE_QUEUED 179 | video.save() 180 | 181 | return json.dumps({"status": video.status}) 182 | 183 | 184 | # Set status 185 | 186 | def _set_status(videoid, status): 187 | video = Video.get(id=videoid) 188 | if video is None: 189 | abort(404) 190 | 191 | video.status = status 192 | video.save() 193 | return json.dumps({"status": video.status}) 194 | 195 | 196 | @app.route('/youtube/api/1/video//mark_viewed') 197 | def mark_viewed(videoid): 198 | return _set_status(videoid, Video.STATE_GRABBED) 199 | 200 | 201 | @app.route('/youtube/api/1/video//mark_ignored') 202 | def mark_ignored(videoid): 203 | return _set_status(videoid, status=Video.STATE_IGNORE) 204 | 205 | 206 | # Query status 207 | 208 | @app.route('/youtube/api/1/video_status') 209 | def video_status(): 210 | ids = request.args.get("ids") 211 | if ids is None or len(ids) == 0: 212 | return "{}" 213 | 214 | ids = ids.split(",") 215 | videos = {} 216 | for cur in ids: 217 | v = Video.get(id=cur) 218 | if v is None: 219 | abort(404) 220 | videos[int(cur)] = v.status 221 | 222 | return json.dumps(videos) 223 | 224 | 225 | @app.route('/youtube/api/1/downloads') 226 | def downloads(): 227 | import redis 228 | r = redis.Redis(host=ytdl.settings.REDIS_HOST, port=ytdl.settings.REDIS_PORT) 229 | 230 | ids = r.smembers("dl") or [] 231 | 232 | all_info = {} 233 | for cur in ids: 234 | cur = cur.decode("utf-8") 235 | key = "dl:{id}:info".format(id=cur) 236 | 237 | status = (r.hget(key, "status") or b"").decode("utf-8") 238 | message = (r.hget(key, "message") or b"").decode("utf-8") 239 | progress = float(r.hget(key, "progress") or 0.0) 240 | 241 | try: 242 | v = Video.get(id=cur) 243 | except Video.DoesNotExist: 244 | title = "Unknown" 245 | else: 246 | title = v.title 247 | 248 | all_info[cur] = { 249 | 'id': cur, 250 | 'title': title, 251 | 'status': status, 252 | 'message': message, 253 | 'progress': progress, 254 | } 255 | 256 | return json.dumps(all_info) 257 | 258 | 259 | # Add channel 260 | 261 | @app.route("/youtube/api/1/channel_add", methods=["POST"]) 262 | def add_channel(): 263 | service = request.form.get("service") 264 | chanid = request.form.get("chanid") 265 | 266 | if service is None or chanid is None: 267 | ret = {"error": "Both 'chanid' (%r) and 'service' (%r) must be specified" % (service, chanid)} 268 | return json.dumps(ret), 500 269 | 270 | if service not in ytdl.models.ALL_SERVICES: 271 | ret = {"error": "service must be one of %s" % ", ".join(ytdl.models.ALL_SERVICES)} 272 | return json.dumps(ret), 500 273 | 274 | try: 275 | existing_chan = Channel.get(chanid=chanid) 276 | except Channel.DoesNotExist: 277 | pass # Good 278 | else: 279 | # Exists 280 | ret = {"error": "channel %r already exists (on service %s)" % (existing_chan.chanid, existing_chan.service)} 281 | return json.dumps(ret), 500 282 | 283 | # Create! 284 | c = Channel( 285 | chanid=chanid, 286 | service=service) 287 | id = c.save() 288 | 289 | # Queue refresh 290 | ytdl.tasks.refresh_channel.delay(id=id) 291 | 292 | return json.dumps({"status": "ok", "id": id}) 293 | 294 | 295 | 296 | @app.before_request 297 | def before_request(): 298 | ytdl.models.database.init(ytdl.settings.DB_PATH) 299 | 300 | 301 | if __name__ == "__main__": 302 | app.debug=True 303 | app.run() 304 | -------------------------------------------------------------------------------- /ytdl/download_api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import traceback 3 | 4 | import redis 5 | import youtube_dl 6 | import ytdl.settings 7 | 8 | 9 | HOUR = 60*60 # 60 seconds in minute, 60 minutes in hour 10 | 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | 16 | class YDL(object): 17 | def __init__(self, id, url, outtmpl): 18 | self.id = id 19 | self.url = url 20 | self.r = redis.Redis(host=ytdl.settings.REDIS_HOST, port=ytdl.settings.REDIS_PORT) 21 | self.outtmpl = outtmpl 22 | 23 | def debug(self, msg): 24 | log.debug("YTDL DEBUG: %s" % msg) 25 | self._append_log("[debug] %s" % msg) 26 | 27 | def warning(self, msg): 28 | log.debug("YTDL WARNING: %s" % msg) 29 | self._append_log("[warning] %s" % msg) 30 | 31 | def error(self, msg): 32 | log.debug("YTDL ERROR: %s" % msg) 33 | self._append_log("[error] %s" % msg) 34 | 35 | def _set_progress(self, percent=None, status=None, msg=None): 36 | key = "dl:{id}:info".format(id=self.id) 37 | if status: 38 | self.r.hset(key, "status", status) 39 | self.r.expire(key, 1*HOUR) 40 | if percent: 41 | self.r.hset(key, "progress", percent) 42 | self.r.expire(key, 1*HOUR) 43 | if msg: 44 | self.r.hset(key, "message", msg) 45 | self.r.expire(key, 1*HOUR) 46 | 47 | def _append_log(self, line): 48 | key = "dl:{id}:log".format(id=self.id) 49 | self.r.rpush(key, line) 50 | self.r.ltrim(key, -100, -1) # Keep only last 100 lines 51 | self.r.expire(key, 1*HOUR) 52 | 53 | def progress_hook(self, d): 54 | def human(byt): 55 | return "%.02fMiB" % (byt/1024.0/1024.0) 56 | def human_seconds(sec): 57 | return "%02d:%02d" % (sec//60, sec%60) 58 | if d['status'] == 'downloading': 59 | downloaded = float(d.get("downloaded_bytes", 0)) 60 | total = float(d.get("total_bytes", 1)) 61 | percent = 100*(float(downloaded) / float(total)) 62 | msg = "%3.01f%% of %s at %.02fKiB/s ETA %s" % ( 63 | percent, 64 | human(total), 65 | (d.get('speed') or 0)/1024.0, 66 | human_seconds(d.get('eta') or 0)) 67 | 68 | self._set_progress(status=d['status'], percent=percent, msg=msg) 69 | 70 | elif d['status'] == 'finished': 71 | self._set_progress(status=d['status']) 72 | self._append_log("done") 73 | 74 | # Remove from active-download set 75 | self.r.srem("dl", self.id) 76 | 77 | def go(self): 78 | opts = {} 79 | opts['logger'] = self 80 | opts['progress_hooks'] = [self.progress_hook] 81 | 82 | opts['restrictfilenames'] = True 83 | opts['continuedl'] = True 84 | opts['outtmpl'] = self.outtmpl 85 | 86 | # Add to active-download set 87 | self.r.sadd("dl", self.id) 88 | 89 | # Clean up stale data 90 | self.r.delete("dl:{id}:info".format(id=self.id)) 91 | self.r.delete("dl:{id}:log".format(id=self.id)) 92 | 93 | self._set_progress(status="new") 94 | 95 | with youtube_dl.YoutubeDL(opts) as ydl: 96 | log.info("Beginning downloading %s" % (self.url)) 97 | try: 98 | ydl.download([self.url]) 99 | except youtube_dl.DownloadError as e: 100 | self._set_progress(status="error", msg='%s' % e) 101 | for line in traceback.format_exc(): 102 | self._append_log(line=line) 103 | raise 104 | 105 | 106 | if __name__ == '__main__': 107 | logging.basicConfig(level=logging.DEBUG) 108 | import random 109 | d = YDL(id=random.randint(1, 100), 110 | url='http://www.youtube.com/watch?v=BaW_jenozKc', 111 | outtmpl="/tmp/test_%(id)s.%(ext)s") 112 | d.go() 113 | -------------------------------------------------------------------------------- /ytdl/flask_rq.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | flask_rq 4 | ~~~~~~~~ 5 | 6 | RQ (Redis Queue) integration for Flask applications. 7 | 8 | :copyright: (c) 2012 by Matt Wright. 9 | :license: MIT, see LICENSE for more details. 10 | 11 | """ 12 | 13 | __version__ = '0.2' 14 | 15 | import redis 16 | 17 | from flask import current_app 18 | from redis._compat import urlparse 19 | from rq import Queue, Worker 20 | 21 | 22 | default_config = { 23 | 'RQ_DEFAULT_HOST': 'localhost', 24 | 'RQ_DEFAULT_PORT': 6379, 25 | 'RQ_DEFAULT_PASSWORD': None, 26 | 'RQ_DEFAULT_DB': 0 27 | } 28 | 29 | 30 | def config_value(name, key): 31 | name = name.upper() 32 | config_key = 'RQ_%s_%s' % (name, key) 33 | if not config_key in current_app.config \ 34 | and not 'RQ_%s_URL' % name in current_app.config: 35 | config_key = 'RQ_DEFAULT_%s' % key 36 | return current_app.config.get(config_key, None) 37 | 38 | 39 | def get_connection(queue='default'): 40 | url = config_value(queue, 'URL') 41 | if url: 42 | return redis.from_url(url, db=config_value(queue, 'DB')) 43 | return redis.Redis(host=config_value(queue, 'HOST'), 44 | port=config_value(queue, 'PORT'), 45 | password=config_value(queue, 'PASSWORD'), 46 | db=config_value(queue, 'DB')) 47 | 48 | 49 | def get_queue(name='default', **kwargs): 50 | kwargs['connection'] = get_connection(name) 51 | return Queue(name, **kwargs) 52 | 53 | 54 | def get_server_url(name): 55 | url = config_value(name, 'URL') 56 | if url: 57 | url_kwargs = urlparse(url) 58 | return '%s://%s' % (url_kwargs.scheme, url_kwargs.netloc) 59 | else: 60 | host = config_value(name, 'HOST') 61 | password = config_value(name, 'HOST') 62 | netloc = host if not password else ':%s@%s' % (password, host) 63 | return 'redis://%s' % netloc 64 | 65 | 66 | def get_worker(*queues): 67 | if len(queues) == 0: 68 | queues = ['default'] 69 | servers = [get_server_url(name) for name in queues] 70 | if not servers.count(servers[0]) == len(servers): 71 | raise Exception('A worker only accept one connection') 72 | return Worker([get_queue(name) for name in queues], 73 | connection=get_connection(queues[0])) 74 | 75 | 76 | def job(func_or_queue=None): 77 | if callable(func_or_queue): 78 | func = func_or_queue 79 | queue = 'default' 80 | else: 81 | func = None 82 | queue = func_or_queue 83 | 84 | def wrapper(fn): 85 | def delay(*args, **kwargs): 86 | q = get_queue(queue) 87 | return q.enqueue(fn, *args, **kwargs) 88 | 89 | fn.delay = delay 90 | return fn 91 | 92 | if func is not None: 93 | return wrapper(func) 94 | 95 | return wrapper 96 | 97 | 98 | class RQ(object): 99 | def __init__(self, app=None): 100 | if app is not None: 101 | self.init_app(app) 102 | 103 | def init_app(self, app): 104 | for key, value in default_config.items(): 105 | app.config.setdefault(key, value) 106 | -------------------------------------------------------------------------------- /ytdl/models.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import dateutil.tz 3 | from typing import Union 4 | 5 | import peewee as p 6 | 7 | from .youtube_api import YoutubeApi 8 | from .vimeo_api import VimeoApi 9 | from .settings import DB_PATH 10 | from .vimeo_api import VimeoApi 11 | from .youtube_api import YoutubeApi 12 | 13 | database = p.SqliteDatabase(DB_PATH) 14 | 15 | 16 | YOUTUBE = 'youtube' 17 | VIMEO = 'vimeo' 18 | 19 | ALL_SERVICES = [YOUTUBE, VIMEO] 20 | 21 | 22 | def getnow(): 23 | # type: () -> datetime.datetime 24 | return datetime.datetime.now(dateutil.tz.tzlocal()) 25 | 26 | 27 | class BaseModel(p.Model): 28 | class Meta: 29 | database = database 30 | 31 | 32 | class Channel(BaseModel): 33 | chanid = p.CharField(max_length=256) 34 | icon_url = p.CharField(max_length=1024, null=True) 35 | last_refresh = p.DateTimeField(null=True) 36 | last_update_content = p.DateTimeField(null=True) 37 | last_update_meta = p.DateTimeField(null=True) 38 | service = p.CharField(max_length=256, null=True) 39 | title = p.CharField(max_length=512, null=True) 40 | 41 | class Meta: 42 | table_name = 'ytdl_channel' 43 | 44 | def __unicode__(self): 45 | return "%s (%s on %s)" % (self.title, self.chanid, self.service) 46 | 47 | def get_api(self): 48 | # type: () -> Union[VimeoApi, YoutubeApi] 49 | if self.service == YOUTUBE: 50 | return YoutubeApi(str(self.chanid)) 51 | elif self.service == VIMEO: 52 | return VimeoApi(str(self.chanid)) 53 | else: 54 | raise ValueError("Unknown service %r" % self.service) 55 | 56 | def refresh_meta(self): 57 | # type: () -> None 58 | """Get stuff like the channel title 59 | """ 60 | api = self.get_api() 61 | now = getnow() 62 | 63 | new_title = api.title 64 | if self.title != new_title: 65 | self.title = self.get_api().title() 66 | self.last_update_meta = now 67 | self.save() 68 | 69 | new_icon = api.icon() 70 | if self.icon_url != new_icon: 71 | self.icon_url = new_icon 72 | self.last_update_meta = now 73 | self.save() 74 | 75 | def grab(chan, limit=1000, stop_on_existing=True): 76 | chan.last_refresh = getnow() 77 | chan.save() 78 | 79 | api = chan.get_api() 80 | 81 | with database.transaction(): 82 | for vid in api.videos_for_user(limit=limit): 83 | exists = Video.select().where(Video.videoid == vid['id']).count() > 0 84 | if exists: 85 | if stop_on_existing: 86 | print("%s exists, stopping" % (vid['id'])) 87 | return 88 | else: 89 | print("%s exists. Onwards!" % (vid['id'])) 90 | else: 91 | # Save it 92 | v = Video( 93 | title = vid['title'], 94 | url = vid['url'], 95 | videoid = vid['id'], 96 | description = vid['descr'], 97 | channel = chan, 98 | _thumbnails = " ".join(vid['thumbs']), 99 | publishdate = vid['published'], 100 | ) 101 | 102 | # TODO: Less queries? 103 | v.save() 104 | chan.last_update_content = datetime.datetime.now(dateutil.tz.tzlocal()) 105 | chan.save() 106 | 107 | 108 | class Video(BaseModel): 109 | STATE_NEW = 'NE' 110 | STATE_QUEUED = 'QU' 111 | STATE_DOWNLOADING = 'DL' 112 | STATE_GRABBED = 'GR' 113 | STATE_GRAB_ERROR = 'GE' 114 | STATE_IGNORE = 'IG' 115 | 116 | STATES = ( 117 | (STATE_NEW, 'New video'), 118 | (STATE_QUEUED, 'Queued for download, but not begun yet'), 119 | (STATE_DOWNLOADING, 'Downloading'), 120 | (STATE_GRABBED, 'Grabbing video now'), 121 | (STATE_GRAB_ERROR, 'Error while grabbing video'), 122 | (STATE_IGNORE, 'Ignored video'), 123 | ) 124 | 125 | 126 | _thumbnails = p.CharField(max_length=1024) 127 | channel = p.ForeignKeyField(Channel, related_name="videos") 128 | description = p.TextField(null=True) 129 | publishdate = p.DateTimeField() 130 | status = p.CharField(max_length=2, default=STATE_NEW) 131 | title = p.CharField(max_length=1024) 132 | url = p.CharField(max_length=1024) 133 | videoid = p.CharField(max_length=256) 134 | 135 | class Meta: 136 | table_name = 'ytdl_video' 137 | 138 | def __unicode__(self): 139 | return "%s (on %s) [%s]" % (self.title, self.channel.chanid, self.status) 140 | 141 | @property 142 | def img(self): 143 | return self._thumbnails.split(" ") 144 | -------------------------------------------------------------------------------- /ytdl/paginator.py: -------------------------------------------------------------------------------- 1 | # From django.core.paginator 2 | import collections 3 | from math import ceil 4 | 5 | 6 | class InvalidPage(Exception): 7 | pass 8 | 9 | 10 | class PageNotAnInteger(InvalidPage): 11 | pass 12 | 13 | 14 | class EmptyPage(InvalidPage): 15 | pass 16 | 17 | 18 | class Paginator(object): 19 | 20 | def __init__(self, object_list, per_page, orphans=0, 21 | allow_empty_first_page=True): 22 | self.object_list = object_list 23 | self.per_page = int(per_page) 24 | self.orphans = int(orphans) 25 | self.allow_empty_first_page = allow_empty_first_page 26 | self._num_pages = self._count = None 27 | 28 | def validate_number(self, number): 29 | """ 30 | Validates the given 1-based page number. 31 | """ 32 | try: 33 | number = int(number) 34 | except (TypeError, ValueError): 35 | raise PageNotAnInteger('That page number is not an integer') 36 | if number < 1: 37 | raise EmptyPage('That page number is less than 1') 38 | if number > self.num_pages: 39 | if number == 1 and self.allow_empty_first_page: 40 | pass 41 | else: 42 | raise EmptyPage('That page contains no results') 43 | return number 44 | 45 | def page(self, number): 46 | """ 47 | Returns a Page object for the given 1-based page number. 48 | """ 49 | number = self.validate_number(number) 50 | bottom = (number - 1) * self.per_page 51 | top = bottom + self.per_page 52 | if top + self.orphans >= self.count: 53 | top = self.count 54 | return self._get_page(self.object_list[bottom:top], number, self) 55 | 56 | def _get_page(self, *args, **kwargs): 57 | """ 58 | Returns an instance of a single page. 59 | 60 | This hook can be used by subclasses to use an alternative to the 61 | standard :cls:`Page` object. 62 | """ 63 | return Page(*args, **kwargs) 64 | 65 | def _get_count(self): 66 | """ 67 | Returns the total number of objects, across all pages. 68 | """ 69 | if self._count is None: 70 | try: 71 | self._count = self.object_list.count() 72 | except (AttributeError, TypeError): 73 | # AttributeError if object_list has no count() method. 74 | # TypeError if object_list.count() requires arguments 75 | # (i.e. is of type list). 76 | self._count = len(self.object_list) 77 | return self._count 78 | count = property(_get_count) 79 | 80 | def _get_num_pages(self): 81 | """ 82 | Returns the total number of pages. 83 | """ 84 | if self._num_pages is None: 85 | if self.count == 0 and not self.allow_empty_first_page: 86 | self._num_pages = 0 87 | else: 88 | hits = max(1, self.count - self.orphans) 89 | self._num_pages = int(ceil(hits / float(self.per_page))) 90 | return self._num_pages 91 | num_pages = property(_get_num_pages) 92 | 93 | def _get_page_range(self): 94 | """ 95 | Returns a 1-based range of pages for iterating through within 96 | a template for loop. 97 | """ 98 | return range(1, self.num_pages + 1) 99 | page_range = property(_get_page_range) 100 | 101 | 102 | QuerySetPaginator = Paginator # For backwards-compatibility. 103 | 104 | 105 | class Page(collections.Sequence): 106 | 107 | def __init__(self, object_list, number, paginator): 108 | self.object_list = object_list 109 | self.number = number 110 | self.paginator = paginator 111 | 112 | def __repr__(self): 113 | return '' % (self.number, self.paginator.num_pages) 114 | 115 | def __len__(self): 116 | return len(self.object_list) 117 | 118 | def __getitem__(self, index): 119 | try: 120 | long 121 | except NameError: 122 | long = int 123 | if not isinstance(index, (slice, int, long)): 124 | raise TypeError 125 | # The object_list is converted to a list so that if it was a QuerySet 126 | # it won't be a database hit per __getitem__. 127 | if not isinstance(self.object_list, list): 128 | self.object_list = list(self.object_list) 129 | return self.object_list[index] 130 | 131 | def has_next(self): 132 | return self.number < self.paginator.num_pages 133 | 134 | def has_previous(self): 135 | return self.number > 1 136 | 137 | def has_other_pages(self): 138 | return self.has_previous() or self.has_next() 139 | 140 | def next_page_number(self): 141 | return self.paginator.validate_number(self.number + 1) 142 | 143 | def previous_page_number(self): 144 | return self.paginator.validate_number(self.number - 1) 145 | 146 | def start_index(self): 147 | """ 148 | Returns the 1-based index of the first object on this page, 149 | relative to total objects in the paginator. 150 | """ 151 | # Special case, return zero if no items. 152 | if self.paginator.count == 0: 153 | return 0 154 | return (self.paginator.per_page * (self.number - 1)) + 1 155 | 156 | def end_index(self): 157 | """ 158 | Returns the 1-based index of the last object on this page, 159 | relative to total objects found (hits). 160 | """ 161 | # Special case for the last page because there can be orphans. 162 | if self.number == self.paginator.num_pages: 163 | return self.paginator.count 164 | return self.number * self.paginator.per_page 165 | -------------------------------------------------------------------------------- /ytdl/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | OUTPUT_DIR = os.getenv("YTDL_DOWNLOAD_PATH", 3 | os.path.expanduser("/data/ytdl_downloads")) 4 | OUTPUT_FORMAT = "%(uploader)s__%(upload_date)s_%(title)s__%(id)s.%(ext)s" 5 | YOUTUBE_DL_FLAGS = ['--max-quality=22',] 6 | 7 | DB_PATH = os.getenv("YTDL_DB_PATH", 8 | '/Users/dbr/code/webtools/dev.sqlite3') 9 | 10 | REDIS_HOST = os.getenv("REDIS_HOST", "localhost") 11 | REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) 12 | -------------------------------------------------------------------------------- /ytdl/static/css/font-awesome.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.0.3 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.0.3');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.0.3') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff?v=4.0.3') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.0.3') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.0.3#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.3333333333333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.2857142857142858em;text-align:center}.fa-ul{padding-left:0;margin-left:2.142857142857143em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.142857142857143em;width:2.142857142857143em;top:.14285714285714285em;text-align:center}.fa-li.fa-lg{left:-1.8571428571428572em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:spin 2s infinite linear;-moz-animation:spin 2s infinite linear;-o-animation:spin 2s infinite linear;animation:spin 2s infinite linear}@-moz-keyframes spin{0%{-moz-transform:rotate(0deg)}100%{-moz-transform:rotate(359deg)}}@-webkit-keyframes spin{0%{-webkit-transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg)}}@-o-keyframes spin{0%{-o-transform:rotate(0deg)}100%{-o-transform:rotate(359deg)}}@-ms-keyframes spin{0%{-ms-transform:rotate(0deg)}100%{-ms-transform:rotate(359deg)}}@keyframes spin{0%{transform:rotate(0deg)}100%{transform:rotate(359deg)}}.fa-rotate-90{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2);-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=3);-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=0,mirror=1);-webkit-transform:scale(-1,1);-moz-transform:scale(-1,1);-ms-transform:scale(-1,1);-o-transform:scale(-1,1);transform:scale(-1,1)}.fa-flip-vertical{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2,mirror=1);-webkit-transform:scale(1,-1);-moz-transform:scale(1,-1);-ms-transform:scale(1,-1);-o-transform:scale(1,-1);transform:scale(1,-1)}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-asc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-desc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-reply-all:before{content:"\f122"}.fa-mail-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"} -------------------------------------------------------------------------------- /ytdl/static/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /ytdl/static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /ytdl/static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /ytdl/static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /ytdl/static/img/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/img/blank.gif -------------------------------------------------------------------------------- /ytdl/static/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /ytdl/static/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /ytdl/static/img/spinner.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbr/webtools/87e16bc1cdca3b44a570d02dffc1c6189582e86a/ytdl/static/img/spinner.gif -------------------------------------------------------------------------------- /ytdl/static/js/director.min.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | // 4 | // Generated on Wed Nov 05 2014 22:50:43 GMT-0500 (EST) by Nodejitsu, Inc (Using Codesurgeon). 5 | // Version 1.2.4 6 | // 7 | (function(a){function c(){return b.hash===""||b.hash==="#"}function f(a,b){for(var c=0;ci.indexOf(d,e)||~i.indexOf(c,e)&&!~i.indexOf(d,e)||!~i.indexOf(c,e)&&~i.indexOf(d,e)){f=i.indexOf(c,e),g=i.indexOf(d,e);if(~f&&!~g||!~f&&~g){var j=a.slice(0,(h||1)+1).join(b);a=[j].concat(a.slice((h||1)+1))}e=(g>f?g:f)+1,h=0}else e=0}return a}var b=document.location,d={mode:"modern",hash:b.hash,history:!1,check:function(){var a=b.hash;a!=this.hash&&(this.hash=a,this.onHashChanged())},fire:function(){this.mode==="modern"?this.history===!0?window.onpopstate():window.onhashchange():this.onHashChanged()},init:function(a,b){function d(a){for(var b=0,c=e.listeners.length;b7))this.history===!0?setTimeout(function(){window.onpopstate=d},500):window.onhashchange=d,this.mode="modern";else{var f=document.createElement("iframe");f.id="state-frame",f.style.display="none",document.body.appendChild(f),this.writeFrame(""),"onpropertychange"in document&&"attachEvent"in document&&document.attachEvent("onpropertychange",function(){event.propertyName==="location"&&c.check()}),window.setInterval(function(){c.check()},50),this.onHashChanged=d,this.mode="legacy"}return e.listeners.push(a),this.mode},destroy:function(a){if(!e||!e.listeners)return;var b=e.listeners;for(var c=b.length-1;c>=0;c--)b[c]===a&&b.splice(c,1)},setHash:function(a){return this.mode==="legacy"&&this.writeFrame(a),this.history===!0?(window.history.pushState({},document.title,a),this.fire()):b.hash=a[0]==="/"?a:"/"+a,this},writeFrame:function(a){var b=document.getElementById("state-frame"),c=b.contentDocument||b.contentWindow.document;c.open(),c.write(" 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /ytdl/tasks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | from redis import Redis 5 | from rq import Queue 6 | 7 | import ytdl.settings 8 | import ytdl.download_api 9 | 10 | 11 | def get_queue(queue): 12 | redis_conn = Redis(host=ytdl.settings.REDIS_HOST, port=ytdl.settings.REDIS_PORT) 13 | q = Queue(queue, connection=redis_conn) 14 | return q 15 | 16 | 17 | def task(queue_name, *queue_args, **queue_kwargs): 18 | def decorator_creation(func): 19 | def delay(*func_args, **func_kwargs): 20 | q = get_queue(queue_name) 21 | t = q.enqueue_call(func, args=func_args, kwargs=func_kwargs, 22 | *queue_args, **queue_kwargs) 23 | return t 24 | func.delay = delay 25 | return func 26 | 27 | return decorator_creation 28 | 29 | 30 | import ytdl.settings 31 | from ytdl.models import Video, Channel 32 | 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | QUEUE_DEFAULT = "ytdl-default" 37 | QUEUE_DOWNLOAD = "ytdl-download" 38 | 39 | 40 | HOUR = 60*60 41 | @task(QUEUE_DOWNLOAD, timeout=2*HOUR) 42 | def grab_video(videoid, force=False): 43 | # Get video from DB 44 | video = Video.get(id=videoid) 45 | 46 | # Validation 47 | grabbable = video.status in [Video.STATE_NEW, Video.STATE_GRAB_ERROR, Video.STATE_QUEUED] 48 | 49 | if not grabbable and not force: 50 | emsg = "%s status not NEW or GRAB_ERROR, and force was %s" % ( 51 | video, force) 52 | raise ValueError(emsg) 53 | 54 | if force and video.status == Video.STATE_DOWNLOADING: 55 | raise ValueError("Already downloading") 56 | 57 | # Grab video 58 | log.info("Starting to grab %s" % video) 59 | video.status = Video.STATE_DOWNLOADING 60 | video.save() 61 | 62 | cwd = ytdl.settings.OUTPUT_DIR 63 | try: 64 | os.makedirs(cwd) 65 | except OSError as e: 66 | if e.errno == 17: 67 | # Ignore errno 17 (File exists) 68 | pass 69 | else: 70 | raise 71 | 72 | # Grab video 73 | outtmpl = os.path.join(ytdl.settings.OUTPUT_DIR, ytdl.settings.OUTPUT_FORMAT) 74 | x = ytdl.download_api.YDL(id=videoid, url=video.url, outtmpl=outtmpl) 75 | try: 76 | x.go() 77 | except Exception as e: # ? 78 | video.status = Video.STATE_GRAB_ERROR 79 | video.save() 80 | log.error("Error grabbing %s: %s" % (video, e), exc_info=True) 81 | return 82 | else: 83 | video.status = Video.STATE_GRABBED 84 | video.save() 85 | log.info("Grab complete %s" % video) 86 | 87 | 88 | @task(QUEUE_DEFAULT) 89 | def refresh_channel(id): 90 | # type: (str) -> None 91 | log.debug("Refreshing channel %s" % id) 92 | channel = Channel.get(id=id) 93 | log.debug("Refreshing channel metadata for %s" % (channel)) 94 | channel.refresh_meta() 95 | log.debug("Grabbing from channel %s" % (channel)) 96 | channel.grab() 97 | log.debug("Refresh complete for %s" % (channel)) 98 | 99 | 100 | def refresh_all_channels(asyncr=True): 101 | # type: (bool) -> None 102 | log.debug("Refreshing all channels") 103 | channels = Channel.select() 104 | for c in channels: 105 | if asyncr: 106 | refresh_channel.delay(id=c.id) 107 | else: 108 | refresh_channel(id=c.id) 109 | -------------------------------------------------------------------------------- /ytdl/test_service_api.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from unittest import TestCase 3 | from mypy_extensions import NoReturn 4 | 5 | 6 | IS_PY2 = sys.version_info[0] == 2 7 | 8 | import peewee 9 | 10 | 11 | class YoutubeTest(TestCase): 12 | def setUp(self): 13 | # type: () -> None 14 | super(YoutubeTest, self).setUp() 15 | from ytdl.youtube_api import YoutubeApi 16 | self.api = YoutubeApi("roosterteeth") 17 | 18 | def test_list_videos(self): 19 | # type: () -> None 20 | videos = list(self.api.videos_for_user(limit=1)) 21 | 22 | assert len(videos) > 0 23 | for v in videos: 24 | assert 'url' in v 25 | assert 'title' in v 26 | assert 'id' in v 27 | assert 'youtube.com' in v['url'] 28 | 29 | def test_icon(self): 30 | # type: () -> None 31 | url = self.api.icon() 32 | assert url.startswith("http://") or url.startswith("https://") 33 | 34 | def test_title(self): 35 | # type: () -> None 36 | title = self.api.title() 37 | assert title == "Rooster Teeth" 38 | 39 | 40 | class VimeoTest(TestCase): 41 | def setUp(self): 42 | # type: () -> None 43 | super(VimeoTest, self).setUp() 44 | from ytdl.vimeo_api import VimeoApi 45 | self.api = VimeoApi("dbr") 46 | 47 | def test_list_videos(self): 48 | # type: () -> None 49 | videos = list(self.api.videos_for_user(limit=1)) 50 | 51 | assert len(videos) > 0 52 | for v in videos: 53 | assert 'url' in v 54 | assert 'title' in v 55 | assert 'id' in v 56 | assert 'vimeo.com' in v['url'], v['url'] 57 | 58 | def test_icon(self): 59 | # type: () -> None 60 | url = self.api.icon() 61 | assert url.startswith("http://") or url.startswith("https://") 62 | # Vimeo icons don't have file extension 63 | 64 | def test_title(self): 65 | # type: () -> None 66 | title = self.api.title() 67 | assert title == "dbr", title 68 | 69 | 70 | class ChannelRefresh(TestCase): 71 | 72 | def test_youtube_refresh(self): 73 | # type: () -> None 74 | import ytdl.models 75 | 76 | chan = ytdl.models.Channel(chanid = 'roosterteeth', service=ytdl.models.YOUTUBE) 77 | chan.save() 78 | 79 | # Check title is updated 80 | chan.refresh_meta() 81 | assert chan.title == "Rooster Teeth" # TODO: Maybe assert != roosterteeth, since it could change? 82 | 83 | # Videos 84 | chan.grab(limit=1) 85 | videos = ytdl.models.Video.select().where(ytdl.models.Video.channel == chan) 86 | assert videos.count() > 0 87 | 88 | def test_vimeo_refresh(self): 89 | # type: () -> None 90 | import ytdl.models 91 | 92 | chan = ytdl.models.Channel(chanid = 'dbr', service=ytdl.models.VIMEO) 93 | chan.save() 94 | 95 | # Check title is updated 96 | chan.refresh_meta() 97 | assert chan.title == "dbr" 98 | 99 | # Videos 100 | chan.grab(limit=1) 101 | videos = ytdl.models.Video.select().where(ytdl.models.Video.channel == chan) 102 | assert videos.count() > 0 103 | -------------------------------------------------------------------------------- /ytdl/vimeo_api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from typing import (Any, Dict, Iterator) 3 | 4 | 5 | class VimeoApi(object): 6 | def __init__(self, chanid): 7 | # type: (str) -> None 8 | self.chanid = chanid 9 | 10 | def videos_for_user(self, limit=10): 11 | # type: (int) -> Iterator[Dict[str, Any]] 12 | # For some bizarre reason, the API returns timestamps in 13 | # Eastern Timezone. WTF? http://vimeo.com/forums/topic:45607 14 | 15 | import dateutil.tz 16 | import dateutil.parser 17 | tz = dateutil.tz.gettz("US/Eastern") 18 | 19 | for page in range(3): 20 | page = page + 1 21 | url = "http://vimeo.com/api/v2/{chanid}/videos.json?page={page}".format( 22 | chanid = self.chanid, 23 | page = page) 24 | data = requests.get(url).json() 25 | for cur in data: 26 | dt = dateutil.parser.parse(cur['upload_date']) 27 | dt = dt.replace(tzinfo=tz) 28 | info = { 29 | 'id': cur['id'], 30 | 'title': cur['title'] or "Untitled", 31 | 'url': cur['url'], 32 | 'thumbs': [cur['thumbnail_medium'], ], 33 | 'descr': cur['description'], 34 | 'published': dt, 35 | } 36 | yield info 37 | 38 | def icon(self): 39 | # type: () -> str 40 | data = requests.get("http://vimeo.com/api/v2/%s/info.json" % self.chanid).json() 41 | return data['portrait_small'] 42 | 43 | def title(self): 44 | # type: () -> str 45 | data = requests.get("http://vimeo.com/api/v2/%s/info.json" % self.chanid).json() 46 | return data['display_name'] 47 | 48 | 49 | if __name__ == '__main__': 50 | v = VimeoApi("cyclocosm") 51 | for video in v.videos_for_user(): 52 | print(video) 53 | print(v.icon()) 54 | -------------------------------------------------------------------------------- /ytdl/youtube_api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | from typing import Any, Dict, Iterator, Union, List, Optional, Tuple, Generator 4 | 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class YoutubeApi(object): 10 | API_KEY = "AIzaSyBBUxzImakMKKW3B6Qu47lR9xMpb6DNqQE" # ytdl public API browser key (for Youtube API v3) 11 | 12 | def __init__(self, chanid): 13 | # type: (str) -> None 14 | self.chanid = chanid 15 | 16 | def videos_for_user(self, limit=10): 17 | # type: (int) -> Generator[Dict[str, Any], None, None] 18 | url = "https://www.googleapis.com/youtube/v3/channels?key={apikey}&forUsername={chanid}&part=contentDetails".format( 19 | apikey = self.API_KEY, 20 | chanid = self.chanid) 21 | resp = requests.get(url) 22 | if len(resp.json()['items']) == 0: 23 | # If nothing found for username, try as a channel ID 24 | # FIXME: Store consistent data - either channel ID or username 25 | log.warning("No items found at %s - trying as ID" % url) 26 | 27 | url = "https://www.googleapis.com/youtube/v3/channels?key={apikey}&id={chanid}&part=contentDetails".format( 28 | apikey = self.API_KEY, 29 | chanid = self.chanid) 30 | resp = requests.get(url) 31 | if len(resp.json()['items']) == 0: 32 | log.warning("No items found at %s either" % url) 33 | return # Nothing found, give up 34 | 35 | upload_playlist = resp.json()['items'][0]['contentDetails']['relatedPlaylists']['uploads'] 36 | 37 | next_page = None 38 | for x in range(limit): 39 | cur, next_page = self._videos_for_playlist(playlist_id=upload_playlist, page_token=next_page) 40 | for c in cur: 41 | yield c 42 | if next_page is None: 43 | break # Signifies no more pages 44 | 45 | def _videos_for_playlist(self, playlist_id, page_token=None): 46 | # type: (str, Optional[Any]) -> Tuple[List[Dict[str, Any]], str] 47 | if page_token is None: 48 | pt = "" 49 | else: 50 | pt = "&pageToken={p}".format(p=page_token) 51 | 52 | url = "https://www.googleapis.com/youtube/v3/playlistItems?key={apikey}&part=snippet&maxResults={num}&playlistId={playlist}{page}".format( 53 | apikey = self.API_KEY, 54 | num=50, 55 | playlist = playlist_id, 56 | page=pt 57 | ) 58 | 59 | resp = requests.get(url) 60 | data = resp.json() 61 | 62 | ret = [] 63 | for v in data['items']: 64 | s = v['snippet'] 65 | 66 | import dateutil.parser 67 | dt = dateutil.parser.parse(s['publishedAt']) 68 | 69 | thumbs = [s['thumbnails']['default']['url'], ] 70 | 71 | info = { 72 | 'id': "http://gdata.youtube.com/feeds/api/videos/%s" % s['resourceId']['videoId'], # TODO: Migrate form silly gdata ID in database 73 | 'title': s['title'], 74 | 'url': 'http://youtube.com/watch?v={id}'.format(id=s['resourceId']['videoId']), 75 | 'thumbs': thumbs, 76 | 'descr': s['description'], 77 | 'published': dt, 78 | } 79 | 80 | ret.append(info) 81 | 82 | return ret, data.get('nextPageToken') 83 | 84 | def _chan_snippet(self): 85 | # type: () -> Optional[Dict[str, Any]] 86 | url = "https://www.googleapis.com/youtube/v3/channels?key={apikey}&forUsername={chanid}&part=snippet".format( 87 | apikey = self.API_KEY, 88 | chanid = self.chanid, 89 | ) 90 | 91 | items = requests.get(url).json()['items'] 92 | if len(items) > 0: 93 | return items[0]['snippet'] 94 | else: 95 | log.warning("No items found at %s - trying as ID" % (url)) 96 | url = "https://www.googleapis.com/youtube/v3/channels?key={apikey}&id={chanid}&part=snippet".format( 97 | apikey = self.API_KEY, 98 | chanid = self.chanid, 99 | ) 100 | items = requests.get(url).json()['items'] 101 | if len(items) > 0: 102 | return items[0]['snippet'] 103 | else: 104 | log.warning("No items found at %s either" % (url)) 105 | return None 106 | 107 | 108 | def icon(self): 109 | # type: () -> str 110 | snippet = self._chan_snippet() 111 | if snippet is None: 112 | return "" # FIXME? 113 | return snippet['thumbnails']['default']['url'] 114 | 115 | def title(self): 116 | # type: () -> str 117 | snippet = self._chan_snippet() 118 | if snippet is None: 119 | return "(no title)" 120 | return snippet['title'] 121 | 122 | 123 | if __name__ == '__main__': 124 | y = YoutubeApi("roosterteeth") 125 | print(y.icon()) 126 | print(y.title()) 127 | for v in y.videos_for_user(): 128 | print(v['title']) 129 | --------------------------------------------------------------------------------