\d+)', url)
45 | if not m:
46 | raise NotImplementedError('%s not supported' % url)
47 | radio_id = m.group('radio_id')
48 | audio_id = m.group('audio_id')
49 | # Look for the audio_id among the full list of episodes
50 | for aid, title, url in lizhi_extract_playlist_info(radio_id):
51 | if aid == audio_id:
52 | lizhi_download_audio(audio_id, title, url, output_dir=output_dir, info_only=info_only)
53 | break
54 | else:
55 | raise NotImplementedError('Audio #%s not found in playlist #%s' % (audio_id, radio_id))
56 |
57 | site_info = "lizhi.fm"
58 | download = lizhi_download
59 | download_playlist = lizhi_download_playlist
60 |
--------------------------------------------------------------------------------
/src/you_get/extractors/huaban.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import json
4 | import os
5 | import re
6 | import math
7 | import traceback
8 | import urllib.parse as urlparse
9 |
10 | from ..common import *
11 |
12 | __all__ = ['huaban_download']
13 |
14 | site_info = '花瓣 (Huaban)'
15 |
16 | LIMIT = 100
17 |
18 |
19 | class Board:
20 | def __init__(self, title, pins):
21 | self.title = title
22 | self.pins = pins
23 | self.pin_count = len(pins)
24 |
25 |
26 | class Pin:
27 | host = 'http://img.hb.aicdn.com/'
28 |
29 | def __init__(self, pin_json):
30 | img_file = pin_json['file']
31 | self.id = str(pin_json['pin_id'])
32 | self.url = urlparse.urljoin(self.host, img_file['key'])
33 | self.ext = img_file['type'].split('/')[-1]
34 |
35 |
36 | def construct_url(url, **params):
37 | param_str = urlparse.urlencode(params)
38 | return url + '?' + param_str
39 |
40 |
41 | def extract_json_data(url, **params):
42 | url = construct_url(url, **params)
43 | html = get_content(url, headers=fake_headers)
44 | json_string = match1(html, r'app.page\["board"\] = (.*?});')
45 | json_data = json.loads(json_string)
46 | return json_data
47 |
48 |
49 | def extract_board_data(url):
50 | json_data = extract_json_data(url, limit=LIMIT)
51 | pin_list = json_data['pins']
52 | title = json_data['title']
53 | pin_count = json_data['pin_count']
54 | pin_count -= len(pin_list)
55 |
56 | while pin_count > 0:
57 | json_data = extract_json_data(url, max=pin_list[-1]['pin_id'],
58 | limit=LIMIT)
59 | pins = json_data['pins']
60 | pin_list += pins
61 | pin_count -= len(pins)
62 |
63 | return Board(title, list(map(Pin, pin_list)))
64 |
65 |
66 | def huaban_download_board(url, output_dir, **kwargs):
67 | kwargs['merge'] = False
68 | board = extract_board_data(url)
69 | output_dir = os.path.join(output_dir, board.title)
70 | print_info(site_info, board.title, 'jpg', float('Inf'))
71 | for pin in board.pins:
72 | download_urls([pin.url], pin.id, pin.ext, float('Inf'),
73 | output_dir=output_dir, faker=True, **kwargs)
74 |
75 |
76 | def huaban_download(url, output_dir='.', **kwargs):
77 | if re.match(r'http://huaban\.com/boards/\d+/', url):
78 | huaban_download_board(url, output_dir, **kwargs)
79 | else:
80 | print('Only board (画板) pages are supported currently')
81 | print('ex: http://huaban.com/boards/12345678/')
82 |
83 |
84 | download = huaban_download
85 | download_playlist = playlist_not_supported("huaban")
86 |
--------------------------------------------------------------------------------
/src/you_get/extractors/toutiao.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import base64
3 |
4 | import binascii
5 |
6 | from ..common import *
7 | import random
8 | from json import loads
9 |
10 | __all__ = ['toutiao_download', ]
11 |
12 |
13 | def sign_video_url(vid):
14 | # some code from http://codecloud.net/110854.html
15 | r = str(random.random())[2:]
16 |
17 | def right_shift(val, n):
18 | return val >> n if val >= 0 else (val + 0x100000000) >> n
19 |
20 | url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % vid
21 | n = url.replace("http://i.snssdk.com", "")+ '?r=' + r
22 | c = binascii.crc32(n.encode("ascii"))
23 | s = right_shift(c, 0)
24 | return url + '?r=%s&s=%s' % (r, s)
25 |
26 |
27 | class ToutiaoVideoInfo(object):
28 |
29 | def __init__(self):
30 | self.bitrate = None
31 | self.definition = None
32 | self.size = None
33 | self.height = None
34 | self.width = None
35 | self.type = None
36 | self.url = None
37 |
38 | def __str__(self):
39 | return json.dumps(self.__dict__)
40 |
41 |
42 | def get_file_by_vid(video_id):
43 | vRet = []
44 | url = sign_video_url(video_id)
45 | ret = get_content(url)
46 | ret = loads(ret)
47 | vlist = ret.get('data').get('video_list')
48 | if len(vlist) > 0:
49 | vInfo = vlist.get(sorted(vlist.keys(), reverse=True)[0])
50 | vUrl = vInfo.get('main_url')
51 | vUrl = base64.decodestring(vUrl.encode('ascii')).decode('ascii')
52 | videoInfo = ToutiaoVideoInfo()
53 | videoInfo.bitrate = vInfo.get('bitrate')
54 | videoInfo.definition = vInfo.get('definition')
55 | videoInfo.size = vInfo.get('size')
56 | videoInfo.height = vInfo.get('vheight')
57 | videoInfo.width = vInfo.get('vwidth')
58 | videoInfo.type = vInfo.get('vtype')
59 | videoInfo.url = vUrl
60 | vRet.append(videoInfo)
61 | return vRet
62 |
63 |
64 | def toutiao_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
65 | html = get_html(url, faker=True)
66 | video_id = match1(html, r"videoid\s*:\s*'([^']+)',\n")
67 | title = match1(html, r"title: '([^']+)'.replace")
68 | video_file_list = get_file_by_vid(video_id) # 调api获取视频源文件
69 | type, ext, size = url_info(video_file_list[0].url, faker=True)
70 | print_info(site_info=site_info, title=title, type=type, size=size)
71 | if not info_only:
72 | download_urls([video_file_list[0].url], title, ext, size, output_dir, merge=merge, faker=True)
73 |
74 |
75 | site_info = "Toutiao.com"
76 | download = toutiao_download
77 | download_playlist = playlist_not_supported("toutiao")
78 |
--------------------------------------------------------------------------------
/src/you_get/extractors/fc2video.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | __all__ = ['fc2video_download']
4 |
5 | from ..common import *
6 | from hashlib import md5
7 | from urllib.parse import urlparse
8 | import re
9 |
10 | #----------------------------------------------------------------------
11 | def makeMimi(upid):
12 | """From http://cdn37.atwikiimg.com/sitescript/pub/dksitescript/FC2.site.js
13 | Also com.hps.util.fc2.FC2EncrptUtil.makeMimiLocal
14 | L110"""
15 | strSeed = "gGddgPfeaf_gzyr"
16 | prehash = upid + "_" + strSeed
17 | return md5(prehash.encode('utf-8')).hexdigest()
18 |
19 | #----------------------------------------------------------------------
20 | def fc2video_download_by_upid(upid, output_dir = '.', merge = True, info_only = False, **kwargs):
21 | """"""
22 | fake_headers = {
23 | 'DNT': '1',
24 | 'Accept-Encoding': 'gzip, deflate, sdch',
25 | 'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
26 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.58 Safari/537.36',
27 | 'Accept': '*/*',
28 | 'X-Requested-With': 'ShockwaveFlash/19.0.0.245',
29 | 'Connection': 'keep-alive',
30 | }
31 | api_base = 'http://video.fc2.com/ginfo.php?upid={upid}&mimi={mimi}'.format(upid = upid, mimi = makeMimi(upid))
32 | html = get_content(api_base, headers=fake_headers)
33 |
34 | video_url = match1(html, r'filepath=(.+)&sec')
35 | video_url = video_url.replace('&mid', '?mid')
36 |
37 | title = match1(html, r'&title=([^&]+)')
38 |
39 | type, ext, size = url_info(video_url, headers=fake_headers)
40 |
41 | print_info(site_info, title, type, size)
42 | if not info_only:
43 | download_urls([video_url], title, ext, size, output_dir, merge=merge, headers = fake_headers)
44 |
45 | #----------------------------------------------------------------------
46 | def fc2video_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
47 | """wrapper"""
48 | #'http://video.fc2.com/en/content/20151021bTVKnbEw'
49 | #'http://xiaojiadianvideo.asia/content/20151021bTVKnbEw'
50 | #'http://video.fc2.com/ja/content/20151021bTVKnbEw'
51 | #'http://video.fc2.com/tw/content/20151021bTVKnbEw'
52 | hostname = urlparse(url).hostname
53 | if not ('fc2.com' in hostname or 'xiaojiadianvideo.asia' in hostname):
54 | return False
55 | upid = match1(url, r'.+/content/(\w+)')
56 |
57 | fc2video_download_by_upid(upid, output_dir, merge, info_only)
58 |
59 | site_info = "FC2Video"
60 | download = fc2video_download
61 | download_playlist = playlist_not_supported('fc2video')
62 |
--------------------------------------------------------------------------------
/src/you_get/extractors/vk.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | __all__ = ['vk_download']
4 |
5 | from ..common import *
6 |
7 |
8 | def get_video_info(url):
9 | video_page = get_content(url)
10 | title = r1(r'(.[^>]+?)]+?)"', video_page)
12 |
13 | for quality in ['.1080.', '.720.', '.480.', '.360.', '.240.']:
14 | for source in sources:
15 | if source.find(quality) != -1:
16 | url = source
17 | break
18 | assert url
19 | type, ext, size = url_info(url)
20 | print_info(site_info, title, type, size)
21 |
22 | return url, title, ext, size
23 |
24 |
25 | def get_video_from_user_videolist(url):
26 | ep = 'https://vk.com/al_video.php'
27 | to_post = dict(act='show', al=1, module='direct', video=re.search(r'video(\d+_\d+)', url).group(1))
28 | page = post_content(ep, post_data=to_post)
29 | video_pt = r'(.+?)
', page).group(1)
32 | mime, ext, size = url_info(url)
33 | print_info(site_info, title, mime, size)
34 |
35 | return url, title, ext, size
36 |
37 |
38 | def get_image_info(url):
39 | image_page = get_content(url)
40 | # used for title - vk page owner
41 | page_of = re.findall(r'Sender:(.[^>]+?)(.[^>]+?)Download full size', image_page)
47 | type, ext, size = url_info(image_link)
48 | print_info(site_info, title, type, size)
49 |
50 | return image_link, title, ext, size
51 |
52 |
53 | def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
54 | link = None
55 | if re.match(r'(.+)z\=video(.+)', url):
56 | link, title, ext, size = get_video_info(url)
57 | elif re.match(r'(.+)vk\.com\/photo(.+)', url):
58 | link, title, ext, size = get_image_info(url)
59 | elif re.search(r'vk\.com\/video\d+_\d+', url):
60 | link, title, ext, size = get_video_from_user_videolist(url)
61 | else:
62 | raise NotImplementedError('Nothing to download here')
63 |
64 | if not info_only and link is not None:
65 | download_urls([link], title, ext, size, output_dir, merge=merge)
66 |
67 |
68 | site_info = "VK.com"
69 | download = vk_download
70 | download_playlist = playlist_not_supported('vk')
71 |
--------------------------------------------------------------------------------
/src/you_get/extractors/bigthink.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | from ..common import *
4 | from ..extractor import VideoExtractor
5 |
6 | import json
7 |
8 | class Bigthink(VideoExtractor):
9 | name = "Bigthink"
10 |
11 | stream_types = [ #this is just a sample. Will make it in prepare()
12 | # {'id': '1080'},
13 | # {'id': '720'},
14 | # {'id': '360'},
15 | # {'id': '288'},
16 | # {'id': '190'},
17 | # {'id': '180'},
18 |
19 | ]
20 |
21 | @staticmethod
22 | def get_streams_by_id(account_number, video_id):
23 | """
24 | int, int->list
25 |
26 | Get the height of the videos.
27 |
28 | Since brightcove is using 3 kinds of links: rtmp, http and https,
29 | we will be using the HTTPS one to make it secure.
30 |
31 | If somehow akamaihd.net is blocked by the Great Fucking Wall,
32 | change the "startswith https" to http.
33 | """
34 | endpoint = 'https://edge.api.brightcove.com/playback/v1/accounts/{account_number}/videos/{video_id}'.format(account_number = account_number, video_id = video_id)
35 | fake_header_id = fake_headers
36 | #is this somehow related to the time? Magic....
37 | fake_header_id['Accept'] ='application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzprPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34weknpfG-sippyQ'
38 |
39 | html = get_content(endpoint, headers= fake_header_id)
40 | html_json = json.loads(html)
41 |
42 | link_list = []
43 |
44 | for i in html_json['sources']:
45 | if 'src' in i: #to avoid KeyError
46 | if i['src'].startswith('https'):
47 | link_list.append((str(i['height']), i['src']))
48 |
49 | return link_list
50 |
51 | def prepare(self, **kwargs):
52 |
53 | html = get_content(self.url)
54 |
55 | self.title = match1(html, r'type=tudou&vid=199687639