├── tests
    ├── __init__.py
    ├── runtests.py
    ├── test_util.py
    ├── test_common.py
    └── util.py
├── lulu
    ├── util
    │   ├── __init__.py
    │   ├── parser.py
    │   ├── strings.py
    │   ├── term.py
    │   ├── git.py
    │   ├── fs.py
    │   └── log.py
    ├── cli_wrapper
    │   ├── __init__.py
    │   ├── player
    │   │   ├── wmp.py
    │   │   ├── mplayer.py
    │   │   ├── dragonplayer.py
    │   │   ├── gnome_mplayer.py
    │   │   ├── vlc.py
    │   │   └── __init__.py
    │   └── transcoder
    │   │   ├── ffmpeg.py
    │   │   ├── libav.py
    │   │   ├── __init__.py
    │   │   └── mencoder.py
    ├── version.py
    ├── processor
    │   ├── __init__.py
    │   ├── join_ts.py
    │   └── rtmpdump.py
    ├── __init__.py
    ├── extractors
    │   ├── khan.py
    │   ├── joy.py
    │   ├── dilidili.py
    │   ├── tudou.py
    │   ├── cbs.py
    │   ├── alive.py
    │   ├── archive.py
    │   ├── ku6.py
    │   ├── giphy.py
    │   ├── freesound.py
    │   ├── magisto.py
    │   ├── ehow.py
    │   ├── huomaotv.py
    │   ├── quanmin.py
    │   ├── douyin.py
    │   ├── bandcamp.py
    │   ├── iqilu.py
    │   ├── metacafe.py
    │   ├── ted.py
    │   ├── bcy.py
    │   ├── heavymusic.py
    │   ├── pixivision.py
    │   ├── kuaishou.py
    │   ├── musicplayon.py
    │   ├── facebook.py
    │   ├── icourses.py
    │   ├── theplatform.py
    │   ├── iwara.py
    │   ├── naver.py
    │   ├── veoh.py
    │   ├── vidto.py
    │   ├── kuwo.py
    │   ├── vine.py
    │   ├── dailymotion.py
    │   ├── w56.py
    │   ├── douban.py
    │   ├── suntv.py
    │   ├── baomihua.py
    │   ├── yizhibo.py
    │   ├── qq_egame.py
    │   ├── qingting.py
    │   ├── soundcloud.py
    │   ├── panda.py
    │   ├── mtv81.py
    │   ├── videomega.py
    │   ├── kugou.py
    │   ├── ifeng.py
    │   ├── fantasy.py
    │   ├── vk.py
    │   ├── nicovideo.py
    │   ├── nanagogo.py
    │   ├── pinterest.py
    │   ├── infoq.py
    │   ├── __init__.py
    │   ├── yinyuetai.py
    │   ├── pixnet.py
    │   ├── yixia.py
    │   ├── fc2video.py
    │   ├── huaban.py
    │   ├── lizhi.py
    │   ├── toutiao.py
    │   ├── showroom.py
    │   ├── instagram.py
    │   ├── zhanqi.py
    │   ├── bigthink.py
    │   ├── tucao.py
    │   ├── weibo.py
    │   ├── qie_video.py
    │   ├── cntv.py
    │   ├── douyutv.py
    │   ├── sohu.py
    │   ├── imgur.py
    │   ├── ckplayer.py
    │   ├── ixigua.py
    │   ├── longzhu.py
    │   ├── bokecc.py
    │   ├── qie.py
    │   ├── wanmen.py
    │   ├── twitter.py
    │   ├── ximalaya.py
    │   ├── coub.py
    │   └── google.py
    ├── json_output.py
    └── __main__.py
├── MANIFEST.in
├── lulu.plugin.zsh
├── Pipfile
├── codecov.yml
├── bin
    └── lulu
├── fabfile.py
├── .travis.yml
├── appveyor.yml
├── .coveragerc
├── contrib
    └── completion
    │   ├── lulu-completion.bash
    │   ├── lulu.fish
    │   └── _lulu
├── LICENSE
├── README.rst
├── .gitignore
└── setup.py


/tests/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/util/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/wmp.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/mplayer.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/transcoder/ffmpeg.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/transcoder/libav.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include README.rst LICENSE
2 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/dragonplayer.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/gnome_mplayer.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/transcoder/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/transcoder/mencoder.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/vlc.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | 


--------------------------------------------------------------------------------
/lulu/cli_wrapper/player/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | 
3 | from .mplayer import *
4 | 


--------------------------------------------------------------------------------
/lulu/version.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | 
3 | __version__ = '0.5.3'
4 | script_name = 'lulu'
5 | 


--------------------------------------------------------------------------------
/lulu.plugin.zsh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env zsh
2 | alias lulu="noglob python3 $(dirname $0)/bin/lulu"
3 | alias lulu-vlc="noglob python3 $(dirname $0)/bin/lulu --player vlc"
4 | 


--------------------------------------------------------------------------------
/lulu/processor/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | 
3 | from .join_flv import concat_flv
4 | from .join_mp4 import concat_mp4
5 | from .ffmpeg import *
6 | from .rtmpdump import *
7 | 


--------------------------------------------------------------------------------
/lulu/util/parser.py:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | 
3 | from bs4 import BeautifulSoup
4 | 
5 | 
6 | def get_parser(page):
7 |     parser = BeautifulSoup(page, 'html.parser')
8 |     return parser
9 | 


--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
 1 | [[source]]
 2 | 
 3 | url = "https://pypi.python.org/simple"
 4 | verify_ssl = true
 5 | 
 6 | 
 7 | [dev-packages]
 8 | 
 9 | twine = "*"
10 | ipdb = "*"
11 | 
12 | 
13 | [packages]
14 | 
15 | coverage = "==4.4.2"
16 | cryptography = "==2.1.4"
17 | beautifulsoup4 = "==4.6.0"
18 | requests = "==2.18.4"
19 | 


--------------------------------------------------------------------------------
/lulu/util/strings.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | from html import unescape
 4 | 
 5 | from lulu.util.fs import legitimize
 6 | 
 7 | 
 8 | def get_filename(htmlstring):
 9 |     return legitimize(unescape(htmlstring))
10 | 
11 | 
12 | def parameterize(string):
13 |     return "'{}'".format(string.replace("'", r"'\''"))
14 | 


--------------------------------------------------------------------------------
/tests/runtests.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # coding=utf-8
 3 | 
 4 | import unittest
 5 | 
 6 | 
 7 | TEST_MODULES = [
 8 |     'tests.test_util',
 9 |     'tests.test_common',
10 |     'tests.test_extractors',
11 | ]
12 | 
13 | 
14 | def all():
15 |     return unittest.defaultTestLoader.loadTestsFromNames(TEST_MODULES)
16 | 
17 | 
18 | if __name__ == '__main__':
19 |     unittest.main(defaultTest='all')
20 | 


--------------------------------------------------------------------------------
/lulu/util/term.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | 
 4 | def get_terminal_size():
 5 |     """Get (width, height) of the current terminal."""
 6 |     try:
 7 |         # fcntl module only available on Unix
 8 |         import fcntl
 9 |         import termios
10 |         import struct
11 |         return struct.unpack('hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234'))
12 |     except Exception:
13 |         return (40, 80)
14 | 


--------------------------------------------------------------------------------
/lulu/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # This file is Python 2 compliant.
 3 | 
 4 | import sys
 5 | 
 6 | if sys.version_info[0] == 3:
 7 |     #from .extractor import Extractor, VideoExtractor
 8 |     #from .util import log
 9 | 
10 |     from .__main__ import *
11 | 
12 |     #from .common import *
13 |     #from .version import *
14 |     #from .cli_wrapper import *
15 |     #from .extractor import *
16 | else:
17 |     # Don't import anything.
18 |     pass
19 | 


--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
 1 | codecov:
 2 |   token: c430fe31-42ab-4fa4-81c3-f4109f1912a1
 3 | 
 4 | coverage:
 5 |   precision: 2
 6 |   round: down
 7 |   range: "70...100"
 8 | 
 9 |   status:
10 |     project:
11 |       default: on
12 |     patch:
13 |       default: on
14 |     changes:
15 |       default: off
16 | 
17 | comment:
18 |   layout: "header, reach, diff, flags, files, footer"
19 |   behavior: default
20 |   require_changes: no
21 |   require_base: no
22 |   require_head: yes
23 | 


--------------------------------------------------------------------------------
/bin/lulu:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | import os
 4 | import sys
 5 | 
 6 | _srcdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 7 | _filepath = os.path.dirname(sys.argv[0])
 8 | sys.path.insert(1, os.path.join(_filepath, _srcdir))
 9 | 
10 | 
11 | if sys.version_info[0] == 3:
12 |     import lulu
13 |     if __name__ == '__main__':
14 |         lulu.main()
15 | else:  # Python 2
16 |     from lulu.util import log
17 |     log.e('[fatal] Python 3 is required!')
18 | 


--------------------------------------------------------------------------------
/lulu/util/git.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import os
 4 | 
 5 | 
 6 | def get_head(repo_path):
 7 |     """Get (branch, commit) from HEAD of a git repo."""
 8 |     try:
 9 |         ref = open(
10 |             os.path.join(repo_path, '.git', 'HEAD'), 'r'
11 |         ).read().strip()[5:].split('/')
12 |         branch = ref[-1]
13 |         commit = open(
14 |             os.path.join(repo_path, '.git', *ref), 'r'
15 |         ).read().strip()[:7]
16 |         return branch, commit
17 |     except Exception:
18 |         return None
19 | 


--------------------------------------------------------------------------------
/fabfile.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | from fabric.api import (
 4 |     local,
 5 | )
 6 | 
 7 | 
 8 | def test(proxy=False):
 9 |     cmd = 'PYTHONPATH=./ {} coverage run tests/runtests.py'.format(
10 |         'proxychains4 -q' if proxy else ''
11 |     )
12 |     local(cmd)
13 | 
14 | 
15 | def test_download(func):
16 |     '''
17 |     fab test_download:acfun
18 |     '''
19 |     cmd = (
20 |         'PYTHONPATH=./ python tests/test_extractors.py '
21 |         'TestExtractors.test_{}'.format(func)
22 |     )
23 |     local(cmd)
24 | 
25 | 
26 | def upload():
27 |     local(
28 |         'python3 setup.py upload'
29 |     )
30 | 


--------------------------------------------------------------------------------
/tests/test_util.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import unittest
 4 | 
 5 | from lulu.util import fs
 6 | from lulu.util import parser
 7 | 
 8 | 
 9 | class TestUtil(unittest.TestCase):
10 |     def test_legitimize(self):
11 |         self.assertEqual(fs.legitimize('1*2', os='Linux'), '1*2')
12 |         self.assertEqual(fs.legitimize('1*2', os='Darwin'), '1*2')
13 |         self.assertEqual(fs.legitimize('1*2', os='Windows'), '1-2')
14 | 
15 |     def test_parser(self):
16 |         p = parser.get_parser('<h1> hello</h1>')
17 |         self.assertEqual(p.h1.string.strip(), 'hello')
18 | 
19 | 
20 | if __name__ == '__main__':
21 |     unittest.main()
22 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | dist: trusty
 2 | 
 3 | matrix:
 4 |   fast_finish: true
 5 | 
 6 | sudo: false
 7 | 
 8 | language: python
 9 | 
10 | python:
11 |   - 3.5
12 |   - 3.6
13 | 
14 | cache:
15 |   pip: true
16 | 
17 | # command to install dependencies
18 | install:
19 |   - pip install codecov
20 |   - pip install pipenv
21 | 
22 | before_script:
23 |   - export CI=True
24 |   - export TRAVIS=True
25 |   - export PYTHONPATH=$PYTHONPATH:$(pwd)
26 |   - pipenv install
27 | 
28 | # command to run tests
29 | script: coverage run tests/runtests.py
30 | 
31 | after_success:
32 |   - codecov
33 | 
34 | branches:
35 |   only:
36 |     - master
37 | 
38 | notifications:
39 |   email: false
40 | 


--------------------------------------------------------------------------------
/tests/test_common.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import unittest
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 | )
 8 | 
 9 | 
10 | class TestCommon(unittest.TestCase):
11 | 
12 |     def test_match1(self):
13 |         self.assertEqual(
14 |             match1('http://youtu.be/1234567890A', r'youtu.be/([^/]+)'),
15 |             '1234567890A'
16 |         )
17 |         self.assertEqual(
18 |             match1(
19 |                 'http://youtu.be/1234567890A',
20 |                 r'youtu.be/([^/]+)', r'youtu.(\w+)'
21 |             ),
22 |             ['1234567890A', 'be']
23 |         )
24 | 
25 | 
26 | if __name__ == '__main__':
27 |     unittest.main()
28 | 


--------------------------------------------------------------------------------
/lulu/extractors/khan.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | 
 4 | from lulu.common import (
 5 |     get_content,
 6 |     playlist_not_supported,
 7 | )
 8 | from lulu.util.parser import get_parser
 9 | from lulu.extractors.youtube import YouTube
10 | 
11 | 
12 | __all__ = ['khan_download']
13 | site_info = 'Khan Academy khanacademy.org'
14 | 
15 | 
16 | def khan_download(url, **kwargs):
17 |     html = get_content(url)
18 |     parser = get_parser(html)
19 |     youtube_url = parser.find('meta', property='og:video')['content']
20 |     YouTube().download_by_url(youtube_url, **kwargs)
21 | 
22 | 
23 | download = khan_download
24 | download_playlist = playlist_not_supported(site_info)
25 | 


--------------------------------------------------------------------------------
/appveyor.yml:
--------------------------------------------------------------------------------
 1 | environment:
 2 |   matrix:
 3 |     - PYTHON: C:\Python35
 4 |     - PYTHON: C:\Python36
 5 | 
 6 | build: off
 7 | 
 8 | init:
 9 |   - SET CI=True
10 |   - SET APPVEYOR=True
11 |   - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
12 |   - SET PYTHONIOENCODING=utf-8
13 |   - SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%
14 | 
15 | install:
16 |   - pip install -U pip
17 |   - pip install pipenv
18 |   - pipenv install
19 | 
20 | test_script:
21 |   - pipenv run coverage run tests/runtests.py
22 | 
23 | branches:
24 |   only:
25 |     - master
26 | 
27 | notifications:
28 |   - provider: Email
29 |     on_build_success: false
30 |     on_build_failure: false
31 |     on_build_status_changed: false
32 | 


--------------------------------------------------------------------------------
/.coveragerc:
--------------------------------------------------------------------------------
 1 | # .coveragerc to control coverage.py
 2 | [run]
 3 | branch = True
 4 | source =
 5 |     lulu
 6 |     tests
 7 | 
 8 | [report]
 9 | # Regexes for lines to exclude from consideration
10 | exclude_lines =
11 |     # Have to re-enable the standard pragma
12 |     pragma: no cover
13 | 
14 |     # Don't complain about missing debug-only code:
15 |     def __repr__
16 |     if self\.debug
17 | 
18 |     # Don't complain if tests don't hit defensive assertion code:
19 |     raise AssertionError
20 |     raise NotImplementedError
21 | 
22 |     # Don't complain if non-runnable code isn't run:
23 |     if 0:
24 |     if __name__ == .__main__.:
25 | 
26 | ignore_errors = True
27 | 
28 | [html]
29 | directory = coverage_html_report
30 | 


--------------------------------------------------------------------------------
/lulu/extractors/joy.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     url_info,
 5 |     print_info,
 6 |     get_content,
 7 |     download_urls,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.util.parser import get_parser
11 | 
12 | 
13 | __all__ = ['joy_download']
14 | site_info = '激动网 joy.cn'
15 | 
16 | 
17 | def joy_download(url, info_only=False, **kwargs):
18 |     page = get_content(url)
19 |     parser = get_parser(page)
20 |     url = parser.source['src']
21 |     title = parser.h1.text.strip()
22 |     _, ext, size = url_info(url)
23 |     print_info(site_info, title, ext, size)
24 |     if not info_only:
25 |         download_urls([url], title, ext, size, **kwargs)
26 | 
27 | 
28 | download = joy_download
29 | download_playlist = playlist_not_supported(site_info)
30 | 


--------------------------------------------------------------------------------
/lulu/extractors/dilidili.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     match1,
 5 |     get_content,
 6 |     any_download,
 7 |     playlist_not_supported,
 8 | )
 9 | 
10 | 
11 | __all__ = ['dilidili_download']
12 | site_info = 'dilidili.com'
13 | 
14 | 
15 | def dilidili_download(url, **kwargs):
16 |     html = get_content(url)
17 |     # player loaded via internal iframe
18 |     # http://www.maoyun.tv/mdparse/index.php?id=http://v.youku.com/v_show/id_XMTYxNzk0NjUzMg==.html  # noqa
19 |     iframe_url = match1(html, r'<iframe src=\"(.+?)\"')
20 |     url = match1(
21 |         iframe_url, r'https?://www.maoyun.tv/mdparse/index.php\?id=(.+)'
22 |     )
23 |     any_download(url, **kwargs)
24 | 
25 | 
26 | download = dilidili_download
27 | download_playlist = playlist_not_supported(site_info)
28 | 


--------------------------------------------------------------------------------
/lulu/extractors/tudou.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     get_content,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.extractors.youku import youku_download_by_vid
11 | 
12 | 
13 | __all__ = ['tudou_download']
14 | site_info = '土豆 tudou.com'
15 | 
16 | 
17 | def tudou_download(url, **kwargs):
18 |     if 'video.tudou.com' in url:
19 |         vid = match1(url, r'.*?video.tudou.com/v/([\w=]+)')
20 |     else:
21 |         page = get_content(url)
22 |         video_info = json.loads(
23 |             match1(page, r'window.__INITIAL_STATE__=\s*(.+?);</script>')
24 |         )
25 |         vid = video_info['videoDesc']['detail']['videoid']
26 |     youku_download_by_vid(vid, **kwargs)
27 | 
28 | 
29 | download = tudou_download
30 | download_playlist = playlist_not_supported(site_info)
31 | 


--------------------------------------------------------------------------------
/lulu/extractors/cbs.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     get_content,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.extractors.theplatform import theplatform_download_by_pid
11 | 
12 | 
13 | __all__ = ['cbs_download']
14 | site_info = 'CBS cbs.com'
15 | 
16 | 
17 | def cbs_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
18 |     """Downloads CBS videos by URL.
19 |     """
20 |     html = get_content(url)
21 |     data = json.loads(match1(html, r'var \$module = (.+);'))
22 |     pid = data['video']['pid']
23 |     title = data['video']['title']
24 |     theplatform_download_by_pid(
25 |         pid, title, output_dir=output_dir, merge=merge, info_only=info_only
26 |     )
27 | 
28 | 
29 | download = cbs_download
30 | download_playlist = playlist_not_supported(site_info)
31 | 


--------------------------------------------------------------------------------
/lulu/extractors/alive.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     match1,
 5 |     url_info,
 6 |     print_info,
 7 |     get_content,
 8 |     download_urls,
 9 |     playlist_not_supported,
10 | )
11 | 
12 | 
13 | __all__ = ['alive_download']
14 | site_info = 'Alive alive.in.th'
15 | 
16 | 
17 | def alive_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
18 |     html = get_content(url)
19 | 
20 |     title = match1(html, r'<meta property="og:title" content="([^"]+)"')
21 | 
22 |     url = match1(html, r'file: "(http://alive[^"]+)"')
23 |     _type, ext, size = url_info(url)
24 | 
25 |     print_info(site_info, title, _type, size)
26 |     if not info_only:
27 |         download_urls([url], title, ext, size, output_dir, merge=merge)
28 | 
29 | 
30 | download = alive_download
31 | download_playlist = playlist_not_supported(site_info)
32 | 


--------------------------------------------------------------------------------
/lulu/extractors/archive.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     url_info,
 5 |     print_info,
 6 |     get_content,
 7 |     download_urls,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.util.parser import get_parser
11 | 
12 | 
13 | __all__ = ['archive_download']
14 | site_info = 'Internet Archive archive.org'
15 | 
16 | 
17 | def archive_download(url, merge=True, info_only=False, **kwargs):
18 |     html = get_content(url)
19 |     parser = get_parser(html)
20 |     title = parser.find('meta', property='og:title')['content']
21 |     source = parser.find('meta', property='og:video')['content']
22 |     mime, ext, size = url_info(source)
23 | 
24 |     print_info(site_info, title, mime, size)
25 |     if not info_only:
26 |         download_urls([source], title, ext, size, merge=merge, **kwargs)
27 | 
28 | 
29 | download = archive_download
30 | download_playlist = playlist_not_supported(site_info)
31 | 


--------------------------------------------------------------------------------
/lulu/extractors/ku6.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import string
 4 | from urllib import parse
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['ku6_download']
17 | site_info = '酷6网 ku6.com'
18 | 
19 | 
20 | def ku6_download(url, info_only=False, **kwargs):
21 |     page = get_content(url)
22 |     video = match1(page, r'type: "video/mp4", src: "(.+)"').replace(' ', '%20')
23 |     video = parse.quote(
24 |         video, safe=string.printable
25 |     )
26 |     title = match1(page, r'document.title = "(.+)"')
27 |     _type, ext, size = url_info(video)
28 |     print_info(site_info, title, _type, size)
29 |     if not info_only:
30 |         download_urls([video], title, ext, size, **kwargs)
31 | 
32 | 
33 | download = ku6_download
34 | download_playlist = playlist_not_supported(site_info)
35 | 


--------------------------------------------------------------------------------
/lulu/extractors/giphy.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     url_info,
 5 |     print_info,
 6 |     get_content,
 7 |     download_urls,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.util.parser import get_parser
11 | 
12 | 
13 | __all__ = ['giphy_download']
14 | site_info = 'GIPHY giphy.com'
15 | 
16 | 
17 | def giphy_download(url, info_only=False, **kwargs):
18 |     html = get_content(url)
19 |     parser = get_parser(html)
20 | 
21 |     title = parser.find('meta', property='og:title')['content']
22 |     gif = parser.find('meta', property='og:image')['content']
23 |     video = parser.find('meta', property='og:video')['content']
24 | 
25 |     for url in [gif, video]:
26 |         _type, ext, size = url_info(url)
27 |         print_info(site_info, title, _type, size)
28 |         if not info_only:
29 |             download_urls([url], title, ext, size, **kwargs)
30 | 
31 | 
32 | download = giphy_download
33 | download_playlist = playlist_not_supported(site_info)
34 | 


--------------------------------------------------------------------------------
/lulu/extractors/freesound.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | 
 4 | from lulu.common import (
 5 |     url_info,
 6 |     print_info,
 7 |     get_content,
 8 |     download_urls,
 9 |     playlist_not_supported,
10 | )
11 | from lulu.util.parser import get_parser
12 | 
13 | 
14 | __all__ = ['freesound_download']
15 | site_info = 'Freesound freesound.org'
16 | 
17 | 
18 | def freesound_download(
19 |     url, output_dir='.', merge=True, info_only=False, **kwargs
20 | ):
21 |     page = get_content(url)
22 |     parser = get_parser(page)
23 |     title = parser.find('meta', property='og:title')['content']
24 |     preview_url = parser.find('meta', property='og:audio')['content']
25 | 
26 |     _type, ext, size = url_info(preview_url)
27 | 
28 |     print_info(site_info, title, _type, size)
29 |     if not info_only:
30 |         download_urls(
31 |             [preview_url], title, ext, size, output_dir, merge=merge, **kwargs
32 |         )
33 | 
34 | 
35 | download = freesound_download
36 | download_playlist = playlist_not_supported(site_info)
37 | 


--------------------------------------------------------------------------------
/lulu/extractors/magisto.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['magisto_download']
16 | site_info = 'Magisto magisto.com'
17 | 
18 | 
19 | def magisto_download(url, info_only=False, **kwargs):
20 |     video_hash = match1(url, r'video\/([a-zA-Z0-9]+)')
21 |     api_url = 'https://www.magisto.com/api/video/{}'.format(video_hash)
22 |     content = get_content(api_url)
23 |     data = json.loads(content)
24 |     title1 = data['title']
25 |     title2 = data['creator']
26 |     title = '{} - {}'.format(title1, title2)
27 |     url = data['video_direct_url']
28 |     _type, ext, size = url_info(url)
29 | 
30 |     print_info(site_info, title, _type, size)
31 |     if not info_only:
32 |         download_urls([url], title, ext, size, **kwargs)
33 | 
34 | 
35 | download = magisto_download
36 | download_playlist = playlist_not_supported(site_info)
37 | 


--------------------------------------------------------------------------------
/contrib/completion/lulu-completion.bash:
--------------------------------------------------------------------------------
 1 | # Bash completion definition for lulu.
 2 | 
 3 | _lulu () {
 4 |     COMPREPLY=()
 5 |     local IFS=$' \n'
 6 |     local cur=$2 prev=$3
 7 |     local -a opts_without_arg opts_with_arg
 8 |     opts_without_arg=(
 9 |         -V --version -h --help -i --info -u --url --json -n --no-merge
10 |         --no-caption -f --force --no-proxy -d --debug
11 |     )
12 |     opts_with_arg=(
13 |         -F --format -O --output-filename -o --output-dir -p --player
14 |         -c --cookies -x --http-proxy -y --extractor-proxy -t --timeout
15 |     )
16 | 
17 |     # Do not complete non option names
18 |     [[ $cur == -* ]] || return 1
19 | 
20 |     # Do not complete when the previous arg is an option expecting an argument
21 |     for opt in "${opts_with_arg[@]}"; do
22 |         [[ $opt == $prev ]] && return 1
23 |     done
24 | 
25 |     # Complete option names
26 |     COMPREPLY=( $(compgen -W "${opts_without_arg[*]} ${opts_with_arg[*]}" \
27 |                           -- "$cur") )
28 |     return 0
29 | }
30 | 
31 | complete -F _lulu lulu
32 | 


--------------------------------------------------------------------------------
/lulu/extractors/ehow.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | from lulu.util.parser import get_parser
14 | 
15 | __all__ = ['ehow_download']
16 | site_info = 'eHow ehow.com'
17 | 
18 | 
19 | def ehow_download(url, info_only=False, **kwargs):
20 | 
21 |     assert re.search(
22 |         r'https?://www.ehow.com/video_', url
23 |     ), 'URL you entered is not supported'
24 | 
25 |     html = get_content(url)
26 |     parser = get_parser(html)
27 |     title = parser.find('meta', property='og:title')['content']
28 |     video = parser.find('meta', property='og:video')['content']
29 |     url = match1(video, r'source=(.+?)&')
30 |     _type, ext, size = url_info(url)
31 |     print_info(site_info, title, _type, size)
32 | 
33 |     if not info_only:
34 |         download_urls([url], title, ext, size, **kwargs)
35 | 
36 | 
37 | download = ehow_download
38 | download_playlist = playlist_not_supported(site_info)
39 | 


--------------------------------------------------------------------------------
/lulu/extractors/huomaotv.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     match1,
 5 |     print_info,
 6 |     get_content,
 7 |     download_url_ffmpeg,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.util.parser import get_parser
11 | 
12 | 
13 | __all__ = ['huomaotv_download']
14 | site_info = '火猫直播 huomao.com'
15 | 
16 | 
17 | def huomaotv_download(
18 |     url, output_dir='.', merge=True, info_only=False, **kwargs
19 | ):
20 |     room_id_pattern = r'huomao.com/(\d+)'
21 |     room_id = match1(url, room_id_pattern)
22 |     html = get_content(
23 |         'http://m.huomao.com/mobile/mob_live/{}'.format(room_id)
24 |     )
25 |     parser = get_parser(html)
26 |     m3u8_url = parser.source['src']
27 |     title = parser.title.text
28 | 
29 |     print_info(site_info, title, 'm3u8', float('inf'))
30 | 
31 |     if not info_only:
32 |         download_url_ffmpeg(
33 |             m3u8_url, title, 'mp4', None, output_dir=output_dir, merge=merge
34 |         )
35 | 
36 | 
37 | download = huomaotv_download
38 | download_playlist = playlist_not_supported(site_info)
39 | 


--------------------------------------------------------------------------------
/lulu/extractors/quanmin.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     print_info,
 7 |     get_content,
 8 |     download_urls,
 9 |     playlist_not_supported,
10 | )
11 | 
12 | 
13 | __all__ = ['quanmin_download']
14 | site_info = '全民直播 quanmin.tv'
15 | 
16 | 
17 | def quanmin_download(url, info_only=False, **kwargs):
18 |     roomid = url.split('/')[3].split('?')[0]
19 | 
20 |     json_request_url = (
21 |         'https://m.quanmin.tv/json/rooms/{}/noinfo6.json'.format(roomid)
22 |     )
23 |     content = get_content(json_request_url)
24 |     data = json.loads(content)
25 | 
26 |     title = data['title']
27 | 
28 |     if not data['play_status']:
29 |         raise ValueError('The live stream is not online!')
30 |     real_url = 'https://flv.quanmin.tv/live/{}.flv'.format(roomid)
31 | 
32 |     print_info(site_info, title, 'flv', float('inf'))
33 |     if not info_only:
34 |         download_urls([real_url], title, 'flv', None, **kwargs)
35 | 
36 | 
37 | download = quanmin_download
38 | download_playlist = playlist_not_supported(site_info)
39 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright 2018-present, iawia002
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 6 | 
 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 8 | 
 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
10 | 


--------------------------------------------------------------------------------
/lulu/extractors/douyin.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     get_content,
 8 |     playlist_not_supported,
 9 | )
10 | from lulu.extractor import SimpleExtractor
11 | 
12 | 
13 | __all__ = ['douyin_download_by_url']
14 | site_info = 'douyin'
15 | 
16 | 
17 | class DouYin(SimpleExtractor):
18 |     def extract(self, url, **kwargs):
19 |         self.site_info = site_info
20 |         page_content = get_content(url)
21 |         match_rule = re.compile(r'var data = \[(.*?)\];')
22 |         video_info = json.loads(match_rule.findall(page_content)[0])
23 |         urls = [video_info['video']['play_addr']['url_list'][0].replace('playwm', 'play')]
24 |         if video_info['cha_list']:
25 |             title = video_info['cha_list'][0]['cha_name']
26 |         else:
27 |             title = video_info['desc']
28 |             
29 |         return {
30 |             'urls': urls,
31 |             'title': title,
32 |         }
33 | 
34 | 
35 | download = douyin_download_by_url = DouYin()
36 | download_playlist = playlist_not_supported(site_info)
37 | 


--------------------------------------------------------------------------------
/lulu/extractors/bandcamp.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 | )
12 | 
13 | 
14 | __all__ = ['bandcamp_download']
15 | site_info = 'Bandcamp bandcamp.com'
16 | 
17 | 
18 | def bandcamp_download(
19 |     url, output_dir='.', merge=True, info_only=False, **kwargs
20 | ):
21 |     html = get_content(url)
22 |     trackinfo = json.loads(
23 |         match1(html, r'trackinfo:\s+(\[.+\]),')
24 |     )
25 |     for track in trackinfo:
26 |         track_num = track['track_num']
27 |         title = '{}. {}'.format(track_num, track['title'])
28 |         file_url = track['file']['mp3-128']
29 |         mime, ext, size = url_info(file_url)
30 | 
31 |         print_info(site_info, title, mime, size)
32 |         if not info_only:
33 |             download_urls(
34 |                 [file_url], title, ext, size, output_dir, merge=merge,
35 |                 **kwargs
36 |             )
37 | 
38 | 
39 | download = bandcamp_download
40 | download_playlist = bandcamp_download
41 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | Lulu
 2 | ====
 3 | 
 4 | .. image:: https://travis-ci.org/iawia002/Lulu.svg?branch=master
 5 |     :target: https://travis-ci.org/iawia002/Lulu
 6 | 
 7 | .. image:: https://ci.appveyor.com/api/projects/status/ph9ypng9g01mdr6d/branch/master?svg=true
 8 |     :target: https://ci.appveyor.com/project/iawia002/lulu/branch/master
 9 | 
10 | `Lulu <https://github.com/iawia002/Lulu>`__ is a friendly you-get fork (⏬ Dumb downloader that scrapes the web)
11 | 
12 | .. code:: console
13 | 
14 |     $ lulu https://www.bilibili.com/video/av18295259/
15 |     site:                Bilibili
16 |     title:               【中文八级】俄罗斯人的名字超乎你的想象
17 |     stream:
18 |     - format:        flv720
19 |       container:     flv
20 |       size:          175.4 MiB (183914793 bytes)
21 |     # download-with: lulu --format=flv720 [URL]
22 | 
23 |     Downloading 【中文八级】俄罗斯人的名字超乎你的想象.flv ...
24 |     100% (175.4/175.4MB) ├████████████████████████████████████████┤[1/1]    3 MB/s
25 | 
26 |     Downloading 【中文八级】俄罗斯人的名字超乎你的想象.cmt.xml ...
27 | 
28 | Full documentation is on `GitHub <https://github.com/iawia002/Lulu/blob/master/README.md>`__
29 | 


--------------------------------------------------------------------------------
/lulu/extractors/iqilu.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['iqilu_download']
17 | site_info = '齐鲁网 iqilu.com'
18 | 
19 | 
20 | def iqilu_download(url, info_only=False, **kwargs):
21 |     if re.match(r'http://v.iqilu.com/\w+', url):
22 |         patt = r'url\s*:\s*\[([^\]]+)\]'
23 | 
24 |         # URL in webpage
25 |         html = get_content(url)
26 |         player_data = '[{}]'.format(match1(html, patt))
27 |         urls = json.loads(player_data)
28 |         url = urls[0]['stream_url']
29 | 
30 |         # grab title
31 |         title = match1(html, r'<meta name="description" content="(.*?)\"\W')
32 | 
33 |         _type, ext, size = url_info(url)
34 |         print_info(site_info, title, _type, size)
35 |         if not info_only:
36 |             download_urls([url], title, ext, size, **kwargs)
37 | 
38 | 
39 | download = iqilu_download
40 | download_playlist = playlist_not_supported(site_info)
41 | 


--------------------------------------------------------------------------------
/lulu/extractors/metacafe.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     print_info,
 9 |     get_content,
10 |     download_url_ffmpeg,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['metacafe_download']
16 | site_info = 'Metacafe metacafe.com'
17 | 
18 | 
19 | def metacafe_download(url, info_only=False, **kwargs):
20 |     if re.match(r'http://www.metacafe.com/watch/\w+', url):
21 |         html = get_content(url)
22 |         title = match1(html, r'<meta property="og:title" content="([^"]*)"')
23 | 
24 |         data = match1(
25 |             html,
26 |             r"<script type='text/json' id='json_video_data'>(.+)</script>"
27 |         )
28 |         data = json.loads(data)
29 |         m3u8_url = data['sources'][0]['src']
30 |         print_info(
31 |             site_info, title, 'm3u8', 0, m3u8_url=m3u8_url, m3u8_type='master'
32 |         )
33 |         if not info_only:
34 |             download_url_ffmpeg(m3u8_url, title, 'mp4', **kwargs)
35 | 
36 | 
37 | download = metacafe_download
38 | download_playlist = playlist_not_supported(site_info)
39 | 


--------------------------------------------------------------------------------
/lulu/extractors/ted.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | 
14 | __all__ = ['ted_download']
15 | site_info = 'TED ted.com'
16 | 
17 | 
18 | def ted_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
19 |     html = get_content(url)
20 |     patt = r'"__INITIAL_DATA__"\s*:\s*\{(.+)\}'
21 |     metadata = json.loads('{' + match1(html, patt) + '}')
22 |     title = metadata['talks'][0]['title']
23 |     nativeDownloads = metadata['talks'][0]['downloads']['nativeDownloads']
24 |     for quality in ['high', 'medium', 'low']:
25 |         if quality in nativeDownloads:
26 |             url = nativeDownloads[quality]
27 |             _type, ext, size = url_info(url)
28 |             print_info(site_info, title, _type, size)
29 |             if not info_only:
30 |                 download_urls([url], title, ext, size, output_dir, merge=merge)
31 |             break
32 | 
33 | 
34 | download = ted_download
35 | download_playlist = playlist_not_supported(site_info)
36 | 


--------------------------------------------------------------------------------
/lulu/extractors/bcy.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from pathlib import Path
 4 | 
 5 | from lulu.common import (
 6 |     print_info,
 7 |     get_content,
 8 |     download_urls,
 9 |     playlist_not_supported,
10 | )
11 | from lulu.util.parser import get_parser
12 | 
13 | 
14 | __all__ = ['bcy_download']
15 | site_info = '半次元 bcy.net'
16 | 
17 | 
18 | def bcy_download(url, output_dir='.', info_only=False, **kwargs):
19 |     html = get_content(url)
20 |     parser = get_parser(html)
21 |     title = parser.h1.text.strip()
22 |     output_dir = Path(output_dir) / title
23 |     imgs = parser.find_all('img', class_='detail_std detail_clickable')
24 |     print_info(site_info, title, 'jpg', 0)
25 |     if not info_only:
26 |         for img in imgs:
27 |             # https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg/w650  # noqa
28 |             img = img['src'][:-5]
29 |             filename, ext = img.split('/')[-1].split('.')
30 |             download_urls(
31 |                 [img], filename, ext, 0, output_dir, **kwargs
32 |             )
33 | 
34 | 
35 | download = bcy_download
36 | download_playlist = playlist_not_supported(site_info)
37 | 


--------------------------------------------------------------------------------
/lulu/extractors/heavymusic.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | from urllib import parse
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 | )
13 | 
14 | 
15 | __all__ = ['heavymusic_download']
16 | site_info = 'Heavy Music Archive heavy-music.ru'
17 | 
18 | 
19 | def heavymusic_download(url, info_only=False, **kwargs):
20 |     html = get_content(url)
21 |     tracks = re.findall(r'href="(online2\.php[^"]+)"', html)
22 |     for track in tracks:
23 |         band = match1(track, r'band=([^&]*)')
24 |         album = match1(track, r'album=([^&]*)')
25 |         title = match1(track, r'track=([^&]*)')
26 |         file_url = (
27 |             'http://www.heavy-music.ru/online2.php?band={}&album={}&'
28 |             'track={}'.format(
29 |                 parse.quote(band), parse.quote(album), parse.quote(title)
30 |             )
31 |         )
32 |         _, _, size = url_info(file_url)
33 | 
34 |         print_info(site_info, title, 'mp3', size)
35 |         if not info_only:
36 |             download_urls([file_url], title[:-4], 'mp3', size, **kwargs)
37 | 
38 | 
39 | download = heavymusic_download
40 | download_playlist = heavymusic_download
41 | 


--------------------------------------------------------------------------------
/lulu/util/fs.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import platform
 4 | 
 5 | 
 6 | def legitimize(text, os=platform.system()):
 7 |     """Converts a string to a valid filename.
 8 |     """
 9 | 
10 |     # POSIX systems
11 |     text = text.translate({
12 |         0: None,
13 |         ord('/'): '-',
14 |         ord('|'): '-',
15 |     })
16 | 
17 |     if os == 'Windows':
18 |         # Windows (non-POSIX namespace)
19 |         text = text.translate({
20 |             # Reserved in Windows VFAT and NTFS
21 |             ord(':'): '-',
22 |             ord('*'): '-',
23 |             ord('?'): '-',
24 |             ord('\\'): '-',
25 |             ord('\"'): '\'',
26 |             # Reserved in Windows VFAT
27 |             ord('+'): '-',
28 |             ord('<'): '-',
29 |             ord('>'): '-',
30 |             ord('['): '(',
31 |             ord(']'): ')',
32 |         })
33 |     else:
34 |         # *nix
35 |         if os == 'Darwin':
36 |             # Mac OS HFS+
37 |             text = text.translate({
38 |                 ord(':'): '-',
39 |             })
40 | 
41 |         # Remove leading .
42 |         if text.startswith("."):
43 |             text = text[1:]
44 | 
45 |     text = text[:80]  # Trim to 82 Unicode characters long
46 |     return text
47 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | *$py.class
 5 | 
 6 | # C extensions
 7 | *.so
 8 | 
 9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | 
27 | # PyInstaller
28 | #  Usually these files are written by a python script from a template
29 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 | 
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 | 
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 | 
48 | # Translations
49 | *.mo
50 | *.pot
51 | 
52 | # Django stuff:
53 | *.log
54 | 
55 | # Sphinx documentation
56 | docs/_build/
57 | 
58 | # PyBuilder
59 | target/
60 | 
61 | # Misc
62 | *.3gp
63 | *.asf
64 | *.download
65 | *.f4v
66 | *.flv
67 | *.gif
68 | *.html
69 | *.jpg
70 | *.lrc
71 | *.mkv
72 | *.mp3
73 | *.mp4
74 | *.mpg
75 | *.png
76 | *.srt
77 | *.ts
78 | *.webm
79 | *.xml
80 | /.env
81 | /.idea
82 | *.m4a
83 | *.DS_Store
84 | 


--------------------------------------------------------------------------------
/lulu/extractors/pixivision.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | from pathlib import Path
 4 | 
 5 | from lulu.common import (
 6 |     url_size,
 7 |     print_info,
 8 |     get_content,
 9 |     download_urls,
10 |     playlist_not_supported,
11 | )
12 | from lulu.config import FAKE_HEADERS
13 | from lulu.util.parser import get_parser
14 | 
15 | 
16 | __all__ = ['pixivision_download']
17 | site_info = 'pixivision pixivision.net'
18 | 
19 | 
20 | def pixivision_download(url, output_dir='.', info_only=False, **kwargs):
21 |     html = get_content(url)
22 |     parser = get_parser(html)
23 |     title = parser.h1.text.strip()
24 |     output_dir = Path(output_dir) / title
25 |     imgs = parser.find_all('img', class_='am__work__illust')
26 |     print_info(site_info, title, 'jpg', 0)
27 |     if not info_only:
28 |         headers = FAKE_HEADERS.copy()
29 |         headers.update({'Referer': url})
30 |         for img in imgs:
31 |             img = img['src']
32 |             size = url_size(img, headers=headers)
33 |             filename, ext = img.split('/')[-1].split('.')
34 |             download_urls(
35 |                 [img], filename, ext, size, output_dir, refer=url, **kwargs
36 |             )
37 | 
38 | 
39 | download = pixivision_download
40 | download_playlist = playlist_not_supported(site_info)
41 | 


--------------------------------------------------------------------------------
/lulu/extractors/kuaishou.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | 
 5 | from lulu.common import (
 6 |     get_content,
 7 |     playlist_not_supported,
 8 | )
 9 | from lulu.extractor import SimpleExtractor
10 | 
11 | 
12 | __all__ = ['kuaishou_download_by_url']
13 | site_info = 'kuaishou.com'
14 | 
15 | 
16 | class Kuaishou(SimpleExtractor):
17 |     def extract(self, url, **kwargs):
18 |         self.site_info = site_info
19 |         page = get_content(url)
20 |         title = re.search(
21 |             r'<meta\s+property="og:title"\s+content="(.+?)"/>', page
22 |         ).group(1)
23 |         try:
24 |             url = re.search(
25 |                 r'<meta\s+property="og:video:url"\s+content="(.+?)"/>', page
26 |             ).group(1)
27 |             file_format = url.split('.')[-1]
28 |         except Exception:  # extract image
29 |             url = re.search(
30 |                 r'<meta\s+property="og:image"\s+content="(.+?)"/>', page
31 |             ).group(1)
32 |             file_format = url.split('.')[-1]
33 |         return {
34 |             'title': title,
35 |             'urls': [url],
36 |             'file_format': file_format,
37 |         }
38 | 
39 | 
40 | download = kuaishou_download_by_url = Kuaishou()
41 | download_playlist = playlist_not_supported(site_info)
42 | 


--------------------------------------------------------------------------------
/lulu/extractors/musicplayon.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     match1,
 5 |     url_info,
 6 |     get_content,
 7 | )
 8 | from lulu.extractor import VideoExtractor
 9 | 
10 | 
11 | class MusicPlayOn(VideoExtractor):
12 |     name = 'MusicPlayOn musicplayon.com'
13 | 
14 |     stream_types = [
15 |         {'id': '720p HD'},
16 |         {'id': '360p SD'},
17 |     ]
18 | 
19 |     def prepare(self, **kwargs):
20 |         content = get_content(self.url)
21 | 
22 |         self.title = match1(
23 |             content, r'setup\[\'title\'\] = "([^"]+)";'
24 |         )
25 | 
26 |         for s in self.stream_types:
27 |             quality = s['id']
28 |             src = match1(
29 |                 content,
30 |                 r'src: "([^"]+)", "data-res": "{}"'.format(quality)
31 |             )
32 |             if src is not None:
33 |                 url = 'https://en.musicplayon.com{}'.format(src)
34 |                 self.streams[quality] = {'url': url}
35 | 
36 |     def extract(self, **kwargs):
37 |         for i in self.streams:
38 |             s = self.streams[i]
39 |             _, s['container'], s['size'] = url_info(s['url'])
40 |             s['src'] = [s['url']]
41 | 
42 | 
43 | site = MusicPlayOn()
44 | download = site.download_by_url
45 | # TBD: implement download_playlist
46 | 


--------------------------------------------------------------------------------
/lulu/extractors/facebook.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     urls_size,
 9 |     unicodize,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | 
16 | 
17 | __all__ = ['facebook_download']
18 | site_info = 'facebook.com'
19 | 
20 | 
21 | def facebook_download(
22 |     url, output_dir='.', merge=True, info_only=False, **kwargs
23 | ):
24 |     html = get_content(url)
25 | 
26 |     title = match1(html, r'<title id="pageTitle">(.+)</title>')
27 | 
28 |     if title is None:
29 |         title = url
30 | 
31 |     sd_urls = list(set([
32 |         unicodize(str.replace(i, '\\/', '/'))
33 |         for i in re.findall(r'sd_src_no_ratelimit:"([^"]*)"', html)
34 |     ]))
35 |     hd_urls = list(set([
36 |         unicodize(str.replace(i, '\\/', '/'))
37 |         for i in re.findall(r'hd_src_no_ratelimit:"([^"]*)"', html)
38 |     ]))
39 |     urls = hd_urls if hd_urls else sd_urls
40 | 
41 |     _type, ext, size = url_info(urls[0])
42 |     size = urls_size(urls)
43 | 
44 |     print_info(site_info, title, _type, size)
45 |     if not info_only:
46 |         download_urls(urls, title, ext, size, output_dir, merge=False)
47 | 
48 | 
49 | download = facebook_download
50 | download_playlist = playlist_not_supported(site_info)
51 | 


--------------------------------------------------------------------------------
/lulu/extractors/icourses.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 | )
12 | 
13 | 
14 | __all__ = ['icourses_download', 'icourses_download_playlist']
15 | site_info = '爱课程 icourses.cn'
16 | 
17 | 
18 | def _download(item, **kwargs):
19 |     url = item['fullLinkUrl']
20 |     title = item['title'].strip()
21 |     _, ext, size = url_info(url)
22 |     print_info(site_info=site_info, title=title, type=ext, size=size)
23 |     if not kwargs.get('info_only'):
24 |         download_urls([url], title, ext, size, **kwargs)
25 | 
26 | 
27 | def icourses_download(url, **kwargs):
28 |     page = get_content(url)
29 |     data = json.loads(match1(page, r'var _sourceArrStr = (.+?);'))
30 | 
31 |     if 'resId' in url:  # 下载播放列表中指定视频
32 |         _id = match1(url, r'resId=([\w-]+)')
33 |         results = list(filter(lambda x: x['id'] == _id, data))
34 |         _download(results[0], **kwargs)
35 |     else:  # 下载第一个
36 |         _download(data[0], **kwargs)
37 | 
38 | 
39 | def icourses_download_playlist(url, **kwargs):
40 |     page = get_content(url)
41 |     data = json.loads(match1(page, r'var _sourceArrStr = (.+?);'))
42 |     for item in data:
43 |         _download(item, **kwargs)
44 | 
45 | 
46 | download = icourses_download
47 | download_playlist = icourses_download_playlist
48 | 


--------------------------------------------------------------------------------
/lulu/extractors/theplatform.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | from html import unescape
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     print_info,
 9 |     get_content,
10 |     download_rtmp_url,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | site_info = 'thePlatform.com'
16 | 
17 | 
18 | def theplatform_download_by_pid(
19 |     pid, title, output_dir='.', merge=True, info_only=False, **kwargs
20 | ):
21 |     smil_url = (
22 |         'http://link.theplatform.com/s/dJ5BDC/{}/meta.smil?format=smil'
23 |         '&mbr=true'.format(pid)
24 |     )
25 |     smil = get_content(smil_url)
26 |     smil_base = unescape(match1(smil, r'<meta base="([^"]+)"'))
27 |     smil_videos = {
28 |         y: x for x, y in dict(
29 |             re.findall(r'<video src="([^"]+)".+height="([^"]+)"', smil)
30 |         ).items()
31 |     }
32 |     for height in ['1080', '720', '480', '360', '240', '216']:
33 |         if height in smil_videos:
34 |             smil_video = smil_videos[height]
35 |             break
36 |     assert smil_video
37 | 
38 |     _type, ext, size = 'mp4', 'mp4', 0
39 | 
40 |     print_info(site_info, title, _type, size)
41 |     if not info_only:
42 |         download_rtmp_url(
43 |             url=smil_base, title=title, ext=ext,
44 |             params={"-y": '{}:{}'.format(ext, smil_video)},
45 |             output_dir=output_dir
46 |         )
47 | 
48 | 
49 | download = theplatform_download_by_pid
50 | download_playlist = playlist_not_supported(site_info)
51 | 


--------------------------------------------------------------------------------
/lulu/extractors/iwara.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | from lulu.config import FAKE_HEADERS
14 | 
15 | 
16 | headers = FAKE_HEADERS.copy()
17 | headers.update({
18 |     'DNT': '1',
19 |     'Accept-Encoding': 'gzip, deflate, sdch, br',
20 |     'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
21 |     'Upgrade-Insecure-Requests': '1',
22 |     'Connection': 'keep-alive',
23 |     'Save-Data': 'on',
24 |     'Cookie': 'has_js=1;show_adult=1',
25 | })
26 | 
27 | 
28 | __all__ = ['iwara_download']
29 | site_info = 'Iwara iwara.tv'
30 | 
31 | 
32 | def iwara_download(url, info_only=False, **kwargs):
33 |     video_hash = match1(url, r'http://\w+.iwara.tv/videos/(\w+)')
34 |     video_url = match1(url, r'(http://\w+.iwara.tv)/videos/\w+')
35 |     html = get_content(url, headers=headers)
36 |     title = match1(html, r'<title>(.*)</title>')
37 |     api_url = '{}/api/video/{}'.format(video_url, video_hash)
38 |     content = get_content(api_url, headers=headers)
39 |     data = json.loads(content)
40 |     _type, ext, size = url_info(data[0]['uri'], headers=headers)
41 |     down_urls = data[0]['uri']
42 |     print_info(down_urls, title, _type, size)
43 | 
44 |     if not info_only:
45 |         download_urls([down_urls], title, ext, size, headers=headers, **kwargs)
46 | 
47 | 
48 | download = iwara_download
49 | download_playlist = playlist_not_supported(site_info)
50 | 


--------------------------------------------------------------------------------
/lulu/extractors/naver.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | import urllib.parse
 6 | import urllib.request
 7 | 
 8 | from lulu.util import log
 9 | from lulu.common import (
10 |     url_size,
11 |     print_info,
12 |     get_content,
13 |     download_urls,
14 |     playlist_not_supported,
15 | )
16 | 
17 | 
18 | __all__ = ['naver_download_by_url']
19 | site_info = 'NAVER naver.com'
20 | 
21 | 
22 | def naver_download_by_url(url, info_only=False, **kwargs):
23 |     ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'
24 |     page = get_content(url)
25 |     og_video_url = re.search(
26 |         r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page
27 |     ).group(1)
28 |     params_dict = urllib.parse.parse_qs(
29 |         urllib.parse.urlparse(og_video_url).query
30 |     )
31 |     vid = params_dict['vid'][0]
32 |     key = params_dict['outKey'][0]
33 |     meta_str = get_content(ep.format(vid, key))
34 |     meta_json = json.loads(meta_str)
35 |     if 'errorCode' in meta_json:
36 |         log.wtf(meta_json['errorCode'])
37 |     title = meta_json['meta']['subject']
38 |     videos = meta_json['videos']['list']
39 |     video_list = sorted(
40 |         videos, key=lambda video: video['encodingOption']['width']
41 |     )
42 |     video_url = video_list[-1]['source']
43 |     size = url_size(video_url)
44 |     print_info(site_info, title, 'mp4', size)
45 |     if not info_only:
46 |         download_urls([video_url], title, 'mp4', size, **kwargs)
47 | 
48 | 
49 | download = naver_download_by_url
50 | download_playlist = playlist_not_supported(site_info)
51 | 


--------------------------------------------------------------------------------
/lulu/extractors/veoh.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['veoh_download']
16 | site_info = 'Veoh veoh.com'
17 | 
18 | 
19 | def veoh_download(url, info_only=False, **kwargs):
20 |     '''Get item_id'''
21 |     if re.match(r'http://www.veoh.com/watch/\w+', url):
22 |         item_id = match1(url, r'http://www.veoh.com/watch/(\w+)')
23 |     elif re.match(r'http://www.veoh.com/m/watch.php\?v=\.*', url):
24 |         item_id = match1(url, r'http://www.veoh.com/m/watch.php\?v=(\w+)')
25 |     else:
26 |         raise NotImplementedError('Cannot find item ID')
27 |     veoh_download_by_id(item_id, info_only=info_only, **kwargs)
28 | 
29 | 
30 | def veoh_download_by_id(item_id, info_only=False, **kwargs):
31 |     """Source: Android mobile
32 |     """
33 |     webpage_url = (
34 |         'http://www.veoh.com/m/watch.php?v={item_id}&quality=1'.format(
35 |             item_id=item_id
36 |         )
37 |     )
38 |     # grab download URL
39 |     a = get_content(webpage_url)
40 |     url = match1(a, r'<source src="(.*?)\"\W')
41 | 
42 |     # grab title
43 |     title = match1(a, r'<meta property="og:title" content="([^"]*)"')
44 | 
45 |     type_, ext, size = url_info(url)
46 |     print_info(site_info, title, type_, size)
47 |     if not info_only:
48 |         download_urls([url], title, ext, total_size=None, **kwargs)
49 | 
50 | 
51 | download = veoh_download
52 | download_playlist = playlist_not_supported(site_info)
53 | 


--------------------------------------------------------------------------------
/lulu/extractors/vidto.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import time
 5 | from urllib import (
 6 |     parse,
 7 |     request,
 8 | )
 9 | 
10 | from lulu.common import (
11 |     url_info,
12 |     print_info,
13 |     get_content,
14 |     download_urls,
15 |     playlist_not_supported,
16 | )
17 | from lulu.util import log
18 | from lulu.config import FAKE_HEADERS
19 | 
20 | 
21 | __all__ = ['vidto_download']
22 | site_info = 'Vidto vidto.me'
23 | 
24 | 
25 | def vidto_download(url, info_only=False, **kwargs):
26 |     html = get_content(url)
27 |     params = {}
28 |     r = re.findall(
29 |         r'type="(?:hidden|submit)?"(?:.*?)name="(.+?)"\s* value="?(.+?)">',
30 |         html
31 |     )
32 |     for name, value in r:
33 |         params[name] = value
34 |     data = parse.urlencode(params).encode('utf-8')
35 |     req = request.Request(url, headers=FAKE_HEADERS)
36 |     print('Please wait for 6 seconds...')
37 |     time.sleep(6)
38 |     print('Starting')
39 |     new_html = request.urlopen(req, data).read().decode('utf-8', 'replace')
40 |     new_stff = re.search(r'lnk_download" href="(.*?)">', new_html)
41 |     if new_stff:
42 |         url = new_stff.group(1)
43 |         title = params['fname']
44 |         _type = ''
45 |         ext = ''
46 |         a, b, size = url_info(url)
47 |         print_info(site_info, title, _type, size)
48 |         if not info_only:
49 |             download_urls([url], title, ext, size, **kwargs)
50 |     else:
51 |         log.wtf("Cann't find link, please review")
52 | 
53 | 
54 | download = vidto_download
55 | download_playlist = playlist_not_supported(site_info)
56 | 


--------------------------------------------------------------------------------
/contrib/completion/lulu.fish:
--------------------------------------------------------------------------------
 1 | # Fish completion definition for lulu.
 2 | 
 3 | complete -c lulu -s V -l version -d 'print version and exit'
 4 | complete -c lulu -s h -l help -d 'print help and exit'
 5 | complete -c lulu -s i -l info -d 'print extracted information'
 6 | complete -c lulu -s u -l url -d 'print extracted information'
 7 | complete -c lulu -l json -d 'print extracted URLs in JSON format'
 8 | complete -c lulu -s n -l no-merge -d 'do not merge video parts'
 9 | complete -c lulu -l no-caption -d 'do not download captions'
10 | complete -c lulu -s f -l force -d 'force overwrite existing files'
11 | complete -c lulu -s F -l format -x -d 'set video format to the specified stream id'
12 | complete -c lulu -s O -l output-filename -d 'set output filename' \
13 |          -x -a '(__fish_complete_path (commandline -ct) "output filename")'
14 | complete -c lulu -s o -l output-dir  -d 'set output directory' \
15 |          -x -a '(__fish_complete_directories (commandline -ct) "output directory")'
16 | complete -c lulu -s p -l player -x -d 'stream extracted URL to the specified player'
17 | complete -c lulu -s c -l cookies -d 'load cookies.txt or cookies.sqlite' \
18 |          -x -a '(__fish_complete_path (commandline -ct) "cookies.txt or cookies.sqlite")'
19 | complete -c lulu -s x -l http-proxy -x -d 'use the specified HTTP proxy for downloading'
20 | complete -c lulu -s y -l extractor-proxy -x -d 'use the specified HTTP proxy for extraction only'
21 | complete -c lulu -l no-proxy -d 'do not use a proxy'
22 | complete -c lulu -s t -l timeout -x -d 'set socket timeout'
23 | complete -c lulu -s d -l debug -d 'show traceback and other debug info'
24 | 


--------------------------------------------------------------------------------
/contrib/completion/_lulu:
--------------------------------------------------------------------------------
 1 | #compdef lulu
 2 | 
 3 | # Zsh completion definition for iawia002/Lulu.
 4 | 
 5 | setopt localoptions noshwordsplit noksharrays
 6 | local -a args
 7 | 
 8 | args=(
 9 |     '(- : *)'{-V,--version}'[print version and exit]'
10 |     '(- : *)'{-h,--help}'[print help and exit]'
11 |     '(-i --info)'{-i,--info}'[print extracted information]'
12 |     '(-u --url)'{-u,--url}'[print extracted information with URLs]'
13 |     '(--json)--json[print extracted URLs in JSON format]'
14 |     '(-n --no-merge)'{-n,--no-merge}'[do not merge video parts]'
15 |     '(--no-caption)--no-caption[do not download captions]'
16 |     '(-f --force)'{-f,--force}'[force overwrite existing files]'
17 |     '(-F --format)'{-F,--format}'[set video format to the specified stream id]:stream id'
18 |     '(-O --output-filename)'{-O,--output-filename}'[set output filename]:filename:_files'
19 |     '(-o --output-dir)'{-o,--output-dir}'[set output directory]:directory:_files -/'
20 |     '(-p --player)'{-p,--player}'[stream extracted URL to the specified player]:player and options'
21 |     '(-c --cookies)'{-c,--cookies}'[load cookies.txt or cookies.sqlite]:cookies file:_files'
22 |     '(-x --http-proxy)'{-x,--http-proxy}'[use the specified HTTP proxy for downloading]:host\:port:'
23 |     '(-y --extractor-proxy)'{-y,--extractor-proxy}'[use the specified HTTP proxy for extraction only]:host\:port'
24 |     '(--no-proxy)--no-proxy[do not use a proxy]'
25 |     '(-t --timeout)'{-t,--timeout}'[set socket timeout]:seconds'
26 |     '(-d --debug)'{-d,--debug}'[show traceback and other debug info]'
27 |     '*: :_guard "^-*" url'
28 | )
29 | _arguments -S -s $args
30 | 


--------------------------------------------------------------------------------
/lulu/extractors/kuwo.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 | )
12 | 
13 | 
14 | __all__ = ['kuwo_download']
15 | site_info = '酷我音乐 kuwo.cn'
16 | 
17 | 
18 | def kuwo_download_by_rid(rid, info_only=False, **kwargs):
19 |     html = get_content(
20 |         'http://player.kuwo.cn/webmusic/st/getNewMuiseByRid?rid='
21 |         'MUSIC_{}'.format(rid)
22 |     )
23 |     title = match1(html, r'<name>(.*)</name>')
24 |     if not title:
25 |         title = rid
26 |     # format =aac|mp3 ->to get aac format=mp3 ->to get mp3
27 |     url = get_content(
28 |         'http://antiserver.kuwo.cn/anti.s?format=mp3&rid=MUSIC_{}&'
29 |         'type=convert_url&response=url'.format(rid)
30 |     )
31 |     songtype, ext, size = url_info(url)
32 |     print_info(site_info, title, songtype, size)
33 |     if not info_only:
34 |         download_urls([url], title, ext, size, **kwargs)
35 | 
36 | 
37 | def kuwo_playlist_download(url, info_only=False, **kwargs):
38 |     html = get_content(url)
39 |     matched = set(re.compile('yinyue/(\d+)').findall(html))
40 |     for rid in matched:
41 |         kuwo_download_by_rid(rid, info_only, **kwargs)
42 | 
43 | 
44 | def kuwo_download(url, info_only=False, **kwargs):
45 |     if 'www.kuwo.cn/yinyue' in url:
46 |         rid = match1(url, 'yinyue/(\d+)')
47 |         kuwo_download_by_rid(rid, info_only, **kwargs)
48 |     else:
49 |         kuwo_playlist_download(url, info_only, **kwargs)
50 | 
51 | 
52 | download = kuwo_download
53 | download_playlist = kuwo_playlist_download
54 | 


--------------------------------------------------------------------------------
/lulu/extractors/vine.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['vine_download']
16 | site_info = 'Vine.co'
17 | 
18 | 
19 | def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
20 |     html = get_content(url)
21 |     video_id = match1(url, r'vine.co/v/([^/]+)')
22 |     title = match1(html, r'<title>([^<]*)</title>')
23 |     stream = match1(
24 |         html,
25 |         r'<meta property="twitter:player:stream" content="([^"]*)">'
26 |     )
27 |     if not stream:  # https://vine.co/v/.../card
28 |         stream = match1(html, r'"videoUrl":"([^"]+)"')
29 |         if stream:
30 |             stream = stream.replace('\\/', '/')
31 |         else:
32 |             posts_url = 'https://archive.vine.co/posts/{}.json'.format(
33 |                 video_id
34 |             )
35 |             json_data = json.loads(get_content(posts_url))
36 |             stream = json_data['videoDashUrl']
37 |             title = json_data['description']
38 |             if title == '':
39 |                 title = '{}_{}'.format(
40 |                     json_data['username'].replace(' ', '_'), video_id
41 |                 )
42 | 
43 |     mime, ext, size = url_info(stream)
44 | 
45 |     print_info(site_info, title, mime, size)
46 |     if not info_only:
47 |         download_urls([stream], title, ext, size, output_dir, merge=merge)
48 | 
49 | 
50 | download = vine_download
51 | download_playlist = playlist_not_supported(site_info)
52 | 


--------------------------------------------------------------------------------
/lulu/extractors/dailymotion.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | import urllib.parse
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     unicodize,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | 
16 | 
17 | __all__ = ['dailymotion_download']
18 | site_info = 'Dailymotion dailymotion.com'
19 | 
20 | 
21 | def rebuilt_url(url):
22 |     path = urllib.parse.urlparse(url).path
23 |     aid = path.split('/')[-1].split('_')[0]
24 |     return 'http://www.dailymotion.com/embed/video/{}?autoplay=1'.format(aid)
25 | 
26 | 
27 | def dailymotion_download(
28 |     url, output_dir='.', merge=True, info_only=False, **kwargs
29 | ):
30 |     """Downloads Dailymotion videos by URL.
31 |     """
32 | 
33 |     html = get_content(rebuilt_url(url))
34 |     info = json.loads(match1(html, r'qualities":({.+?}),"'))
35 |     title = match1(html, r'"video_title"\s*:\s*"([^"]+)"') or \
36 |         match1(html, r'"title"\s*:\s*"([^"]+)"')
37 |     title = unicodize(title)
38 | 
39 |     for quality in ['1080', '720', '480', '380', '240', '144', 'auto']:
40 |         try:
41 |             real_url = info[quality][1]["url"]
42 |             if real_url:
43 |                 break
44 |         except KeyError:
45 |             pass
46 | 
47 |     mime, ext, size = url_info(real_url)
48 | 
49 |     print_info(site_info, title, mime, size)
50 |     if not info_only:
51 |         download_urls(
52 |             [real_url], title, ext, size, output_dir=output_dir, merge=merge,
53 |             **kwargs
54 |         )
55 | 
56 | 
57 | download = dailymotion_download
58 | download_playlist = playlist_not_supported(site_info)
59 | 


--------------------------------------------------------------------------------
/lulu/extractors/w56.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     print_info,
 8 |     get_content,
 9 |     download_urls,
10 |     playlist_not_supported,
11 | )
12 | from lulu.extractors.sohu import sohu_download
13 | 
14 | 
15 | __all__ = ['w56_download', 'w56_download_by_id']
16 | site_info = '56网 56.com'
17 | 
18 | 
19 | def w56_download_by_id(_id, title=None, info_only=False, **kwargs):
20 |     content = json.loads(get_content(
21 |         'http://vxml.56.com/json/{}/?src=site'.format(_id)
22 |     ))
23 |     info = content['info']
24 |     title = title or info['Subject']
25 |     assert title
26 |     hd = info['hd']
27 |     assert hd in (0, 1, 2)
28 |     hd_types = [['normal', 'qvga'], ['clear', 'vga'], ['super', 'wvga']][hd]
29 |     files = [x for x in info['rfiles'] if x['type'] in hd_types]
30 |     assert len(files) == 1
31 |     size = int(files[0]['filesize'])
32 |     url = files[0]['url'] + '&prod=56'
33 |     ext = 'mp4'
34 | 
35 |     print_info(site_info, title, ext, size)
36 |     if not info_only:
37 |         download_urls([url], title, ext, size, **kwargs)
38 | 
39 | 
40 | def w56_download(url, info_only=False, **kwargs):
41 |     content = get_content(url)
42 |     sohu_url = match1(content, r'url:\s*"(.+)"')
43 |     if sohu_url:
44 |         sohu_download(sohu_url, info_only=info_only, **kwargs)
45 |         return
46 |     _id = match1(url, r'http://www.56.com/u\d+/v_(\w+).html') or \
47 |         match1(url, r'http://www.56.com/.*vid-(\w+).html')
48 |     w56_download_by_id(_id, info_only=info_only, **kwargs)
49 | 
50 | 
51 | download = w56_download
52 | download_playlist = playlist_not_supported(site_info)
53 | 


--------------------------------------------------------------------------------
/lulu/extractors/douban.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import os
 5 | from pathlib import Path
 6 | 
 7 | from lulu.common import (
 8 |     match1,
 9 |     url_info,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 | )
14 | from lulu.util.parser import get_parser
15 | 
16 | 
17 | __all__ = ['douban_download']
18 | site_info = '豆瓣 douban.com'
19 | 
20 | 
21 | def douban_download(
22 |     url, output_dir='.', merge=True, info_only=False, **kwargs
23 | ):
24 |     html = get_content(url)
25 |     parser = get_parser(html)
26 | 
27 |     if re.match(r'https?://movie', url):
28 |         title = ' '.join(
29 |             [string.strip() for string in parser.h1.strings]
30 |         )
31 |         tid = match1(url, 'trailer/(\d+)')
32 |         real_url = 'https://movie.douban.com/trailer/video_url?tid={}'.format(
33 |             tid
34 |         )
35 |         _type, ext, size = url_info(real_url, refer=url)
36 | 
37 |         print_info(site_info, title, _type, size)
38 |         if not info_only:
39 |             download_urls(
40 |                 [real_url], title, ext, size, output_dir, merge=merge
41 |             )
42 | 
43 | 
44 | def douban_download_playlist(url, output_dir='.', **kwargs):
45 |     html = get_content(url)
46 |     parser = get_parser(html)
47 |     video_dir = Path(output_dir) / parser.h1.a.text
48 |     if not kwargs['info_only']:
49 |         if not video_dir.exists():
50 |             os.mkdir(video_dir)
51 |     urls = parser.find_all('a', class_='pr-video')
52 |     for url in urls:
53 |         douban_download(url['href'], output_dir=video_dir, **kwargs)
54 | 
55 | 
56 | download = douban_download
57 | download_playlist = douban_download_playlist
58 | 


--------------------------------------------------------------------------------
/lulu/extractors/suntv.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import urllib
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['suntv_download']
17 | site_info = 'ISUNVIDEO isuntv.com'
18 | 
19 | 
20 | def suntv_download(url, info_only=False, **kwargs):
21 |     if re.match(r'http://www.isuntv.com/\w+', url):
22 |         API_URL = (
23 |             'http://www.isuntv.com/ajaxpro/SunTv.pro_vod_playcatemp4,'
24 |             'App_Web_playcatemp4.ascx.9f08f04f.ashx'
25 |         )
26 |         itemid = match1(url, r'http://www.isuntv.com/pro/ct(\d+).html')
27 |         values = {'itemid': itemid, 'vodid': ''}
28 | 
29 |         data = str(values).replace("'", '"')
30 |         data = data.encode('utf-8')
31 |         req = urllib.request.Request(API_URL, data)
32 |         req.add_header('AjaxPro-Method', 'ToPlay')  # important!
33 |         resp = urllib.request.urlopen(req)
34 |         respData = resp.read()
35 |         respData = respData.decode('ascii').strip('"')  # Ahhhhhhh!
36 | 
37 |         video_url = 'http://www.isuntv.com' + str(respData)
38 | 
39 |         html = get_content(url, decoded=False)
40 |         html = html.decode('gbk')
41 |         title = match1(html, '<title>([^<]+)').strip()  # get rid of \r\n s
42 | 
43 |         size = 0
44 |         _type, ext, size = url_info(video_url)
45 | 
46 |         print_info(site_info, title, _type, size)
47 |         if not info_only:
48 |             download_urls([url], title, 'mp4', size, **kwargs)
49 | 
50 | 
51 | download = suntv_download
52 | download_playlist = playlist_not_supported(site_info)
53 | 


--------------------------------------------------------------------------------
/lulu/extractors/baomihua.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from lulu.common import (
 4 |     match1,
 5 |     url_info,
 6 |     print_info,
 7 |     get_content,
 8 |     download_urls,
 9 |     playlist_not_supported,
10 | )
11 | 
12 | 
13 | __all__ = ['baomihua_download', 'baomihua_download_by_id']
14 | site_info = '爆米花 baomihua.com'
15 | 
16 | 
17 | def baomihua_download_by_id(
18 |     _id, title=None, output_dir='.', merge=True, info_only=False, **kwargs
19 | ):
20 |     html = get_content(
21 |         'http://play.baomihua.com/getvideourl.aspx?flvid={}&devicetype='
22 |         'phone_app'.format(_id)
23 |     )
24 |     host = match1(html, r'host=([^&]*)')
25 |     assert host
26 |     _type = match1(html, r'videofiletype=([^&]*)')
27 |     assert _type
28 |     vid = match1(html, r'&stream_name=([^&]*)')
29 |     assert vid
30 |     dir_str = match1(html, r'&dir=([^&]*)').strip()
31 |     url = 'http://{}/{}/{}.{}'.format(host, dir_str, vid, _type)
32 |     _, ext, size = url_info(url)
33 |     print_info(site_info, title, _type, size)
34 |     if not info_only:
35 |         download_urls(
36 |             [url], title, ext, size, output_dir, merge=merge, **kwargs
37 |         )
38 | 
39 | 
40 | def baomihua_download(
41 |     url, output_dir='.', merge=True, info_only=False, **kwargs
42 | ):
43 |     html = get_content(url)
44 |     title = match1(html, r'<title>(.*)</title>')
45 |     assert title
46 |     _id = match1(html, r'flvid\s*=\s*(\d+)')
47 |     assert _id
48 |     baomihua_download_by_id(
49 |         _id, title, output_dir=output_dir, merge=merge, info_only=info_only,
50 |         **kwargs
51 |     )
52 | 
53 | 
54 | download = baomihua_download
55 | download_playlist = playlist_not_supported(site_info)
56 | 


--------------------------------------------------------------------------------
/lulu/extractors/yizhibo.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     player,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     launch_player,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['yizhibo_download']
17 | site_info = 'yizhibo.com'
18 | 
19 | 
20 | def yizhibo_download(
21 |     url, output_dir='.', merge=True, info_only=False, **kwargs
22 | ):
23 |     video_id = url[url.rfind('/')+1:].split('.')[0]
24 |     json_request_url = (
25 |         'http://www.yizhibo.com/live/h5api/get_basic_live_info?'
26 |         'scid={}'.format(video_id)
27 |     )
28 |     content = get_content(json_request_url)
29 |     error = json.loads(content)['result']
30 |     if error != 1:
31 |         raise ValueError('Error : {}'.format(error))
32 | 
33 |     data = json.loads(content)
34 |     title = data.get('data')['live_title']
35 |     if (title == ''):
36 |         title = data.get('data')['nickname']
37 |     m3u8_url = data.get('data')['play_url']
38 |     m3u8 = get_content(m3u8_url)
39 |     base_url = '{}/'.format(
40 |         '/'.join(data.get('data')['play_url'].split('/')[:7])
41 |     )
42 |     part_url = re.findall(r'([0-9]+\.ts)', m3u8)
43 |     real_url = []
44 |     for i in part_url:
45 |         url = base_url + i
46 |         real_url.append(url)
47 |     print_info(site_info, title, 'ts', float('inf'))
48 |     if not info_only:
49 |         if player:
50 |             launch_player(player, [m3u8_url])
51 |         download_urls(
52 |             real_url, title, 'ts', float('inf'), output_dir, merge=merge
53 |         )
54 | 
55 | 
56 | download = yizhibo_download
57 | download_playlist = playlist_not_supported(site_info)
58 | 


--------------------------------------------------------------------------------
/lulu/processor/join_ts.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | 
 4 | ##################################################
 5 | # main
 6 | ##################################################
 7 | 
 8 | 
 9 | def guess_output(inputs):
10 |     import os.path
11 |     inputs = map(os.path.basename, inputs)
12 |     n = min(map(len, inputs))
13 |     for i in reversed(range(1, n)):
14 |         if len(set(s[:i] for s in inputs)) == 1:
15 |             return inputs[0][:i] + '.ts'
16 |     return 'output.ts'
17 | 
18 | 
19 | def concat_ts(ts_parts, output=None):
20 |     assert ts_parts, 'no ts files found'
21 |     import os.path
22 |     if not output:
23 |         output = guess_output(ts_parts)
24 |     elif os.path.isdir(output):
25 |         output = os.path.join(output, guess_output(ts_parts))
26 | 
27 |     print('Merging video parts...')
28 | 
29 |     ts_out_file = open(output, "wb")
30 |     for ts_in in ts_parts:
31 |         ts_in_file = open(ts_in, "rb")
32 |         ts_in_data = ts_in_file.read()
33 |         ts_in_file.close()
34 |         ts_out_file.write(ts_in_data)
35 |     ts_out_file.close()
36 |     return output
37 | 
38 | 
39 | def usage():
40 |     print('Usage: [python3] join_ts.py --output TARGET.ts ts...')
41 | 
42 | 
43 | def main():
44 |     import sys
45 |     import getopt
46 |     try:
47 |         opts, args = getopt.getopt(sys.argv[1:], 'ho:', ['help', 'output='])
48 |     except getopt.GetoptError as err:
49 |         usage()
50 |         sys.exit(1)
51 |     output = None
52 |     for o, a in opts:
53 |         if o in ('-h', '--help'):
54 |             usage()
55 |             sys.exit()
56 |         elif o in ('-o', '--output'):
57 |             output = a
58 |         else:
59 |             usage()
60 |             sys.exit(1)
61 |     if not args:
62 |         usage()
63 |         sys.exit(1)
64 | 
65 |     concat_ts(args, output)
66 | 
67 | 
68 | if __name__ == '__main__':
69 |     main()
70 | 


--------------------------------------------------------------------------------
/lulu/json_output.py:
--------------------------------------------------------------------------------
 1 | 
 2 | import json
 3 | 
 4 | # save info from common.print_info()
 5 | last_info = None
 6 | 
 7 | 
 8 | def output(video_extractor, pretty_print=True):
 9 |     ve = video_extractor
10 |     out = {}
11 |     out['url'] = ve.url
12 |     out['title'] = ve.title
13 |     out['site'] = ve.name
14 |     out['streams'] = ve.streams
15 |     try:
16 |         if ve.audiolang:
17 |             out['audiolang'] = ve.audiolang
18 |     except AttributeError:
19 |         pass
20 |     extra = {}
21 |     if getattr(ve, 'referer', None) is not None:
22 |         extra["referer"] = ve.referer
23 |     if getattr(ve, 'ua', None) is not None:
24 |         extra["ua"] = ve.ua
25 |     if extra:
26 |         out["extra"] = extra
27 |     if pretty_print:
28 |         print(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))
29 |     else:
30 |         print(json.dumps(out))
31 | 
32 | 
33 | # a fake VideoExtractor object to save info
34 | class VideoExtractor(object):
35 |     pass
36 | 
37 | 
38 | def print_info(site_info=None, title=None, type=None, size=None):
39 |     global last_info
40 |     # create a VideoExtractor and save info for download_urls()
41 |     ve = VideoExtractor()
42 |     last_info = ve
43 |     ve.name = site_info
44 |     ve.title = title
45 |     ve.url = None
46 | 
47 | 
48 | def download_urls(
49 |     urls=None, title=None, ext=None, total_size=None, refer=None
50 | ):
51 |     ve = last_info
52 |     if not ve:
53 |         ve = VideoExtractor()
54 |         ve.name = ''
55 |         ve.url = urls
56 |         ve.title = title
57 |     # save download info in streams
58 |     stream = {}
59 |     stream['container'] = ext
60 |     stream['size'] = total_size
61 |     stream['src'] = urls
62 |     if refer:
63 |         stream['refer'] = refer
64 |     stream['video_profile'] = '__default__'
65 |     ve.streams = {}
66 |     ve.streams['__default__'] = stream
67 |     output(ve)
68 | 


--------------------------------------------------------------------------------
/lulu/extractors/qq_egame.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import json
 3 | from html import unescape
 4 | 
 5 | from lulu.util import log
 6 | from lulu.common import get_content
 7 | from lulu.extractor import VideoExtractor
 8 | 
 9 | 
10 | __all__ = ['qq_egame_download']
11 | 
12 | 
13 | class QQEgame(VideoExtractor):
14 |     stream_types = [
15 |         {'id': '1080p', 'video_profile': 0, 'container': 'flv'},
16 |         {'id': '720p', 'video_profile': 3000, 'container': 'flv'},
17 |         {'id': '540p', 'video_profile': 1500, 'container': 'flv'}
18 |     ]
19 |     name = '企鹅电竞 egame.qq.com'
20 | 
21 |     def prepare(self, **kwargs):
22 |         page = get_content(self.url)
23 |         server_data = re.search(r'window\.__NUXT__=({.+?});', page)
24 |         if server_data is None:
25 |             log.wtf('cannot find server_data')
26 |         json_data = json.loads(server_data.group(1))['state']
27 |         live_info = json_data['live-info']['liveInfo']
28 |         self.title = '{}_{}'.format(
29 |             json_data['anchor-info']['anchorInfo']['nickName'],
30 |             live_info['videoInfo']['title']
31 |         )
32 |         for exsited_stream in live_info['videoInfo']['streamInfos']:
33 |             for s in self.__class__.stream_types:
34 |                 if s['video_profile'] == exsited_stream['bitrate']:
35 |                     current_stream_id = s['id']
36 |                     stream_info = dict(
37 |                         src=[unescape(exsited_stream['playUrl'])]
38 |                     )
39 |                     stream_info['video_profile'] = exsited_stream['desc']
40 |                     stream_info['container'] = s['container']
41 |                     stream_info['size'] = float('inf')
42 |                     self.streams[current_stream_id] = stream_info
43 | 
44 | 
45 | def qq_egame_download(url, **kwargs):
46 |     QQEgame().download_by_url(url, **kwargs)
47 |     # url dispatching has been done in qq.py
48 | 


--------------------------------------------------------------------------------
/lulu/extractors/qingting.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     url_size,
 8 |     get_content,
 9 |     playlist_not_supported,
10 | )
11 | from lulu.util import log
12 | from lulu.extractors import VideoExtractor
13 | 
14 | 
15 | __all__ = ['qingting_download_by_url']
16 | site_info = '蜻蜓FM qingting.fm'
17 | 
18 | 
19 | class Qingting(VideoExtractor):
20 |     # every resource is described by its channel id and program id
21 |     # so vid is tuple (chaanel_id, program_id)
22 | 
23 |     name = site_info
24 |     stream_types = [
25 |         {'id': '_default'}
26 |     ]
27 | 
28 |     ep = 'http://i.qingting.fm/wapi/channels/{}/programs/{}'
29 |     file_host = 'http://od.qingting.fm/{}'
30 |     mobile_pt = r'channels\/(\d+)\/programs/(\d+)'
31 | 
32 |     def prepare(self, **kwargs):
33 |         if self.vid is None:
34 |             hit = re.search(self.__class__.mobile_pt, self.url)
35 |             self.vid = (hit.group(1), hit.group(2))
36 | 
37 |         ep_url = self.__class__.ep.format(self.vid[0], self.vid[1])
38 |         meta = json.loads(get_content(ep_url))
39 | 
40 |         if meta['code'] != 0:
41 |             log.wtf(meta['message']['errormsg'])
42 | 
43 |         file_path = self.__class__.file_host.format(meta['data']['file_path'])
44 |         self.title = meta['data']['name']
45 |         duration = str(meta['data']['duration']) + 's'
46 | 
47 |         self.streams['_default'] = {
48 |             'src': [file_path], 'video_profile': duration, 'container': 'm4a'
49 |         }
50 | 
51 |     def extract(self, **kwargs):
52 |         self.streams['_default']['size'] = url_size(
53 |             self.streams['_default']['src'][0]
54 |         )
55 | 
56 | 
57 | def qingting_download_by_url(url, **kwargs):
58 |     Qingting().download_by_url(url, **kwargs)
59 | 
60 | 
61 | download = qingting_download_by_url
62 | download_playlist = playlist_not_supported(site_info)
63 | 


--------------------------------------------------------------------------------
/lulu/extractors/soundcloud.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     url_info,
 7 |     print_info,
 8 |     get_content,
 9 |     download_urls,
10 |     playlist_not_supported,
11 | )
12 | 
13 | 
14 | __all__ = ['soundcloud_download', 'soundcloud_download_by_id']
15 | site_info = 'SoundCloud soundCloud.com'
16 | 
17 | client_id = 'WKcQQdEZw7Oi01KqtHWxeVSxNyRzgT8M'
18 | 
19 | 
20 | def soundcloud_download_by_id(_id, title=None, info_only=False, **kwargs):
21 |     assert title
22 |     url = 'https://api.soundcloud.com/tracks/{}/{}?client_id={}'.format(
23 |         _id, 'stream', client_id
24 |     )
25 | 
26 |     _type, ext, size = url_info(url)
27 |     print_info(site_info, title, _type, size)
28 |     if not info_only:
29 |         download_urls([url], title, ext, size, **kwargs)
30 | 
31 | 
32 | def soundcloud_i1_api(track_id):
33 |     url = (
34 |         'https://api.soundcloud.com/i1/tracks/{}/streams?client_id={}'.format(
35 |             track_id, client_id
36 |         )
37 |     )
38 |     return json.loads(get_content(url))['http_mp3_128_url']
39 | 
40 | 
41 | def soundcloud_download(url, info_only=False, **kwargs):
42 |     url = 'https://api.soundcloud.com/resolve.json?url={}&client_id={}'.format(
43 |         url, client_id
44 |     )
45 |     metadata = get_content(url)
46 |     info = json.loads(metadata)
47 |     title = info['title']
48 |     real_url = info.get('download_url')
49 |     if real_url is None:
50 |         real_url = info.get('steram_url')
51 |     if real_url is None:
52 |         raise Exception('Cannot get media URI for {}'.format(url))
53 |     real_url = soundcloud_i1_api(info['id'])
54 |     mime, ext, size = url_info(real_url)
55 |     print_info(site_info, title, mime, size)
56 |     if not info_only:
57 |         download_urls([real_url], title, ext, size, **kwargs)
58 | 
59 | 
60 | download = soundcloud_download
61 | download_playlist = playlist_not_supported(site_info)
62 | 


--------------------------------------------------------------------------------
/lulu/extractors/panda.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | import time
 6 | 
 7 | from lulu.util import log
 8 | from lulu.common import (
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['panda_download']
17 | site_info = '熊猫直播 panda.tv'
18 | 
19 | 
20 | def panda_download(url, info_only=False, **kwargs):
21 |     roomid = re.search('/(\d+)', url)
22 |     if roomid is None:
23 |         log.wtf('Cannot found room id for this url')
24 |     roomid = roomid.group(1)
25 |     json_request_url = (
26 |         'http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}'.format(
27 |             roomid, int(time.time())
28 |         )
29 |     )
30 |     content = get_content(json_request_url)
31 |     api_json = json.loads(content)
32 | 
33 |     errno = api_json['errno']
34 |     errmsg = api_json['errmsg']
35 |     if errno:
36 |         raise ValueError('Errno : {}, Errmsg : {}'.format(errno, errmsg))
37 |     data = api_json['data']
38 |     title = data['roominfo']['name']
39 |     room_key = data['videoinfo']['room_key']
40 |     plflag = data['videoinfo']['plflag'].split('_')
41 |     status = data['videoinfo']['status']
42 |     if status is not '2':
43 |         raise ValueError('The live stream is not online! (status:{})'.format(
44 |             status)
45 |         )
46 | 
47 |     data2 = json.loads(data['videoinfo']['plflag_list'])
48 |     rid = data2['auth']['rid']
49 |     sign = data2['auth']['sign']
50 |     ts = data2['auth']['time']
51 |     real_url = (
52 |         'http://pl{}.live.panda.tv/live_panda/{}.flv?sign={}&ts={}&'
53 |         'rid={}'.format(plflag[1], room_key, sign, ts, rid)
54 |     )
55 |     print_info(site_info, title, 'flv', float('inf'))
56 |     if not info_only:
57 |         download_urls([real_url], title, 'flv', None, **kwargs)
58 | 
59 | 
60 | download = panda_download
61 | download_playlist = playlist_not_supported(site_info)
62 | 


--------------------------------------------------------------------------------
/lulu/extractors/mtv81.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from html import unescape
 4 | from xml.dom.minidom import parseString
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     print_info,
 9 |     get_content,
10 |     download_rtmp_url,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['mtv81_download']
16 | site_info = 'MTV 81 mtv81.com'
17 | 
18 | 
19 | def mtv81_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
20 |     html = get_content(url)
21 |     title = unescape(
22 |         '|'.join(match1(html, r'<title>(.*?)</title>').split('|')[:-2])
23 |     )
24 | 
25 |     # mgid%3Auma%3Avideo%3Amtv81.com%3A897974
26 |     vid = match1(html, r'getTheVideo\("(.*?)"')
27 |     xml = parseString(get_content(
28 |         'http://intl.esperanto.mtvi.com/www/xml/media/mediaGen.jhtml?uri={}&'
29 |         'flashPlayer=LNX%2013,0,0,206&geo=CN&sid=123456'.format(vid)
30 |     ))
31 | 
32 |     url = sorted(
33 |         map(lambda x: x.firstChild.nodeValue, xml.getElementsByTagName("src")),
34 |         key=lambda x: int(match1(x, r'_(\d+?)_'))
35 |     )[-1]
36 | 
37 |     mediatype, ext, size = 'mp4', 'mp4', 0
38 |     print_info(site_info, title, mediatype, size)
39 |     # rtmpdump  -r 'rtmpe://cp30865.edgefcs.net/ondemand/mtviestor/_!/intlod/MTVInternational/MBUS/GeoLocals/00JP/VIAMTVI/PYC/201304/7122HVAQ4/00JPVIAMTVIPYC7122HVAQ4_640x_360_1200_m30.mp4' -o "title.mp4" --swfVfy http://media.mtvnservices.com/player/prime/mediaplayerprime.1.10.8.swf  # noqa
40 | 
41 |     # because rtmpdump is unstable,may try serveral times
42 |     if not info_only:
43 |         download_rtmp_url(
44 |             url=url, title=title, ext=ext, params={
45 |                 '--swfVfy': (
46 |                     'http://media.mtvnservices.com/player/prime/mediaplayer'
47 |                     'prime.1.10.8.swf'
48 |                 )
49 |             }, output_dir=output_dir
50 |         )
51 | 
52 | 
53 | download = mtv81_download
54 | download_playlist = playlist_not_supported(site_info)
55 | 


--------------------------------------------------------------------------------
/lulu/extractors/videomega.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import ssl
 5 | from urllib import request
 6 | 
 7 | from lulu.common import (
 8 |     match1,
 9 |     url_info,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | 
16 | 
17 | __all__ = ['videomega_download']
18 | site_info = 'Videomega.tv'
19 | 
20 | 
21 | def videomega_download(url, info_only=False, **kwargs):
22 |     # Hot-plug cookie handler
23 |     ssl_context = request.HTTPSHandler(
24 |         context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)
25 |     )
26 |     cookie_handler = request.HTTPCookieProcessor()
27 |     opener = request.build_opener(ssl_context, cookie_handler)
28 |     opener.addheaders = [('Referer', url), ('Cookie', 'noadvtday=0')]
29 |     request.install_opener(opener)
30 | 
31 |     if re.search(r'view\.php', url):
32 |         php_url = url
33 |     else:
34 |         content = get_content(url)
35 |         m = re.search(
36 |             r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content
37 |         )
38 |         ref = m.group(1)
39 |         width, height = m.group(2), m.group(3)
40 |         php_url = (
41 |             'http://videomega.tv/view.php?ref={}&width={}&height={}'.format(
42 |                 ref, width, height
43 |             )
44 |         )
45 |     content = get_content(php_url)
46 | 
47 |     title = match1(content, r'<title>(.*)</title>')
48 |     js = match1(content, r'(eval.*)')
49 |     t = match1(js, r'\$\("\w+"\)\.\w+\("\w+","([^"]+)"\)')
50 |     t = re.sub(r'(\w)', r'{\1}', t)
51 |     t = t.translate({87 + i: str(i) for i in range(10, 36)})
52 |     s = match1(js, r"'([^']+)'\.split").split('|')
53 |     src = t.format(*s)
54 | 
55 |     _type, ext, size = url_info(src)
56 | 
57 |     print_info(site_info, title, _type, size)
58 |     if not info_only:
59 |         download_urls([src], title, ext, size, **kwargs)
60 | 
61 | 
62 | download = videomega_download
63 | download_playlist = playlist_not_supported(site_info)
64 | 


--------------------------------------------------------------------------------
/lulu/extractors/kugou.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | from base64 import b64decode
 6 | 
 7 | from lulu.common import (
 8 |     match1,
 9 |     url_info,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 | )
14 | 
15 | 
16 | __all__ = ['kugou_download']
17 | site_info = '酷狗音乐 kugou.com'
18 | 
19 | 
20 | def kugou_download_by_hash(hash_val, info_only=False, **kwargs):
21 |     # http://www.kugou.com/yy/album/single/536957.html
22 |     html = get_content(
23 |         'http://www.kugou.com/yy/index.php?r=play/getdata&hash={}'.format(
24 |             hash_val
25 |         )
26 |     )
27 |     data = json.loads(html)
28 |     url = data['data']['play_url']
29 |     title = data['data']['audio_name']
30 |     songtype, ext, size = url_info(url)
31 |     print_info(site_info, title, songtype, size)
32 |     if not info_only:
33 |         download_urls([url], title, ext, size, **kwargs)
34 | 
35 | 
36 | def kugou_download_playlist(url, info_only=False, **kwargs):
37 |     html = get_content(url)
38 |     pattern = re.compile('title="(.*?)".* data="(\w*)\|.*?"')
39 |     pairs = pattern.findall(html)
40 |     for title, hash_val in pairs:
41 |         kugou_download_by_hash(hash_val, info_only, **kwargs)
42 | 
43 | 
44 | def kugou_download(url, info_only=False, **kwargs):
45 |     html = get_content(url)
46 |     if url.lower().find('5sing') != -1:
47 |         # for 5sing.kugou.com
48 |         ticket = match1(html, r'"ticket":\s*"(.*)"')
49 |         j = json.loads(str(b64decode(ticket), encoding='utf-8'))
50 |         url = j['file']
51 |         title = j['songName']
52 |         songtype, ext, size = url_info(url)
53 |         print_info(site_info, title, songtype, size)
54 |         if not info_only:
55 |             download_urls([url], title, ext, size, **kwargs)
56 |     else:
57 |         # for the www.kugou.com
58 |         hash_val = match1(url, r'hash=(\w+)')
59 |         kugou_download_by_hash(hash_val, info_only, **kwargs)
60 | 
61 | 
62 | download = kugou_download
63 | download_playlist = kugou_download_playlist
64 | 


--------------------------------------------------------------------------------
/lulu/processor/rtmpdump.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import os.path
 4 | import subprocess
 5 | 
 6 | 
 7 | def get_usable_rtmpdump(cmd):
 8 |     try:
 9 |         p = subprocess.Popen(
10 |             [cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE
11 |         )
12 |         out, err = p.communicate()
13 |         return cmd
14 |     except Exception:
15 |         return None
16 | 
17 | 
18 | RTMPDUMP = get_usable_rtmpdump('rtmpdump')
19 | 
20 | 
21 | def has_rtmpdump_installed():
22 |     return RTMPDUMP is not None
23 | 
24 | 
25 | # params ={"-y":"playlist","-q":None,}
26 | # if Only Key ,Value should be None
27 | # -r -o should not be included in params
28 | 
29 | 
30 | def download_rtmpdump_stream(url, title, ext, params={}, output_dir='.'):
31 |     filename = '%s.%s' % (title, ext)
32 |     filepath = os.path.join(output_dir, filename)
33 | 
34 |     cmdline = [RTMPDUMP, '-r']
35 |     cmdline.append(url)
36 |     cmdline.append('-o')
37 |     cmdline.append(filepath)
38 | 
39 |     for key in params.keys():
40 |         cmdline.append(key)
41 |         if params[key] is not None:
42 |             cmdline.append(params[key])
43 | 
44 |     # cmdline.append('-y')
45 |     # cmdline.append(playpath)
46 |     print("Call rtmpdump:\n"+" ".join(cmdline)+"\n")
47 |     subprocess.call(cmdline)
48 |     return
49 | 
50 | 
51 | def play_rtmpdump_stream(player, url, params={}):
52 |     # construct left side of pipe
53 |     cmdline = [RTMPDUMP, '-r']
54 |     cmdline.append(url)
55 | 
56 |     # append other params if exist
57 |     for key in params.keys():
58 |         cmdline.append(key)
59 |         if params[key] is not None:
60 |             cmdline.append(params[key])
61 | 
62 |     cmdline.append('-o')
63 |     cmdline.append('-')
64 | 
65 |     # pipe start
66 |     cmdline.append('|')
67 |     cmdline.append(player)
68 |     cmdline.append('-')
69 | 
70 |     # logging
71 |     print('Call rtmpdump:\n{}\n'.format(' '.join(cmdline)))
72 | 
73 |     # call RTMPDump!
74 |     subprocess.call(cmdline)
75 | 
76 |     # os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player))  # noqa
77 |     return
78 | 


--------------------------------------------------------------------------------
/lulu/extractors/ifeng.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from html import unescape
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     playlist_not_supported,
12 | )
13 | 
14 | 
15 | __all__ = ['ifeng_download', 'ifeng_download_by_id']
16 | site_info = '凤凰网 ifeng.com'
17 | 
18 | 
19 | def ifeng_download_by_id(_id, title=None, info_only=False, **kwargs):
20 |     assert match1(
21 |         _id, r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
22 |     ), _id
23 |     url = 'http://vxml.ifengimg.com/video_info_new/{}/{}/{}.xml'.format(
24 |         _id[-2], _id[-2:], _id
25 |     )
26 |     xml = get_content(url)
27 |     title = match1(xml, r'Name="([^"]+)"')
28 |     title = unescape(title)
29 |     url = match1(xml, r'VideoPlayUrl="([^"]+)"')
30 |     url = url.replace(
31 |         'http://wideo.ifeng.com/', 'http://ips.ifeng.com/wideo.ifeng.com/'
32 |     )
33 |     _, ext, size = url_info(url)
34 | 
35 |     print_info(site_info, title, ext, size)
36 |     if not info_only:
37 |         download_urls([url], title, ext, size, **kwargs)
38 | 
39 | 
40 | def ifeng_download(url, info_only=False, **kwargs):
41 |     # old pattern /uuid.shtml
42 |     # now it could be #uuid
43 |     _id = match1(
44 |         url,
45 |         r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
46 |     )
47 |     if _id:
48 |         return ifeng_download_by_id(_id, None, info_only=info_only, **kwargs)
49 | 
50 |     html = get_content(url)
51 |     uuid_pattern = (
52 |         r'"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"'
53 |     )
54 |     _id = match1(
55 |         html,
56 |         r'var vid="([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-'
57 |         '[0-9a-f]{12})"'
58 |     )
59 |     if _id is None:
60 |         video_pattern = r'"vid"\s*:\s*' + uuid_pattern
61 |         _id = match1(html, video_pattern)
62 |     assert _id, "Can't find video info"
63 |     return ifeng_download_by_id(_id, None, info_only=info_only, **kwargs)
64 | 
65 | 
66 | download = ifeng_download
67 | download_playlist = playlist_not_supported(site_info)
68 | 


--------------------------------------------------------------------------------
/lulu/extractors/fantasy.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | import random
 5 | from urllib.parse import (
 6 |     urlparse,
 7 |     parse_qs,
 8 | )
 9 | 
10 | from lulu.config import FAKE_HEADERS
11 | from lulu.common import (
12 |     url_info,
13 |     print_info,
14 |     get_content,
15 |     download_urls,
16 |     playlist_not_supported,
17 | )
18 | 
19 | 
20 | __all__ = ['fantasy_download']
21 | site_info = 'fantasy.tv'
22 | 
23 | 
24 | def fantasy_download_by_id_channelId(
25 |     id=0, channelId=0, output_dir='.', merge=True, info_only=False, **kwargs
26 | ):
27 |     api_url = (
28 |         'http://www.fantasy.tv/tv/playDetails.action?'
29 |         'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format(
30 |             id=id,
31 |             channelId=channelId,
32 |             t=str(random.random())
33 |         )
34 |     )
35 |     html = get_content(api_url)
36 |     html = json.loads(html)
37 | 
38 |     if int(html['status']) != 100000:
39 |         raise Exception('API error!')
40 | 
41 |     title = html['data']['tv']['title']
42 | 
43 |     video_url = html['data']['tv']['videoPath']
44 |     headers = FAKE_HEADERS.copy()
45 |     headers['Referer'] = api_url
46 |     type, ext, size = url_info(video_url, headers=headers)
47 | 
48 |     print_info(site_info, title, type, size)
49 |     if not info_only:
50 |         download_urls(
51 |             [video_url], title, ext, size, output_dir, merge=merge,
52 |             headers=headers
53 |         )
54 | 
55 | 
56 | def fantasy_download(
57 |     url, output_dir='.', merge=True, info_only=False, **kwargs
58 | ):
59 |     if 'fantasy.tv' not in url:
60 |         raise Exception('Wrong place!')
61 | 
62 |     q = parse_qs(urlparse(url).query)
63 | 
64 |     if 'tvId' not in q or 'channelId' not in q:
65 |         raise Exception('No enough arguments!')
66 | 
67 |     tvId = q['tvId'][0]
68 |     channelId = q['channelId'][0]
69 | 
70 |     fantasy_download_by_id_channelId(
71 |         id=tvId, channelId=channelId, output_dir=output_dir, merge=merge,
72 |         info_only=info_only, **kwargs
73 |     )
74 | 
75 | 
76 | download = fantasy_download
77 | download_playlist = playlist_not_supported(site_info)
78 | 


--------------------------------------------------------------------------------
/lulu/extractors/vk.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     post_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | 
16 | 
17 | __all__ = ['vk_download']
18 | site_info = 'VK vk.com'
19 | 
20 | 
21 | def get_video_info(url):
22 |     ep = 'https://vk.com/al_video.php'
23 |     to_post = dict(
24 |         act='show', al=1, module='direct',
25 |         video=re.search(r'video(-\d+_\d+)', url).group(1)
26 |     )
27 |     page = post_content(ep, post_data=to_post)
28 |     url = match1(page, r'"url720":"(.+?)",')
29 |     url = url.replace('\\', '')
30 |     title = match1(page, r'<div class="mv_title".+?>(.+?)</div>')
31 |     mime, ext, size = url_info(url)
32 |     print_info(site_info, title, mime, size)
33 | 
34 |     return url, title, ext, size
35 | 
36 | 
37 | def get_image_info(url):
38 |     image_page = get_content(url)
39 |     photo_id = url.split('-')[-1]
40 |     photo_data = json.loads(
41 |         match1(image_page, r'({{"id":"-{}.+?}}),'.format(photo_id))
42 |     )
43 |     for quality in ['z_src', 'y_src', 'x_src', 'r_src', 'q_src', 'p_src']:
44 |         if photo_data.get(quality):
45 |             image_link = photo_data[quality]
46 |             break
47 |     title = image_link.split('/')[-1].split('.')[0]
48 |     _type, ext, size = url_info(image_link)
49 |     print_info(site_info, title, _type, size)
50 | 
51 |     return image_link, title, ext, size
52 | 
53 | 
54 | def vk_download(
55 |     url, output_dir='.', stream_type=None, merge=True, info_only=False,
56 |     **kwargs
57 | ):
58 |     link = None
59 |     if re.match(r'(.+)vk\.com\/video(.+)', url):
60 |         link, title, ext, size = get_video_info(url)
61 |     elif re.match(r'(.+)vk\.com\/photo(.+)', url):
62 |         link, title, ext, size = get_image_info(url)
63 |     else:
64 |         raise NotImplementedError('Nothing to download here')
65 | 
66 |     if not info_only and link is not None:
67 |         download_urls([link], title, ext, size, output_dir, merge=merge)
68 | 
69 | 
70 | download = vk_download
71 | download_playlist = playlist_not_supported(site_info)
72 | 


--------------------------------------------------------------------------------
/lulu/extractors/nicovideo.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from urllib import (
 4 |     parse,
 5 |     request,
 6 | )
 7 | 
 8 | from lulu.common import (
 9 |     match1,
10 |     url_info,
11 |     print_info,
12 |     get_content,
13 |     download_urls,
14 |     playlist_not_supported,
15 | )
16 | from lulu.config import FAKE_HEADERS
17 | 
18 | 
19 | __all__ = ['nicovideo_download']
20 | site_info = 'niconico nicovideo.jp'
21 | fake_headers = FAKE_HEADERS.copy()
22 | 
23 | 
24 | def nicovideo_login(user, password):
25 |     data = 'current_form=login&mail={}&password={}&login_submit=Log+In'.format(
26 |         user, password
27 |     )
28 |     response = request.urlopen(request.Request(
29 |         'https://secure.nicovideo.jp/secure/login?site=niconico',
30 |         headers=fake_headers, data=data.encode('utf-8')
31 |     ))
32 |     return response.headers
33 | 
34 | 
35 | def nicovideo_download(url, info_only=False, **kwargs):
36 |     import ssl
37 |     ssl_context = request.HTTPSHandler(
38 |         context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)
39 |     )
40 |     cookie_handler = request.HTTPCookieProcessor()
41 |     opener = request.build_opener(ssl_context, cookie_handler)
42 |     request.install_opener(opener)
43 | 
44 |     import netrc
45 |     import getpass
46 |     try:
47 |         info = netrc.netrc().authenticators('nicovideo')
48 |     except Exception:
49 |         info = None
50 |     if info is None:
51 |         user = input('User:     ')
52 |         password = getpass.getpass('Password: ')
53 |     else:
54 |         user, password = info[0], info[2]
55 |     print('Logging in...')
56 |     nicovideo_login(user, password)
57 | 
58 |     html = get_content(url)  # necessary!
59 |     title = match1(html, r'<title>(.+?)</title>')
60 | 
61 |     vid = url.split('/')[-1].split('?')[0]
62 |     api_html = get_content('http://flapi.nicovideo.jp/api/getflv?v={}'.format(
63 |         vid
64 |     ))
65 |     real_url = parse.unquote(match1(api_html, r'url=([^&]+)&'))
66 | 
67 |     _type, ext, size = url_info(real_url)
68 |     print_info(site_info, title, _type, size)
69 |     if not info_only:
70 |         download_urls([real_url], title, ext, size, **kwargs)
71 | 
72 | 
73 | download = nicovideo_download
74 | download_playlist = playlist_not_supported(site_info)
75 | 


--------------------------------------------------------------------------------
/lulu/extractors/nanagogo.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | from lulu.extractors.universal import universal_download
15 | 
16 | 
17 | __all__ = ['nanagogo_download']
18 | site_info = '755 7gogo.jp'
19 | 
20 | 
21 | def nanagogo_download(url, info_only=False, **kwargs):
22 |     if re.match(r'https?://stat.7gogo.jp', url):
23 |         universal_download(url, info_only=info_only, **kwargs)
24 |         return
25 | 
26 |     talk_id = match1(url, r'7gogo.jp/([^/]+)/')
27 |     post_id = match1(url, r'7gogo.jp/[^/]+/(\d+)')
28 |     title = '{}_{}'.format(talk_id, post_id)
29 |     api_url = 'https://api.7gogo.jp/web/v2/talks/{}/posts/{}'.format(
30 |         talk_id, post_id
31 |     )
32 |     info = json.loads(get_content(api_url))
33 | 
34 |     items = []
35 |     if info['data']['posts']['post'] is None:
36 |         return
37 |     if info['data']['posts']['post']['body'] is None:
38 |         return
39 |     for i in info['data']['posts']['post']['body']:
40 |         if 'image' in i:
41 |             image_url = i['image']
42 |             if image_url[:2] == '//':
43 |                 continue  # skip stamp images
44 |             _, ext, size = url_info(image_url)
45 |             items.append({
46 |                 'title': title, 'url': image_url, 'ext': ext, 'size': size
47 |             })
48 |         elif 'movieUrlHq' in i:
49 |             movie_url = i['movieUrlHq']
50 |             _, ext, size = url_info(movie_url)
51 |             items.append({
52 |                 'title': title, 'url': movie_url, 'ext': ext, 'size': size
53 |             })
54 | 
55 |     size = sum([i['size'] for i in items])
56 |     if size == 0:
57 |         return  # do not fail the whole process
58 |     print_info(site_info, title, ext, size)
59 |     if not info_only:
60 |         for i in items:
61 |             print_info(site_info, i['title'], i['ext'], i['size'])
62 |             download_urls(
63 |                 [i['url']], i['title'], i['ext'], i['size'], **kwargs
64 |             )
65 | 
66 | 
67 | download = nanagogo_download
68 | download_playlist = playlist_not_supported(site_info)
69 | 


--------------------------------------------------------------------------------
/lulu/extractors/pinterest.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     get_content,
 9 | )
10 | from lulu.extractor import VideoExtractor
11 | 
12 | 
13 | class Pinterest(VideoExtractor):
14 |     # site name
15 |     name = 'Pinterest pinterest.com'
16 | 
17 |     # ordered list of supported stream types / qualities on this site
18 |     # order: high quality -> low quality
19 |     stream_types = [
20 |         {'id': 'original'},  # contains an 'id' or 'itag' field at minimum
21 |         {'id': 'small'},
22 |     ]
23 | 
24 |     def prepare(self, **kwargs):
25 |         # scrape the html
26 |         content = get_content(self.url)
27 |         # extract title
28 |         self.title = match1(
29 |             content,
30 |             r'<meta property="og:description" name="og:description" '
31 |             r'content="([^"]+)"'
32 |         )
33 | 
34 |         data = match1(
35 |             content,
36 |             r'<script type="application/json" id=\'initial-state\'>(.+)'
37 |             r'</script>'
38 |         )
39 |         data = json.loads(data)
40 |         keys = list(data['resources']['data']['PinPageResource'].keys())
41 |         orig_img = data['resources']['data']['PinPageResource'][keys[0]][
42 |             'data'
43 |         ]['images']['orig']['url']
44 |         twit_img = match1(
45 |             content,
46 |             r'<meta property="twitter:image:src" name="twitter:image:src" '
47 |             r'content="([^"]+)"'
48 |         )
49 |         # construct available streams
50 |         if orig_img:
51 |             self.streams['original'] = {'url': orig_img}
52 |         if twit_img:
53 |             self.streams['small'] = {'url': twit_img}
54 | 
55 |     def extract(self, **kwargs):
56 |         for i in self.streams:
57 |             # for each available stream
58 |             s = self.streams[i]
59 |             # fill in 'container' field and 'size' field (optional)
60 |             _, s['container'], s['size'] = url_info(s['url'])
61 |             # 'src' field is a list of processed urls for direct downloading
62 |             # usually derived from 'url'
63 |             s['src'] = [s['url']]
64 | 
65 | 
66 | site = Pinterest()
67 | download = site.download_by_url
68 | # TBD: implement download_playlist
69 | 


--------------------------------------------------------------------------------
/lulu/extractors/infoq.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import ssl
 4 | from urllib import request
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     get_content,
10 | )
11 | from lulu.extractor import VideoExtractor
12 | 
13 | 
14 | class Infoq(VideoExtractor):
15 |     name = 'InfoQ infoq.com'
16 | 
17 |     stream_types = [
18 |         {'id': 'video'},
19 |         {'id': 'audio'},
20 |         {'id': 'slides'}
21 |     ]
22 | 
23 |     def prepare(self, **kwargs):
24 |         content = get_content(self.url)
25 |         self.title = match1(content, r'<title>([^<]+)</title>')
26 |         s = match1(content, r'P\.s\s*=\s*\'([^\']+)\'')
27 |         scp = match1(content, r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'')
28 |         scs = match1(content, r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'')
29 |         sck = match1(content, r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'')
30 | 
31 |         mp3 = match1(content, r'name="filename"\s*value="([^"]+\.mp3)"')
32 |         if mp3:
33 |             mp3 = 'http://res.infoq.com/downloads/mp3downloads/{}'.format(mp3)
34 | 
35 |         pdf = match1(content, r'name="filename"\s*value="([^"]+\.pdf)"')
36 |         if pdf:
37 |             pdf = 'http://res.infoq.com/downloads/pdfdownloads/{}'.format(pdf)
38 | 
39 |         # cookie handler
40 |         ssl_context = request.HTTPSHandler(
41 |             context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)
42 |         )
43 |         cookie_handler = request.HTTPCookieProcessor()
44 |         opener = request.build_opener(ssl_context, cookie_handler)
45 |         opener.addheaders = [
46 |             ('Referer', self.url),
47 |             (
48 |                 'Cookie',
49 |                 'CloudFront-Policy={};CloudFront-Signature={};'
50 |                 'CloudFront-Key-Pair-Id={}'.format(scp, scs, sck)
51 |             )
52 |         ]
53 |         request.install_opener(opener)
54 |         if s:
55 |             self.streams['video'] = {'url': s}
56 |         if mp3:
57 |             self.streams['audio'] = {'url': mp3}
58 |         if pdf:
59 |             self.streams['slides'] = {'url': pdf}
60 | 
61 |     def extract(self, **kwargs):
62 |         for i in self.streams:
63 |             s = self.streams[i]
64 |             _, s['container'], s['size'] = url_info(s['url'])
65 |             s['src'] = [s['url']]
66 | 
67 | 
68 | site = Infoq()
69 | download = site.download_by_url
70 | download_playlist = site.download_by_url
71 | 


--------------------------------------------------------------------------------
/lulu/extractors/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from .acfun import *
 4 | from .alive import *
 5 | from .archive import *
 6 | from .baidu import *
 7 | from .bandcamp import *
 8 | from .baomihua import *
 9 | from .bcy import *
10 | from .bigthink import *
11 | from .bilibili import *
12 | from .bokecc import *
13 | from .cbs import *
14 | from .ckplayer import *
15 | from .cntv import *
16 | from .coub import *
17 | from .dailymotion import *
18 | from .dilidili import *
19 | from .douban import *
20 | from .douyin import *
21 | from .douyutv import *
22 | from .ehow import *
23 | from .facebook import *
24 | from .fantasy import *
25 | from .fc2video import *
26 | from .flickr import *
27 | from .freesound import *
28 | from .funshion import *
29 | from .giphy import *
30 | from .google import *
31 | from .heavymusic import *
32 | from .huaban import *
33 | from .icourses import *
34 | from .ifeng import *
35 | from .imgur import *
36 | from .infoq import *
37 | from .instagram import *
38 | from .iqilu import *
39 | from .iqiyi import *
40 | from .iwara import *
41 | from .joy import *
42 | from .ku6 import *
43 | from .kugou import *
44 | from .kuwo import *
45 | from .le import *
46 | from .lizhi import *
47 | from .longzhu import *
48 | from .magisto import *
49 | from .metacafe import *
50 | from .mgtv import *
51 | from .mtv81 import *
52 | from .musicplayon import *
53 | from .nanagogo import *
54 | from .naver import *
55 | from .netease import *
56 | from .nicovideo import *
57 | from .panda import *
58 | from .pinterest import *
59 | from .pixivision import *
60 | from .pixnet import *
61 | from .pptv import *
62 | from .qie import *
63 | from .qingting import *
64 | from .qq import *
65 | from .showroom import *
66 | from .sina import *
67 | from .sohu import *
68 | from .soundcloud import *
69 | from .suntv import *
70 | from .theplatform import *
71 | from .tucao import *
72 | from .tudou import *
73 | from .tumblr import *
74 | from .twitter import *
75 | from .ucas import *
76 | from .veoh import *
77 | from .videomega import *
78 | from .vimeo import *
79 | from .vine import *
80 | from .vk import *
81 | from .w56 import *
82 | from .wanmen import *
83 | from .weibo import *
84 | from .ximalaya import *
85 | from .xiami import *
86 | from .yinyuetai import *
87 | from .yixia import *
88 | from .youku import *
89 | from .youtube import *
90 | from .ted import *
91 | from .khan import *
92 | from .zhanqi import *
93 | from .kuaishou import *
94 | 


--------------------------------------------------------------------------------
/lulu/extractors/yinyuetai.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | from urllib import parse
 6 | 
 7 | from lulu.util import fs
 8 | from lulu.common import (
 9 |     match1,
10 |     url_info,
11 |     print_info,
12 |     get_content,
13 |     download_urls
14 | )
15 | 
16 | 
17 | __all__ = ['yinyuetai_download', 'yinyuetai_download_by_id']
18 | site_info = 'YinYueTai.com'
19 | 
20 | 
21 | def yinyuetai_download_by_id(
22 |     vid, title=None, output_dir='.', merge=True, info_only=False
23 | ):
24 |     video_info = json.loads(get_content(
25 |         'http://www.yinyuetai.com/insite/get-video-info?json=true&'
26 |         'videoId={}'.format(vid)
27 |     ))
28 |     url_models = video_info['videoInfo']['coreVideoInfo']['videoUrlModels']
29 |     url_models = sorted(url_models, key=lambda i: i['qualityLevel'])
30 |     url = url_models[-1]['videoUrl']
31 |     _type = ext = match1(url, r'\.(flv|mp4)')
32 |     _, _, size = url_info(url)
33 | 
34 |     print_info(site_info, title, _type, size)
35 |     if not info_only:
36 |         download_urls([url], title, ext, size, output_dir, merge=merge)
37 | 
38 | 
39 | def yinyuetai_download(
40 |     url, output_dir='.', merge=True, info_only=False, **kwargs
41 | ):
42 |     _id = match1(url, r'http://\w+.yinyuetai.com/video/(\d+)') or match1(
43 |         url, r'http://\w+.yinyuetai.com/video/h5/(\d+)'
44 |     )
45 |     if not _id:
46 |         yinyuetai_download_playlist(
47 |             url, output_dir=output_dir, merge=merge, info_only=info_only
48 |         )
49 |         return
50 | 
51 |     html = get_content(url)
52 |     title = match1(
53 |         html, r'<meta property="og:title"\s+content="([^"]+)"/>'
54 |     ) or match1(html, r'<title>(.*)')
55 |     assert title
56 |     title = parse.unquote(title)
57 |     title = fs.legitimize(title)
58 |     yinyuetai_download_by_id(
59 |         _id, title, output_dir, merge=merge, info_only=info_only
60 |     )
61 | 
62 | 
63 | def yinyuetai_download_playlist(
64 |     url, output_dir='.', merge=True, info_only=False, **kwargs
65 | ):
66 |     html = get_content(url)
67 | 
68 |     data_ids = re.findall(r'data-id="(\d+)"\s*data-index="\d+"', html)
69 |     for data_id in data_ids:
70 |         yinyuetai_download(
71 |             'http://v.yinyuetai.com/video/{}'.format(data_id),
72 |             output_dir=output_dir, merge=merge, info_only=info_only
73 |         )
74 | 
75 | 
76 | download = yinyuetai_download
77 | download_playlist = yinyuetai_download_playlist
78 | 


--------------------------------------------------------------------------------
/lulu/extractors/pixnet.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | from time import time
 6 | from urllib.parse import quote
 7 | 
 8 | from lulu.util import log
 9 | from lulu.common import (
10 |     match1,
11 |     url_info,
12 |     print_info,
13 |     get_content,
14 |     download_urls,
15 |     playlist_not_supported,
16 | )
17 | 
18 | 
19 | __all__ = ['pixnet_download']
20 | site_info = '痞客邦 pixnet.net'
21 | 
22 | 
23 | def pixnet_download(url, info_only=False, **kwargs):
24 |     if not re.match(r'http://(\w)+.pixnet.net/album/video/(\d)+', url):
25 |         log.wtf('[Failed] Unsupported URL pattern.')
26 |         return
27 |     # http://eric6513.pixnet.net/album/video/206644535
28 |     html = get_content(url)
29 |     title = ''.join(match1(
30 |         html, r'<meta property="og:description\" content="([^"]*)"'
31 |     ).split('-')[1:]).strip()
32 | 
33 |     time_now = int(time())
34 | 
35 |     m = re.match(r'http://(\w+).pixnet.net/album/video/(\d+)', url)
36 | 
37 |     username = m.group(1)
38 |     # eric6513
39 |     _id = m.group(2)
40 |     # 206644535
41 | 
42 |     data_dict = {
43 |         'username': username, 'autoplay': 1, 'id': _id, 'loop': 0,
44 |         'profile': 9, 'time': time_now,
45 |     }
46 |     # have to be like this
47 |     data_dict_str = quote(str(data_dict).replace("'", '"'), safe='"')
48 |     url2 = 'http://api.pixnet.tv/content?type=json&customData={}'.format(
49 |         data_dict_str
50 |     )
51 |     # &sig=edb07258e6a9ff40e375e11d30607983  can be blank for now
52 |     # if required, can be obtained from url like
53 |     # http://s.ext.pixnet.tv/user/eric6513/html5/autoplay/206644507.js
54 |     # http://api.pixnet.tv/content?type=json&customData={%22username%22:%22eric6513%22,%22id%22:%22206644535%22,%22time%22:1441823350,%22autoplay%22:0,%22loop%22:0,%22profile%22:7}
55 | 
56 |     video_json = get_content(url2)
57 |     content = json.loads(video_json)
58 |     url_main = content['element']['video_url']
59 |     url_backup = content['element']['backup_video_uri']
60 | 
61 |     try:
62 |         # In some rare cases the main URL is IPv6 only...
63 |         # Something like #611
64 |         url_info(url_main)
65 |         url = url_main
66 |     except Exception:
67 |         url = url_backup
68 | 
69 |     _type, ext, size = url_info(url)
70 |     print_info(site_info, title, _type, size)
71 |     if not info_only:
72 |         download_urls([url], title, ext, size, **kwargs)
73 | 
74 | 
75 | download = pixnet_download
76 | download_playlist = playlist_not_supported(site_info)
77 | 


--------------------------------------------------------------------------------
/lulu/extractors/yixia.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | from urllib.parse import urlparse
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['yixia_download']
17 | site_info = '秒拍 miaopai.com'
18 | 
19 | 
20 | def yixia_miaopai_download_by_scid(scid, info_only=False, **kwargs):
21 |     api_endpoint = (
22 |         'http://api.miaopai.com/m/v2_channel.json?fillType=259&scid='
23 |         '{scid}&vend=miaopai'.format(scid=scid)
24 |     )
25 |     html = get_content(api_endpoint)
26 |     api_content = json.loads(html)
27 |     video_url = match1(api_content['result']['stream']['base'], r'(.+)\?vend')
28 |     title = api_content['result']['ext']['t']
29 |     _type, ext, size = url_info(video_url)
30 |     print_info(site_info, title, _type, size)
31 |     if not info_only:
32 |         download_urls([video_url], title, ext, size, **kwargs)
33 | 
34 | 
35 | def yixia_xiaokaxiu_download_by_scid(scid, info_only=False, **kwargs):
36 |     site_info = '小咖秀 xiaokaxiu.com'
37 |     api_endpoint = (
38 |         'http://api.xiaokaxiu.com/video/web/get_play_video?'
39 |         'scid={scid}'.format(scid=scid)
40 |     )
41 |     html = get_content(api_endpoint)
42 |     api_content = json.loads(html)
43 |     video_url = api_content['data']['linkurl']
44 |     title = api_content['data']['title']
45 |     _type, ext, size = url_info(video_url)
46 |     print_info(site_info, title, _type, size)
47 |     if not info_only:
48 |         download_urls([video_url], title, ext, size, **kwargs)
49 | 
50 | 
51 | def yixia_download(url, info_only=False, **kwargs):
52 |     hostname = urlparse(url).hostname
53 |     if 'miaopai.com' in hostname:  # Miaopai
54 |         yixia_download_by_scid = yixia_miaopai_download_by_scid
55 | 
56 |         scid = match1(url, r'miaopai\.com/show/channel/(.+)\.htm') or \
57 |             match1(url, r'miaopai\.com/show/(.+)\.htm') or \
58 |             match1(url, r'm\.miaopai\.com/show/channel/(.+)\.htm') or \
59 |             match1(url, r'm\.miaopai\.com/show/channel/([^\?]+)')
60 | 
61 |     elif 'xiaokaxiu.com' in hostname:  # Xiaokaxiu
62 |         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid
63 | 
64 |         scid = match1(url, r'v.xiaokaxiu.com/v/(.+)\.html') or \
65 |             match1(url, r'm.xiaokaxiu.com/m/(.+)\.html')
66 | 
67 |     else:
68 |         pass
69 | 
70 |     yixia_download_by_scid(scid, info_only, **kwargs)
71 | 
72 | 
73 | download = yixia_download
74 | download_playlist = playlist_not_supported(site_info)
75 | 


--------------------------------------------------------------------------------
/lulu/extractors/fc2video.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from hashlib import md5
 4 | from urllib.parse import urlparse
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | from lulu.config import FAKE_HEADERS
15 | 
16 | 
17 | __all__ = ['fc2video_download']
18 | site_info = 'FC2 Video video.fc2.com'
19 | 
20 | 
21 | def makeMimi(upid):
22 |     """From http://cdn37.atwikiimg.com/sitescript/pub/dksitescript/FC2.site.js
23 |     Also com.hps.util.fc2.FC2EncrptUtil.makeMimiLocal
24 |     L110
25 |     """
26 |     strSeed = 'gGddgPfeaf_gzyr'
27 |     prehash = '{}_{}'.format(upid, strSeed)
28 |     return md5(prehash.encode('utf-8')).hexdigest()
29 | 
30 | 
31 | def fc2video_download_by_upid(
32 |     upid, output_dir='.', merge=True, info_only=False, **kwargs
33 | ):
34 |     fake_headers = FAKE_HEADERS.copy()
35 |     fake_headers.update({
36 |         'DNT': '1',
37 |         'Accept-Encoding': 'gzip, deflate, sdch',
38 |         'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
39 |         'X-Requested-With': 'ShockwaveFlash/19.0.0.245',
40 |         'Connection': 'keep-alive',
41 |     })
42 |     api_base = (
43 |         'https://video.fc2.com/ginfo.php?upid={upid}&mimi='
44 |         '{mimi}'.format(upid=upid, mimi=makeMimi(upid))
45 |     )
46 |     html = get_content(api_base, headers=fake_headers)
47 |     video_url = match1(html, r'filepath=(.+)&sec')
48 |     video_url = video_url.replace('&mid', '?mid')
49 | 
50 |     title = match1(html, r'&title=([^&]+)')
51 | 
52 |     _type, ext, size = url_info(video_url, headers=fake_headers)
53 |     print_info(site_info, title, _type, size)
54 |     if not info_only:
55 |         download_urls(
56 |             [video_url], title, ext, size, output_dir, merge=merge,
57 |             headers=fake_headers, **kwargs
58 |         )
59 | 
60 | 
61 | def fc2video_download(
62 |     url, output_dir='.', merge=True, info_only=False, **kwargs
63 | ):
64 |     """wrapper"""
65 |     # 'http://video.fc2.com/en/content/20151021bTVKnbEw'
66 |     # 'http://xiaojiadianvideo.asia/content/20151021bTVKnbEw'
67 |     # 'http://video.fc2.com/ja/content/20151021bTVKnbEw'
68 |     # 'http://video.fc2.com/tw/content/20151021bTVKnbEw'
69 |     hostname = urlparse(url).hostname
70 |     if not ('fc2.com' in hostname or 'xiaojiadianvideo.asia' in hostname):
71 |         return False
72 |     upid = match1(url, r'.+/content/(\w+)')
73 | 
74 |     fc2video_download_by_upid(upid, output_dir, merge, info_only)
75 | 
76 | 
77 | download = fc2video_download
78 | download_playlist = playlist_not_supported(site_info)
79 | 


--------------------------------------------------------------------------------
/lulu/extractors/huaban.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import os
 5 | import json
 6 | import urllib.parse as urlparse
 7 | 
 8 | from lulu.common import (
 9 |     match1,
10 |     url_size,
11 |     print_info,
12 |     get_content,
13 |     download_urls,
14 |     playlist_not_supported,
15 | )
16 | 
17 | 
18 | __all__ = ['huaban_download']
19 | site_info = '花瓣 (Huaban)'
20 | LIMIT = 100
21 | 
22 | 
23 | class Board:
24 |     def __init__(self, title, pins):
25 |         self.title = title
26 |         self.pins = pins
27 |         self.pin_count = len(pins)
28 | 
29 | 
30 | class Pin:
31 |     host = 'http://img.hb.aicdn.com/'
32 | 
33 |     def __init__(self, pin_json):
34 |         img_file = pin_json['file']
35 |         self.id = str(pin_json['pin_id'])
36 |         self.url = urlparse.urljoin(self.host, img_file['key'])
37 |         self.ext = img_file['type'].split('/')[-1]
38 | 
39 | 
40 | def construct_url(url, **params):
41 |     param_str = urlparse.urlencode(params)
42 |     return url + '?' + param_str
43 | 
44 | 
45 | def extract_json_data(url, **params):
46 |     url = construct_url(url, **params)
47 |     html = get_content(url)
48 |     json_string = match1(html, r'app.page\["board"\] = (.*?});')
49 |     json_data = json.loads(json_string)
50 |     return json_data
51 | 
52 | 
53 | def extract_board_data(url):
54 |     json_data = extract_json_data(url, limit=LIMIT)
55 |     pin_list = json_data['pins']
56 |     title = json_data['title']
57 |     pin_count = json_data['pin_count']
58 |     pin_count -= len(pin_list)
59 | 
60 |     while pin_count > 0:
61 |         json_data = extract_json_data(
62 |             url, max=pin_list[-1]['pin_id'], limit=LIMIT
63 |         )
64 |         pins = json_data['pins']
65 |         pin_list += pins
66 |         pin_count -= len(pins)
67 | 
68 |     return Board(title, list(map(Pin, pin_list)))
69 | 
70 | 
71 | def huaban_download_board(url, output_dir, **kwargs):
72 |     board = extract_board_data(url)
73 |     output_dir = os.path.join(output_dir, board.title)
74 |     print_info(site_info, board.title, 'jpg', 0)
75 |     if not kwargs['info_only']:
76 |         for pin in board.pins:
77 |             download_urls(
78 |                 [pin.url], pin.id, pin.ext, url_size(pin.url),
79 |                 output_dir=output_dir, **kwargs
80 |             )
81 | 
82 | 
83 | def huaban_download(url, output_dir='.', **kwargs):
84 |     if re.match(r'http://huaban\.com/boards/\d+/', url):
85 |         huaban_download_board(url, output_dir, **kwargs)
86 |     else:
87 |         print('Only board (画板) pages are supported currently')
88 |         print('For example: http://huaban.com/boards/16687763/')
89 | 
90 | 
91 | download = huaban_download
92 | download_playlist = playlist_not_supported(site_info)
93 | 


--------------------------------------------------------------------------------
/lulu/extractors/lizhi.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 | )
13 | 
14 | 
15 | __all__ = ['lizhi_download']
16 | site_info = '荔枝 lizhi.fm'
17 | 
18 | 
19 | # radio_id: e.g. 549759 from http://www.lizhi.fm/549759/
20 | #
21 | # Returns a list of tuples (audio_id, title, url) for each episode
22 | # (audio) in the radio playlist. url is the direct link to the audio
23 | # file.
24 | def lizhi_extract_playlist_info(radio_id):
25 |     # /api/radio_audios API parameters:
26 |     #
27 |     # - s: starting episode
28 |     # - l: count (per page)
29 |     # - band: radio_id
30 |     #
31 |     # We use l=65535 for poor man's pagination (that is, no pagination
32 |     # at all -- hope all fits on a single page).
33 |     #
34 |     # TODO: Use /api/radio?band={radio_id} to get number of episodes
35 |     # (au_cnt), then handle pagination properly.
36 |     api_url = (
37 |         'http://www.lizhi.fm/api/radio_audios?s=0&l=65535&band={}'.format(
38 |             radio_id
39 |         )
40 |     )
41 |     api_response = json.loads(get_content(api_url))
42 |     return api_response
43 | 
44 | 
45 | def lizhi_download_audio(audio_id, title, url, info_only=False, **kwargs):
46 |     filetype, ext, size = url_info(url)
47 |     print_info(site_info, title, filetype, size)
48 |     if not info_only:
49 |         download_urls([url], title, ext, size, **kwargs)
50 | 
51 | 
52 | def lizhi_download_playlist(url, info_only=False, **kwargs):
53 |     # Sample URL: http://www.lizhi.fm/549759/
54 |     radio_id = match1(url, r'/(\d+)')
55 |     if not radio_id:
56 |         raise NotImplementedError('{} not supported'.format(url))
57 |     for data in lizhi_extract_playlist_info(radio_id):
58 |         lizhi_download_audio(
59 |             data['id'], data['name'], data['fixedHighPlayUrl'],
60 |             info_only=info_only, **kwargs
61 |         )
62 | 
63 | 
64 | def lizhi_download(url, output_dir='.', info_only=False, **kwargs):
65 |     # Sample URL: http://www.lizhi.fm/549759/2508612053517249030
66 |     m = re.search(r'/(?P<radio_id>\d+)/(?P<audio_id>\d+)', url)
67 |     if not m:
68 |         raise NotImplementedError('{} not supported'.format(url))
69 |     radio_id = m.group('radio_id')
70 |     audio_id = m.group('audio_id')
71 |     # Look for the audio_id among the full list of episodes
72 |     data = list(filter(
73 |         lambda x: x['id'] == audio_id, lizhi_extract_playlist_info(radio_id)
74 |     ))[0]
75 |     lizhi_download_audio(
76 |         audio_id, data['name'], data['fixedHighPlayUrl'], info_only=info_only,
77 |         **kwargs
78 |     )
79 | 
80 | 
81 | download = lizhi_download
82 | download_playlist = lizhi_download_playlist
83 | 


--------------------------------------------------------------------------------
/lulu/extractors/toutiao.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | import base64
 5 | import random
 6 | import binascii
 7 | 
 8 | from lulu.common import (
 9 |     match1,
10 |     url_info,
11 |     print_info,
12 |     get_content,
13 |     download_urls,
14 |     playlist_not_supported,
15 | )
16 | 
17 | 
18 | __all__ = ['toutiao_download', ]
19 | site_info = 'Toutiao.com'
20 | 
21 | 
22 | def sign_video_url(vid):
23 |     # some code from http://codecloud.net/110854.html
24 |     r = str(random.random())[2:]
25 | 
26 |     def right_shift(val, n):
27 |         return val >> n if val >= 0 else (val + 0x100000000) >> n
28 | 
29 |     url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % vid
30 |     n = '{}?r={}'.format(
31 |         url.replace('http://i.snssdk.com', ''), r
32 |     )
33 |     c = binascii.crc32(n.encode('ascii'))
34 |     s = right_shift(c, 0)
35 |     return '{}?r={}&s={}'.format(url, r, s)
36 | 
37 | 
38 | class ToutiaoVideoInfo(object):
39 | 
40 |     def __init__(self):
41 |         self.bitrate = None
42 |         self.definition = None
43 |         self.size = None
44 |         self.height = None
45 |         self.width = None
46 |         self.type = None
47 |         self.url = None
48 | 
49 |     def __str__(self):
50 |         return json.dumps(self.__dict__)
51 | 
52 | 
53 | def get_file_by_vid(video_id):
54 |     vRet = []
55 |     url = sign_video_url(video_id)
56 |     ret = get_content(url)
57 |     ret = json.loads(ret)
58 |     vlist = ret.get('data').get('video_list')
59 |     if len(vlist) > 0:
60 |         vInfo = vlist.get(sorted(vlist.keys(), reverse=True)[0])
61 |         vUrl = vInfo.get('main_url')
62 |         vUrl = base64.decodestring(vUrl.encode('ascii')).decode('ascii')
63 |         videoInfo = ToutiaoVideoInfo()
64 |         videoInfo.bitrate = vInfo.get('bitrate')
65 |         videoInfo.definition = vInfo.get('definition')
66 |         videoInfo.size = vInfo.get('size')
67 |         videoInfo.height = vInfo.get('vheight')
68 |         videoInfo.width = vInfo.get('vwidth')
69 |         videoInfo.type = vInfo.get('vtype')
70 |         videoInfo.url = vUrl
71 |         vRet.append(videoInfo)
72 |     return vRet
73 | 
74 | 
75 | def toutiao_download(url, info_only=False, **kwargs):
76 |     html = get_content(url)
77 |     video_id = match1(html, r"videoid\s*:\s*'([^']+)',\n")
78 |     title = match1(html, r"title: '([^']+)'.replace")
79 |     video_file_list = get_file_by_vid(video_id)  # 调api获取视频源文件
80 |     _type, ext, size = url_info(video_file_list[0].url)
81 |     print_info(site_info=site_info, title=title, type=_type, size=size)
82 |     if not info_only:
83 |         download_urls([video_file_list[0].url], title, ext, size, **kwargs)
84 | 
85 | 
86 | download = toutiao_download
87 | download_playlist = playlist_not_supported(site_info)
88 | 


--------------------------------------------------------------------------------
/tests/util.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | import os
 4 | import unittest
 5 | import functools
 6 | from socket import (
 7 |     gaierror,
 8 | )
 9 | from string import Formatter
10 | from urllib3.exceptions import (
11 |     MaxRetryError,
12 |     NewConnectionError,
13 | )
14 | from http.client import (
15 |     IncompleteRead,
16 |     RemoteDisconnected,
17 | )
18 | 
19 | from requests.exceptions import (
20 |     ConnectionError,
21 | )
22 | 
23 | 
24 | # 在 CI 上百分百失败用下面的 skipIf，有可能失败的用 ignore_network_issue
25 | # 使用 skip 或者 ignore 的必须在本地上跑通过
26 | NETWORK_ISSUE = 'tests will fail due to the network issue'
27 | skipOnCI = unittest.skipIf('CI' in os.environ, NETWORK_ISSUE)
28 | skipOnAppVeyor = unittest.skipIf('APPVEYOR' in os.environ, NETWORK_ISSUE)
29 | skipOnTravis = unittest.skipIf('TRAVIS' in os.environ, NETWORK_ISSUE)
30 | 
31 | 
32 | class ErrorMessageFormatter(Formatter):
33 |     def __init__(self, **kwargs):
34 |         super().__init__()
35 |         self.namespace = kwargs
36 | 
37 |     def get_value(self, key, args, kwargs):
38 |         if isinstance(key, str):
39 |             try:
40 |                 return kwargs[key]
41 |             except KeyError:
42 |                 return self.namespace[key]
43 |         else:
44 |             return super().get_value(key, args, kwargs)
45 | 
46 | 
47 | def ignore_network_issue(func):
48 |     @functools.wraps(func)
49 |     def wrapper(*args, **kwargs):
50 |         error = ErrorMessageFormatter(func_name=func.__name__)
51 |         error_message = '{func_name}: {module_name} error: {err}'
52 |         try:
53 |             func(*args, **kwargs)
54 |         except gaierror as err:
55 |             print(error.format(
56 |                 error_message, module_name='socket', err=err
57 |             ))
58 |         except NewConnectionError as err:
59 |             print(error.format(
60 |                 error_message, module_name='urllib3', err=err
61 |             ))
62 |         except MaxRetryError as err:
63 |             print(error.format(
64 |                 error_message, module_name='urllib3', err=err
65 |             ))
66 |         except ConnectionError as err:
67 |             print(error.format(
68 |                 error_message, module_name='requests', err=err
69 |             ))
70 |         except IncompleteRead as err:
71 |             # test_qq: httplib error: IncompleteRead(126198 bytes read, 62466 more expected)  # noqa
72 |             print(error.format(
73 |                 error_message, module_name='httplib', err=err
74 |             ))
75 |         except RemoteDisconnected as err:
76 |             print(error.format(
77 |                 error_message, module_name='httplib', err=err
78 |             ))
79 |         except Exception:
80 |             raise
81 |     return wrapper
82 | 
83 | 
84 | if __name__ == '__main__':
85 |     __import__('ipdb').set_trace()
86 | 


--------------------------------------------------------------------------------
/lulu/extractors/showroom.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | from time import time, sleep
 6 | 
 7 | from lulu.common import (
 8 |     match1,
 9 |     url_info,
10 |     print_info,
11 |     get_content,
12 |     download_url_ffmpeg,
13 |     playlist_not_supported,
14 | )
15 | from lulu.util import log
16 | from lulu.config import FAKE_HEADERS_MOBILE
17 | 
18 | 
19 | __all__ = ['showroom_download']
20 | site_info = 'SHOWROOM showroom-live.com'
21 | 
22 | 
23 | def showroom_get_roomid_by_room_url_key(room_url_key):
24 |     """str->str
25 |     """
26 |     webpage_url = 'https://www.showroom-live.com/{}'.format(room_url_key)
27 |     html = get_content(webpage_url, headers=FAKE_HEADERS_MOBILE)
28 |     roomid = match1(html, r'room\?room_id\=(\d+)')
29 |     assert roomid
30 |     return roomid
31 | 
32 | 
33 | def showroom_download_by_room_id(room_id, info_only=False, **kwargs):
34 |     '''Source: Android mobile
35 |     '''
36 |     while True:
37 |         timestamp = str(int(time() * 1000))
38 |         api_endpoint = (
39 |             'https://www.showroom-live.com/api/live/streaming_url?room_id='
40 |             '{room_id}&_={timestamp}'.format(
41 |                 room_id=room_id, timestamp=timestamp
42 |             )
43 |         )
44 |         html = get_content(api_endpoint)
45 |         html = json.loads(html)
46 |         if len(html) >= 1:
47 |             break
48 |         log.w('The live show is currently offline.')
49 |         sleep(1)
50 | 
51 |     # This is mainly for testing the M3U FFmpeg parser so I would ignore
52 |     # any non-m3u ones
53 |     stream_url = [
54 |         i['url'] for i in html['streaming_url_list']
55 |         if i['is_default'] and i['type'] == 'hls'
56 |     ][0]
57 |     assert stream_url
58 |     # title
59 |     title = ''
60 |     profile_api = (
61 |         'https://www.showroom-live.com/api/room/profile?room_id='
62 |         '{room_id}'.format(room_id=room_id)
63 |     )
64 |     html = json.loads(get_content(profile_api))
65 |     try:
66 |         title = html['main_name']
67 |     except KeyError:
68 |         title = 'Showroom_{room_id}'.format(room_id=room_id)
69 | 
70 |     type_, ext, size = url_info(stream_url)
71 |     print_info(site_info, title, type_, size)
72 |     if not info_only:
73 |         download_url_ffmpeg(
74 |             url=stream_url, title=title, ext='mp4', **kwargs
75 |         )
76 | 
77 | 
78 | def showroom_download(url, info_only=False, **kwargs):
79 |     if re.match(r'(\w+)://www.showroom-live.com/([-\w]+)', url):
80 |         room_url_key = match1(url, r'\w+://www.showroom-live.com/([-\w]+)')
81 |         room_id = showroom_get_roomid_by_room_url_key(room_url_key)
82 |         showroom_download_by_room_id(room_id, info_only, **kwargs)
83 | 
84 | 
85 | download = showroom_download
86 | download_playlist = playlist_not_supported(site_info)
87 | 


--------------------------------------------------------------------------------
/lulu/extractors/instagram.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     url_size,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | from lulu.util.parser import get_parser
16 | 
17 | 
18 | __all__ = ['instagram_download']
19 | site_info = 'Instagram.com'
20 | 
21 | 
22 | def instagram_download(
23 |     url, output_dir='.', merge=True, info_only=False, **kwargs
24 | ):
25 |     url = match1(url, r'([^?]*)')
26 |     html = get_content(url)
27 | 
28 |     vid = match1(url, r'instagram.com/p/([^/]+)')
29 |     parser = get_parser(html)
30 |     description = parser.find('meta', property='og:title')['content']
31 |     title = '{} [{}]'.format(description, vid)
32 |     stream = parser.find('meta', property='og:video')
33 |     if stream:
34 |         stream = stream['content']
35 |         _, ext, size = url_info(stream)
36 | 
37 |         print_info(site_info, title, ext, size)
38 |         if not info_only:
39 |             download_urls([stream], title, ext, size, output_dir, merge=merge)
40 |     else:
41 |         data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
42 |         info = json.loads(data.group(1))
43 | 
44 |         if 'edge_sidecar_to_children' in info['entry_data']['PostPage'][0][
45 |             'graphql'
46 |         ]['shortcode_media']:
47 |             edges = info['entry_data']['PostPage'][0]['graphql'][
48 |                 'shortcode_media'
49 |             ]['edge_sidecar_to_children']['edges']
50 |             for edge in edges:
51 |                 title = edge['node']['shortcode']
52 |                 image_url = edge['node']['display_url']
53 |                 ext = image_url.split('.')[-1]
54 |                 size = url_size(image_url)
55 |                 print_info(site_info, title, ext, size)
56 |                 if not info_only:
57 |                     download_urls(
58 |                         urls=[image_url], title=title, ext=ext,
59 |                         total_size=size, output_dir=output_dir
60 |                     )
61 |         else:
62 |             title = info['entry_data']['PostPage'][0]['graphql'][
63 |                 'shortcode_media'
64 |             ]['shortcode']
65 |             image_url = info['entry_data']['PostPage'][0]['graphql'][
66 |                 'shortcode_media'
67 |             ]['display_url']
68 |             ext = image_url.split('.')[-1]
69 |             size = url_size(image_url)
70 |             print_info(site_info, title, ext, size)
71 |             if not info_only:
72 |                 download_urls(
73 |                     urls=[image_url], title=title, ext=ext, total_size=size,
74 |                     output_dir=output_dir
75 |                 )
76 | 
77 | 
78 | download = instagram_download
79 | download_playlist = playlist_not_supported(site_info)
80 | 


--------------------------------------------------------------------------------
/lulu/extractors/zhanqi.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | import base64
 5 | from urllib.parse import urlparse
 6 | 
 7 | from lulu.common import (
 8 |     print_info,
 9 |     get_content,
10 |     download_urls,
11 |     download_url_ffmpeg,
12 |     general_m3u8_extractor,
13 |     playlist_not_supported,
14 | )
15 | 
16 | 
17 | __all__ = ['zhanqi_download']
18 | site_info = '战旗 zhanqi.tv'
19 | 
20 | 
21 | def zhanqi_download(
22 |     url, output_dir='.', merge=True, info_only=False, **kwargs
23 | ):
24 |     path = urlparse(url).path[1:]
25 | 
26 |     if not (path.startswith('videos') or path.startswith('v2/videos')):
27 |         path_list = path.split('/')
28 |         room_id = path_list[1] if path_list[0] == 'topic' else path_list[0]
29 |         zhanqi_live(
30 |             room_id, merge=merge, output_dir=output_dir, info_only=info_only,
31 |             **kwargs
32 |         )
33 |     else:
34 |         # https://www.zhanqi.tv/videos/Lyingman/2017/01/182308.html
35 |         # https://www.zhanqi.tv/v2/videos/215593.html
36 |         video_id = path.split('.')[0].split('/')[-1]
37 |         zhanqi_video(
38 |             video_id, merge=merge, output_dir=output_dir, info_only=info_only,
39 |             **kwargs
40 |         )
41 | 
42 | 
43 | def zhanqi_live(
44 |     room_id, merge=True, output_dir='.', info_only=False, **kwargs
45 | ):
46 |     api_url = (
47 |         'https://www.zhanqi.tv/api/static/v2.2/room/domain/{}.json'.format(
48 |             room_id
49 |         )
50 |     )
51 |     json_data = json.loads(get_content(api_url))['data']
52 |     status = json_data['status']
53 |     if status != '4':
54 |         raise Exception('The live stream is not online!')
55 | 
56 |     nickname = json_data['nickname']
57 |     title = '{}：{}'.format(
58 |         nickname, json_data['title']
59 |     )
60 |     video_levels = base64.b64decode(
61 |         json_data['flashvars']['VideoLevels']
62 |     ).decode('utf8')
63 |     m3u8_url = json.loads(video_levels)['streamUrl']
64 | 
65 |     print_info(
66 |         site_info, title, 'm3u8', 0, m3u8_url=m3u8_url, m3u8_type='master'
67 |     )
68 |     if not info_only:
69 |         download_url_ffmpeg(
70 |             m3u8_url, title, 'mp4', output_dir=output_dir, merge=merge
71 |         )
72 | 
73 | 
74 | def zhanqi_video(
75 |     video_id, output_dir='.', info_only=False, merge=True, **kwargs
76 | ):
77 |     api_url = 'https://www.zhanqi.tv/api/static/v2.2/video/{}.json'.format(
78 |         video_id
79 |     )
80 |     json_data = json.loads(get_content(api_url))['data']
81 |     title = json_data['title']
82 |     m3u8_url = json_data['playUrl']
83 |     urls = general_m3u8_extractor(m3u8_url)
84 |     print_info(site_info, title, 'm3u8', 0)
85 |     if not info_only:
86 |         download_urls(
87 |             urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs
88 |         )
89 | 
90 | 
91 | download = zhanqi_download
92 | download_playlist = playlist_not_supported(site_info)
93 | 


--------------------------------------------------------------------------------
/lulu/__main__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import getopt
 4 | import os
 5 | import platform
 6 | import sys
 7 | from .version import script_name, __version__
 8 | from .util import git, log
 9 | 
10 | _options = [
11 |     'help',
12 |     'version',
13 |     'gui',
14 |     'force',
15 |     'playlists',
16 | ]
17 | _short_options = 'hVgfl'
18 | 
19 | _help = """Usage: {} [OPTION]... [URL]...
20 | TODO
21 | """.format(script_name)
22 | 
23 | # TBD
24 | def main_dev(**kwargs):
25 |     """Main entry point.
26 |     you-get-dev
27 |     """
28 | 
29 |     # Get (branch, commit) if running from a git repo.
30 |     head = git.get_head(kwargs['repo_path'])
31 | 
32 |     # Get options and arguments.
33 |     try:
34 |         opts, args = getopt.getopt(sys.argv[1:], _short_options, _options)
35 |     except getopt.GetoptError as e:
36 |         log.wtf("""
37 |     [Fatal] {}.
38 |     Try '{} --help' for more options.""".format(e, script_name))
39 | 
40 |     if not opts and not args:
41 |         # Display help.
42 |         print(_help)
43 |         # Enter GUI mode.
44 |         #from .gui import gui_main
45 |         #gui_main()
46 |     else:
47 |         conf = {}
48 |         for opt, arg in opts:
49 |             if opt in ('-h', '--help'):
50 |                 # Display help.
51 |                 print(_help)
52 | 
53 |             elif opt in ('-V', '--version'):
54 |                 # Display version.
55 |                 log.println("you-get:", log.BOLD)
56 |                 log.println("    version:  {}".format(__version__))
57 |                 if head is not None:
58 |                     log.println("    branch:   {}\n    commit:   {}".format(*head))
59 |                 else:
60 |                     log.println("    branch:   {}\n    commit:   {}".format("(stable)", "(tag v{})".format(__version__)))
61 | 
62 |                 log.println("    platform: {}".format(platform.platform()))
63 |                 log.println("    python:   {}".format(sys.version.split('\n')[0]))
64 | 
65 |             elif opt in ('-g', '--gui'):
66 |                 # Run using GUI.
67 |                 conf['gui'] = True
68 | 
69 |             elif opt in ('-f', '--force'):
70 |                 # Force download.
71 |                 conf['force'] = True
72 | 
73 |             elif opt in ('-l', '--playlist', '--playlists'):
74 |                 # Download playlist whenever possible.
75 |                 conf['playlist'] = True
76 | 
77 |         if args:
78 |             if 'gui' in conf and conf['gui']:
79 |                 # Enter GUI mode.
80 |                 from .gui import gui_main
81 |                 gui_main(*args, **conf)
82 |             else:
83 |                 # Enter console mode.
84 |                 from .console import console_main
85 |                 console_main(*args, **conf)
86 | 
87 | def main(**kwargs):
88 |     """Main entry point.
89 |     you-get (legacy)
90 |     """
91 |     from .common import main
92 |     main(**kwargs)
93 | 
94 | if __name__ == '__main__':
95 |     main()
96 | 


--------------------------------------------------------------------------------
/lulu/extractors/bigthink.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | 
 5 | from lulu.common import (
 6 |     match1,
 7 |     url_info,
 8 |     get_content,
 9 | )
10 | from lulu.config import FAKE_HEADERS
11 | from lulu.extractor import VideoExtractor
12 | 
13 | 
14 | class Bigthink(VideoExtractor):
15 |     name = 'Bigthink bigthink.com'
16 | 
17 |     # this is just a sample. Will make it in prepare()
18 |     stream_types = [
19 |         # {'id': '1080'},
20 |         # {'id': '720'},
21 |         # {'id': '360'},
22 |         # {'id': '288'},
23 |         # {'id': '190'},
24 |         # {'id': '180'},
25 |     ]
26 | 
27 |     @staticmethod
28 |     def get_streams_by_id(account_number, video_id):
29 |         """
30 |         int, int->list
31 | 
32 |         Get the height of the videos.
33 | 
34 |         Since brightcove is using 3 kinds of links: rtmp, http and https,
35 |         we will be using the HTTPS one to make it secure.
36 | 
37 |         If somehow akamaihd.net is blocked by the Great Fucking Wall,
38 |         change the "startswith https" to http.
39 |         """
40 |         endpoint = (
41 |             'https://edge.api.brightcove.com/playback/v1/accounts/'
42 |             '{account_number}/videos/{video_id}'.format(
43 |                 account_number=account_number, video_id=video_id
44 |             )
45 |         )
46 |         fake_header_id = FAKE_HEADERS.copy()
47 |         # is this somehow related to the time? Magic....
48 |         fake_header_id['Accept'] = (
49 |             'application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzp'
50 |             'rPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34w'
51 |             'eknpfG-sippyQ'
52 |         )
53 | 
54 |         html = get_content(endpoint, headers=fake_header_id)
55 |         html_json = json.loads(html)
56 | 
57 |         link_list = []
58 |         for i in html_json['sources']:
59 |             if 'src' in i and i.get('container') == 'MP4':  # to avoid KeyError
60 |                 if i['src'].startswith('https'):
61 |                     link_list.append((str(i['height']), i['src']))
62 | 
63 |         return link_list
64 | 
65 |     def prepare(self, **kwargs):
66 |         html = get_content(self.url)
67 |         self.title = match1(
68 |             html, r'<meta property="og:title" content="([^"]*)"'
69 |         )
70 |         account_number = match1(html, r'data-account="(\d+)"')
71 |         video_id = match1(html, r'data-brightcove-id="(\d+)"')
72 |         assert account_number, video_id
73 |         link_list = self.get_streams_by_id(account_number, video_id)
74 |         for i in link_list:
75 |             self.stream_types.append({'id': str(i[0])})
76 |             self.streams[i[0]] = {'url': i[1]}
77 | 
78 |     def extract(self, **kwargs):
79 |         for i in self.streams:
80 |             s = self.streams[i]
81 |             _, s['container'], s['size'] = url_info(s['url'])
82 |             s['src'] = [s['url']]
83 | 
84 | 
85 | site = Bigthink()
86 | download = site.download_by_url
87 | 


--------------------------------------------------------------------------------
/lulu/util/log.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # This file is Python 2 compliant.
  3 | 
  4 | import os
  5 | import sys
  6 | 
  7 | from lulu.version import script_name
  8 | 
  9 | 
 10 | TERM = os.getenv('TERM', '')
 11 | IS_ANSI_TERMINAL = TERM in (
 12 |     'eterm-color',
 13 |     'linux',
 14 |     'screen',
 15 |     'vt100',
 16 | ) or TERM.startswith('xterm')
 17 | 
 18 | # ANSI escape code
 19 | # See <http://en.wikipedia.org/wiki/ANSI_escape_code>
 20 | RESET = 0
 21 | BOLD = 1
 22 | UNDERLINE = 4
 23 | NEGATIVE = 7
 24 | NO_BOLD = 21
 25 | NO_UNDERLINE = 24
 26 | POSITIVE = 27
 27 | BLACK = 30
 28 | RED = 31
 29 | GREEN = 32
 30 | YELLOW = 33
 31 | BLUE = 34
 32 | MAGENTA = 35
 33 | CYAN = 36
 34 | LIGHT_GRAY = 37
 35 | DEFAULT = 39
 36 | BLACK_BACKGROUND = 40
 37 | RED_BACKGROUND = 41
 38 | GREEN_BACKGROUND = 42
 39 | YELLOW_BACKGROUND = 43
 40 | BLUE_BACKGROUND = 44
 41 | MAGENTA_BACKGROUND = 45
 42 | CYAN_BACKGROUND = 46
 43 | LIGHT_GRAY_BACKGROUND = 47
 44 | DEFAULT_BACKGROUND = 49
 45 | DARK_GRAY = 90                  # xterm
 46 | LIGHT_RED = 91                  # xterm
 47 | LIGHT_GREEN = 92                # xterm
 48 | LIGHT_YELLOW = 93               # xterm
 49 | LIGHT_BLUE = 94                 # xterm
 50 | LIGHT_MAGENTA = 95              # xterm
 51 | LIGHT_CYAN = 96                 # xterm
 52 | WHITE = 97                      # xterm
 53 | DARK_GRAY_BACKGROUND = 100      # xterm
 54 | LIGHT_RED_BACKGROUND = 101      # xterm
 55 | LIGHT_GREEN_BACKGROUND = 102    # xterm
 56 | LIGHT_YELLOW_BACKGROUND = 103   # xterm
 57 | LIGHT_BLUE_BACKGROUND = 104     # xterm
 58 | LIGHT_MAGENTA_BACKGROUND = 105  # xterm
 59 | LIGHT_CYAN_BACKGROUND = 106     # xterm
 60 | WHITE_BACKGROUND = 107          # xterm
 61 | 
 62 | 
 63 | def sprint(text, *colors):
 64 |     """Format text with color or other effects into ANSI escaped string."""
 65 |     return '\33[{}m{content}\33[{}m'.format(
 66 |         ';'.join([str(color) for color in colors]), RESET, content=text
 67 |     ) if IS_ANSI_TERMINAL and colors else text
 68 | 
 69 | 
 70 | def println(text, *colors):
 71 |     """Print text to standard output."""
 72 |     sys.stdout.write(sprint(text, *colors) + '\n')
 73 | 
 74 | 
 75 | def print_err(text, *colors):
 76 |     """Print text to standard error."""
 77 |     sys.stderr.write(sprint(text, *colors) + '\n')
 78 | 
 79 | 
 80 | def print_log(text, *colors):
 81 |     """Print a log message to standard error."""
 82 |     sys.stderr.write(
 83 |         sprint('{}: {}'.format(script_name, text), *colors) + '\n'
 84 |     )
 85 | 
 86 | 
 87 | def i(message):
 88 |     """Print a normal log message."""
 89 |     print_log(message)
 90 | 
 91 | 
 92 | def d(message):
 93 |     """Print a debug log message."""
 94 |     print_log(message, BLUE)
 95 | 
 96 | 
 97 | def w(message):
 98 |     """Print a warning log message."""
 99 |     print_log(message, YELLOW)
100 | 
101 | 
102 | def e(message, exit_code=None):
103 |     """Print an error log message."""
104 |     print_log(message, YELLOW, BOLD)
105 |     if exit_code is not None:
106 |         sys.exit(exit_code)
107 | 
108 | 
109 | def wtf(message, exit_code=1):
110 |     """What a Terrible Failure!"""
111 |     print_log(message, RED, BOLD)
112 |     if exit_code is not None:
113 |         sys.exit(exit_code)
114 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import io
  4 | import os
  5 | import sys
  6 | from shutil import rmtree
  7 | from setuptools import find_packages, setup, Command
  8 | 
  9 | 
 10 | # Package meta-data.
 11 | NAME = 'lulu'
 12 | DESCRIPTION = (
 13 |     'A simple and clean video/music/image downloader '
 14 |     'that supports many websites 👾'
 15 | )
 16 | URL = 'https://github.com/iawia002/Lulu'
 17 | EMAIL = 'z2d@jifangcheng.com'
 18 | AUTHOR = 'iawia002'
 19 | 
 20 | # What packages are required for this module to be executed?
 21 | REQUIRED = [
 22 |     'cryptography==2.1.4',
 23 |     'beautifulsoup4==4.6.0',
 24 |     'requests==2.18.4',
 25 | ]
 26 | 
 27 | here = os.path.abspath(os.path.dirname(__file__))
 28 | 
 29 | # Import the README and use it as the long-description.
 30 | with io.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
 31 |     long_description = '\n' + f.read()
 32 | 
 33 | # Load the package's __version__.py module as a dictionary.
 34 | about = {}
 35 | with open(os.path.join(here, NAME, 'version.py')) as f:
 36 |     exec(f.read(), about)
 37 | 
 38 | 
 39 | class UploadCommand(Command):
 40 |     """Support setup.py upload."""
 41 | 
 42 |     description = 'Build and publish the package.'
 43 |     user_options = []
 44 | 
 45 |     @staticmethod
 46 |     def status(s):
 47 |         """Prints things in bold."""
 48 |         print('\033[1m{0}\033[0m'.format(s))
 49 | 
 50 |     def initialize_options(self):
 51 |         pass
 52 | 
 53 |     def finalize_options(self):
 54 |         pass
 55 | 
 56 |     def run(self):
 57 |         try:
 58 |             self.status('Removing previous builds…')
 59 |             rmtree(os.path.join(here, 'dist'))
 60 |         except OSError:
 61 |             pass
 62 | 
 63 |         self.status('Building Source and Wheel (universal) distribution…')
 64 |         os.system('{0} setup.py sdist bdist_wheel --universal'.format(
 65 |             sys.executable
 66 |         ))
 67 | 
 68 |         self.status('Uploading the package to PyPi via Twine…')
 69 |         os.system('twine upload dist/*')
 70 | 
 71 |         sys.exit()
 72 | 
 73 | 
 74 | # Where the magic happens:
 75 | setup(
 76 |     name=NAME,
 77 |     version=about['__version__'],
 78 |     description=DESCRIPTION,
 79 |     long_description=long_description,
 80 |     author=AUTHOR,
 81 |     author_email=EMAIL,
 82 |     url=URL,
 83 |     packages=find_packages(exclude=('tests',)),
 84 |     entry_points={
 85 |         'console_scripts': ['lulu=lulu.__main__:main'],
 86 |     },
 87 |     install_requires=REQUIRED,
 88 |     include_package_data=True,
 89 |     zip_safe=False,
 90 |     license='MIT',
 91 |     classifiers=[
 92 |         'License :: OSI Approved :: MIT License',
 93 |         'Programming Language :: Python',
 94 |         'Programming Language :: Python :: 3',
 95 |         "Programming Language :: Python :: 3 :: Only",
 96 |         'Programming Language :: Python :: 3.4',
 97 |         'Programming Language :: Python :: 3.5',
 98 |         'Programming Language :: Python :: 3.6',
 99 |         'Programming Language :: Python :: Implementation :: CPython',
100 |     ],
101 |     # $ setup.py publish support.
102 |     cmdclass={
103 |         'upload': UploadCommand,
104 |     },
105 | )
106 | 


--------------------------------------------------------------------------------
/lulu/extractors/tucao.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | import time
 3 | import random
 4 | from xml.dom import minidom
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     url_info,
 9 |     print_info,
10 |     get_content,
11 |     download_urls,
12 |     playlist_not_supported,
13 | )
14 | 
15 | 
16 | __all__ = ['tucao_download']
17 | site_info = '吐槽弹幕网 tucao.tv'
18 | 
19 | 
20 | # possible raw list types
21 | # 1. <li>type=tudou&vid=199687639</li>
22 | # 2. <li>type=tudou&vid=199506910|</li>
23 | # 3. <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|</li>  # noqa
24 | # 4 may ? <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|xx**type=&vid=?</li>  # noqa
25 | # 5. <li>type=tudou&vid=200003098|07**type=tudou&vid=200000350|08</li>
26 | # 6. <li>vid=49454694&type=sina|</li>
27 | # 7. <li>type=189&vid=513031813243909|</li>
28 | # re_pattern=re.compile(r"(type=(.+?)&(vid|file)=(.*?))[\|<]")
29 | 
30 | 
31 | def tucao_single_download(type_link, title, info_only=False, **kwargs):
32 |     if 'file' in type_link:
33 |         url = type_link[type_link.find('file=')+5:]
34 |         vtype, ext, size = url_info(url)
35 |         print_info(site_info, title, vtype, size)
36 |         if not info_only:
37 |             download_urls([url], title, ext, size, **kwargs)
38 |     # fix for 189 video source, see raw list types 7
39 |     elif '189' in type_link:
40 |         vid = match1(type_link, r'vid=(\d+)')
41 |         assert vid, 'vid not exsits'
42 |         url = 'http://api.tucao.tv/api/down/{}'.format(vid)
43 |         vtype, ext, size = url_info(url)
44 |         print_info(site_info, title, vtype, size)
45 |         if not info_only:
46 |             download_urls([url], title, ext, size, **kwargs)
47 |     else:
48 |         u = (
49 |             'http://www.tucao.tv/api/playurl.php?{}&key=tucao{:07x}.'
50 |             'cc&r={}'.format(
51 |                 type_link, random.getrandbits(28), int(time.time()*1000)
52 |             )
53 |         )
54 |         xml = minidom.parseString(get_content(u))
55 |         urls = []
56 |         size = 0
57 |         for i in xml.getElementsByTagName('url'):
58 |             urls.append(i.firstChild.nodeValue)
59 |             vtype, ext, _size = url_info(i.firstChild.nodeValue)
60 |             size += _size
61 |         print_info(site_info, title, vtype, size)
62 |         if not info_only:
63 |             download_urls(urls, title, ext, size, **kwargs)
64 | 
65 | 
66 | def tucao_download(url, info_only=False, **kwargs):
67 |     html = get_content(url)
68 |     title = match1(html, r'<h1 class="show_title">(.*?)<\w')
69 |     # fix for raw list that vid goes before type, see raw list types 6
70 |     raw_list = match1(html, r'<li>\s*(type=.+?|vid=.+?)</li>')
71 |     raw_l = raw_list.split("**")
72 |     if len(raw_l) == 1:
73 |         format_link = raw_l[0][:-1] if raw_l[0].endswith('|') else raw_l[0]
74 |         tucao_single_download(format_link, title, info_only, **kwargs)
75 |     else:
76 |         for i in raw_l:
77 |             format_link, sub_title = i.split('|')
78 |             tucao_single_download(
79 |                 format_link, '{}-{}'.format(title, sub_title), info_only,
80 |                 **kwargs
81 |             )
82 | 
83 | 
84 | download = tucao_download
85 | download_playlist = playlist_not_supported(site_info)
86 | 


--------------------------------------------------------------------------------
/lulu/extractors/weibo.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import json
 4 | from urllib import parse
 5 | 
 6 | from lulu import config
 7 | from lulu.common import (
 8 |     match1,
 9 |     url_size,
10 |     print_info,
11 |     get_content,
12 |     download_urls,
13 |     playlist_not_supported,
14 | )
15 | from lulu.extractors.yixia import yixia_miaopai_download_by_scid
16 | 
17 | 
18 | __all__ = ['weibo_download']
19 | site_info = '微博 weibo.com'
20 | 
21 | 
22 | def weibo_download_by_fid(fid, info_only=False, **kwargs):
23 |     page_url = 'http://video.weibo.com/show?fid={}&type=mp4'.format(fid)
24 | 
25 |     mobile_page = get_content(page_url, headers=config.FAKE_HEADERS_MOBILE)
26 |     url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
27 |     title = match1(mobile_page, r'<title>((.|\n)+?)</title>')
28 |     if not title:
29 |         title = fid
30 |     title = title.replace('\n', '_')
31 |     ext, size = 'mp4', url_size(url)
32 |     print_info(site_info, title, ext, size)
33 |     if not info_only:
34 |         download_urls([url], title, ext, size, **kwargs)
35 | 
36 | 
37 | def get_fid(url):
38 |     return match1(url, r'\?fid=(\d{4}:\w{32})')
39 | 
40 | 
41 | def weibo_download(url, info_only=False, **kwargs):
42 |     fid = get_fid(url)
43 |     if fid:
44 |         weibo_download_by_fid(fid, info_only, **kwargs)
45 |     elif '/p/230444' in url:
46 |         fid = match1(url, r'/p/230444(\w+)')
47 |         weibo_download_by_fid('1034:{}'.format(fid), info_only, **kwargs)
48 |     else:
49 |         status_id = url.split('?')[0].split('/')[-1]
50 |         video_info = json.loads(
51 |             get_content(
52 |                 'https://m.weibo.cn/statuses/show?id={}'.format(status_id),
53 |                 headers=config.FAKE_HEADERS_MOBILE
54 |             )
55 |         )
56 |         video_url = video_info['data']['page_info']['media_info'][
57 |             'stream_url_hd'
58 |         ]
59 |         if not video_url:
60 |             video_url = parse.unquote(
61 |                 video_info['data']['page_info']['page_url']
62 |             )
63 |             if 'miaopai' in video_url:
64 |                 # https://weibo.cn/sinaurl/blocked817bc30b?u=http%3A%2F%2Fmiaopai.com%2Fshow%2FnPWJvdR4z2Bg1Sz3PJpNYffjpDgEiuv4msALgw__.htm  # noqa
65 |                 scid = match1(
66 |                     video_url.split('=')[-1],
67 |                     r'.*?miaopai.com/show/(.+)\.htm'
68 |                 )
69 |                 return yixia_miaopai_download_by_scid(
70 |                     scid, info_only=info_only, **kwargs
71 |                 )
72 |             elif 'fid=' in video_url:
73 |                 # http://video.weibo.com/show?fid=1034:b91d1ecf44b0e2f18c436d819744b333  # noqa
74 |                 fid = get_fid(video_url)
75 |                 return weibo_download_by_fid(fid, info_only, **kwargs)
76 |         title = video_info['data']['page_info']['content2']
77 |         video_format = 'mp4'
78 |         size = url_size(video_url)
79 |         print_info(
80 |             site_info=site_info, title=title, type=video_format, size=size
81 |         )
82 |         if not info_only:
83 |             download_urls(
84 |                 urls=[video_url], title=title, ext=video_format,
85 |                 total_size=size, **kwargs
86 |             )
87 | 
88 | 
89 | download = weibo_download
90 | download_playlist = playlist_not_supported(site_info)
91 | 


--------------------------------------------------------------------------------
/lulu/extractors/qie_video.py:
--------------------------------------------------------------------------------
 1 | # coding=utf-8
 2 | 
 3 | import re
 4 | import json
 5 | import math
 6 | 
 7 | from lulu.util import log
 8 | from lulu.common import get_content
 9 | from lulu.extractor import VideoExtractor
10 | 
11 | 
12 | class QieVideo(VideoExtractor):
13 |     name = '企鹅直播 live.qq.com'
14 |     vid_patt = r'"stream_name":"(\d+)"'
15 |     title_patt = r'"title":"([^\"]+)"'
16 |     cdn = 'http://qietv-play.wcs.8686c.com/'
17 |     ep = 'http://api.qiecdn.com/api/v1/video/stream/{}'
18 |     stream_types = [
19 |         {'id': '1080p', 'video_profile': '1920x1080', 'container': 'm3u8'},
20 |         {'id': '720p', 'video_profile': '1280x720', 'container': 'm3u8'},
21 |         {'id': '480p', 'video_profile': '853x480', 'container': 'm3u8'}
22 |     ]
23 | 
24 |     def get_vid_from_url(self):
25 |         hit = re.search(self.__class__.vid_patt, self.page)
26 |         if hit is None:
27 |             log.wtf('Cannot get stream_id')
28 |         return hit.group(1)
29 | 
30 |     def get_title(self):
31 |         hit = re.search(self.__class__.title_patt, self.page)
32 |         if hit is None:
33 |             return self.vid
34 |         return hit.group(1).strip()
35 | 
36 |     def prepare(self, **kwargs):
37 |         self.page = get_content(self.url)
38 |         if self.vid is None:
39 |             self.vid = self.get_vid_from_url()
40 |         self.title = self.get_title()
41 |         meta = json.loads(get_content(self.__class__.ep.format(self.vid)))
42 |         if meta['code'] != 200:
43 |             log.wtf(meta['message'])
44 |         for video in meta['result']['videos']:
45 |             height = video['height']
46 |             url = self.__class__.cdn + video['key']
47 |             stream_meta = dict(m3u8_url=url, size=0, container='mp4')
48 |             video_profile = '{}x{}'.format(video['width'], video['height'])
49 |             stream_meta['video_profile'] = video_profile
50 |             for stream_type in self.__class__.stream_types:
51 |                 if height // 10 == int(stream_type['id'][:-1]) // 10:
52 |                     # width 481, 482... 489 are all 480p here
53 |                     stream_id = stream_type['id']
54 |                     self.streams[stream_id] = stream_meta
55 | 
56 |     def extract(self, **kwargs):
57 |         for stream_id in self.streams:
58 |             self.streams[stream_id]['src'], dur = general_m3u8_extractor(
59 |                 self.streams[stream_id]['m3u8_url']
60 |             )
61 |             self.streams[stream_id]['video_profile'] += \
62 |                 ', Duration: {}s'.format(math.floor(dur))
63 | 
64 | 
65 | def general_m3u8_extractor(url):
66 |     dur = 0
67 |     base_url = url[:url.rfind('/')]
68 |     m3u8_content = get_content(url).split('\n')
69 |     result = []
70 |     for line in m3u8_content:
71 |         trimmed = line.strip()
72 |         if len(trimmed) > 0:
73 |             if trimmed.startswith('#'):
74 |                 if trimmed.startswith('#EXTINF'):
75 |                     t_str = re.search(r'(\d+\.\d+)', trimmed).group(1)
76 |                     dur += float(t_str)
77 |             else:
78 |                 if trimmed.startswith('http'):
79 |                     result.append(trimmed)
80 |                 else:
81 |                     result.append(base_url + '/' + trimmed)
82 |     return result, dur
83 | 
84 | 
85 | site = QieVideo()
86 | download_by_url = site.download_by_url
87 | 


--------------------------------------------------------------------------------
/lulu/extractors/cntv.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import re
 4 | import json
 5 | 
 6 | from lulu.common import (
 7 |     match1,
 8 |     get_content,
 9 |     playlist_not_supported,
10 | )
11 | from lulu.extractor import VideoExtractor
12 | 
13 | 
14 | __all__ = ['cntv_download', 'cntv_download_by_id']
15 | site_info = '央视网 cctv.com'
16 | 
17 | 
18 | class CNTV(VideoExtractor):
19 |     name = site_info
20 |     stream_types = [
21 |         {
22 |             'id': '1', 'video_profile': '1280x720_2000kb/s',
23 |             'map_to': 'chapters4'
24 |         },
25 |         {
26 |             'id': '2', 'video_profile': '1280x720_1200kb/s',
27 |             'map_to': 'chapters3'
28 |         },
29 |         {
30 |             'id': '3', 'video_profile': '640x360_850kb/s',
31 |             'map_to': 'chapters2'
32 |         },
33 |         {
34 |             'id': '4', 'video_profile': '480x270_450kb/s',
35 |             'map_to': 'chapters'
36 |         },
37 |         {
38 |             'id': '5', 'video_profile': '320x180_200kb/s',
39 |             'map_to': 'lowChapters'
40 |         },
41 |     ]
42 | 
43 |     ep = 'http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid={}'
44 | 
45 |     def __init__(self):
46 |         super().__init__()
47 |         self.api_data = None
48 | 
49 |     def prepare(self, **kwargs):
50 |         self.api_data = json.loads(
51 |             get_content(self.__class__.ep.format(self.vid))
52 |         )
53 |         self.title = self.api_data['title']
54 |         for s in self.api_data['video']:
55 |             for st in self.__class__.stream_types:
56 |                 if st['map_to'] == s:
57 |                     urls = self.api_data['video'][s]
58 |                     src = [u['url'] for u in urls]
59 |                     stream_data = dict(
60 |                         src=src, size=0, container='mp4',
61 |                         video_profile=st['video_profile']
62 |                     )
63 |                     self.streams[st['id']] = stream_data
64 | 
65 | 
66 | def cntv_download_by_id(rid, **kwargs):
67 |     CNTV().download_by_vid(rid, **kwargs)
68 | 
69 | 
70 | def cntv_download(url, **kwargs):
71 |     page = get_content(url)
72 |     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
73 |         rid = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
74 |     elif re.match(r'http://tv\.cctv\.com/\d+/\d+/\d+/\w+.shtml', url):
75 |         rid = match1(page, r'var guid = "(\w+)"')
76 |     elif re.match(
77 |         r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml',
78 |         url
79 |     ) or re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or re.match(
80 |         r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url
81 |     ) or re.match(
82 |         r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url
83 |     ) or re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url):
84 |         rid = match1(page, r'videoCenterId","(\w+)"')
85 |         if rid is None:
86 |             guid = re.search(r'guid\s*=\s*"([0-9a-z]+)"', page).group(1)
87 |             rid = guid
88 |     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
89 |         rid = match1(url, r'http://xiyou.cntv.cn/v-([\w-]+)\.html')
90 |     else:
91 |         raise NotImplementedError(url)
92 | 
93 |     CNTV().download_by_vid(rid, **kwargs)
94 | 
95 | 
96 | download = cntv_download
97 | download_playlist = playlist_not_supported(site_info)
98 | 


--------------------------------------------------------------------------------
/lulu/extractors/douyutv.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | import time
  6 | import hashlib
  7 | 
  8 | 
  9 | from lulu.util import log
 10 | from lulu.common import (
 11 |     match1,
 12 |     print_info,
 13 |     get_content,
 14 |     download_urls,
 15 |     download_url_ffmpeg,
 16 |     general_m3u8_extractor,
 17 |     playlist_not_supported,
 18 | )
 19 | 
 20 | 
 21 | __all__ = ['douyutv_download']
 22 | site_info = '斗鱼 douyu.com'
 23 | 
 24 | 
 25 | def douyutv_video_download(
 26 |     url, output_dir='.', merge=True, info_only=False, **kwargs
 27 | ):
 28 |     ep = 'http://vmobile.douyu.com/video/getInfo?vid='
 29 |     patt = r'show/([0-9A-Za-z]+)'
 30 |     title_patt = r'<h1>(.+?)</h1>'
 31 | 
 32 |     hit = re.search(patt, url)
 33 |     if hit is None:
 34 |         log.wtf('Unknown url pattern')
 35 |     vid = hit.group(1)
 36 | 
 37 |     page = get_content(url)
 38 |     hit = re.search(title_patt, page)
 39 |     if hit is None:
 40 |         title = vid
 41 |     else:
 42 |         title = hit.group(1)
 43 | 
 44 |     meta = json.loads(get_content(ep + vid))
 45 |     if meta['error'] != 0:
 46 |         log.wtf('Error from API server')
 47 |     m3u8_url = meta['data']['video_url']
 48 |     print_info(site_info, title, 'm3u8', 0, m3u8_url=m3u8_url)
 49 |     if not info_only:
 50 |         urls = general_m3u8_extractor(m3u8_url)
 51 |         download_urls(
 52 |             urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs
 53 |         )
 54 | 
 55 | 
 56 | def douyutv_download(
 57 |     url, output_dir='.', merge=True, info_only=False, **kwargs
 58 | ):
 59 |     if 'v.douyu.com/show/' in url:
 60 |         douyutv_video_download(
 61 |             url, output_dir=output_dir, merge=merge, info_only=info_only,
 62 |             **kwargs
 63 |         )
 64 |         return
 65 |     url = re.sub(r'[\w.]*douyu.com', 'm.douyu.com', url)
 66 |     html = get_content(url)
 67 |     room_id_patt = r'room_id\s*:\s*(\d+),'
 68 |     room_id = match1(html, room_id_patt)
 69 |     if room_id == '0':
 70 |         room_id = url[url.rfind('/')+1:]
 71 | 
 72 |     api_url = 'http://www.douyutv.com/api/v1/'
 73 |     args = 'room/{}?aid=wp&client_sys=wp&time={}'.format(
 74 |         room_id, int(time.time())
 75 |     )
 76 |     auth_md5 = (args + 'zNzMV1y4EMxOHS6I5WKm').encode('utf-8')
 77 |     auth_str = hashlib.md5(auth_md5).hexdigest()
 78 |     json_request_url = '{}{}&auth={}'.format(api_url, args, auth_str)
 79 | 
 80 |     content = get_content(json_request_url)
 81 |     json_content = json.loads(content)
 82 |     data = json_content['data']
 83 |     server_status = json_content.get('error', 0)
 84 |     if server_status is not 0:
 85 |         raise ValueError('Server returned error: {}'.format(server_status))
 86 | 
 87 |     title = data.get('room_name')
 88 |     show_status = data.get('show_status')
 89 |     if show_status is not '1':
 90 |         raise ValueError('The live stream is not online! (Errno: {})'.format(
 91 |             server_status
 92 |         ))
 93 | 
 94 |     real_url = '{}/{}'.format(
 95 |         data.get('rtmp_url'), data.get('rtmp_live')
 96 |     )
 97 |     print_info(site_info, title, 'flv', float('inf'))
 98 |     if not info_only:
 99 |         download_url_ffmpeg(
100 |             real_url, title, 'flv', None, output_dir=output_dir,
101 |             merge=merge
102 |         )
103 | 
104 | 
105 | download = douyutv_download
106 | download_playlist = playlist_not_supported(site_info)
107 | 


--------------------------------------------------------------------------------
/lulu/extractors/sohu.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | import time
  6 | from random import random
  7 | from urllib.parse import urlparse
  8 | 
  9 | from lulu.common import (
 10 |     match1,
 11 |     print_info,
 12 |     get_content,
 13 |     download_urls,
 14 |     playlist_not_supported,
 15 | )
 16 | 
 17 | __all__ = ['sohu_download']
 18 | site_info = '搜狐视频 tv.sohu.com'
 19 | 
 20 | 
 21 | '''
 22 | Changelog:
 23 |     1. http://tv.sohu.com/upload/swf/20150604/Main.swf
 24 |         new api
 25 | '''
 26 | 
 27 | 
 28 | def real_url(host, vid, tvid, new, clipURL, ck):
 29 |     url = (
 30 |         'http://{}/?prot=9&prod=flash&pt=1&file={}&new={}&key={}&vid={}'
 31 |         '&uid={}&t={}&rb=1'.format(
 32 |             host, clipURL, new, ck, str(vid), str(int(time.time()*1000)),
 33 |             str(random())
 34 |         )
 35 |     )
 36 |     return json.loads(get_content(url))['url']
 37 | 
 38 | 
 39 | def sohu_download(url, info_only=False, **kwargs):
 40 |     if re.match(r'http://share.vrs.sohu.com', url):
 41 |         vid = match1(url, 'id=(\d+)')
 42 |     else:
 43 |         html = get_content(url)
 44 |         vid = match1(html, r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?')
 45 |     assert vid
 46 | 
 47 |     if re.match(r'http[s]://tv.sohu.com/', url):
 48 |         info = json.loads(get_content(
 49 |             'http://hot.vrs.sohu.com/vrs_flash.action?vid={}'.format(vid)
 50 |         ))
 51 |         for qtyp in ['oriVid', 'superVid', 'highVid', 'norVid', 'relativeId']:
 52 |             if 'data' in info:
 53 |                 hqvid = info['data'][qtyp]
 54 |             else:
 55 |                 hqvid = info[qtyp]
 56 |             if hqvid != 0 and hqvid != vid:
 57 |                 info = json.loads(get_content(
 58 |                     'http://hot.vrs.sohu.com/vrs_flash.action?vid={}'.format(
 59 |                         hqvid
 60 |                     )
 61 |                 ))
 62 |                 if 'allot' not in info:
 63 |                     continue
 64 |                 break
 65 |         host = info['allot']
 66 |         tvid = info['tvid']
 67 |         urls = []
 68 |         data = info['data']
 69 |         title = data['tvName']
 70 |         size = sum(data['clipsBytes'])
 71 |         assert len(data['clipsURL']) == len(data['clipsBytes']) \
 72 |             == len(data['su'])
 73 |         for new, clip, ck in zip(data['su'], data['clipsURL'], data['ck']):
 74 |             clipURL = urlparse(clip).path
 75 |             urls.append(real_url(host, hqvid, tvid, new, clipURL, ck))
 76 | 
 77 |     else:
 78 |         info = json.loads(get_content(
 79 |             'http://my.tv.sohu.com/play/videonew.do?vid={}&referer='
 80 |             'http://my.tv.sohu.com'.format(vid)
 81 |         ))
 82 |         host = info['allot']
 83 |         tvid = info['tvid']
 84 |         urls = []
 85 |         data = info['data']
 86 |         title = data['tvName']
 87 |         size = sum(map(int, data['clipsBytes']))
 88 |         assert len(data['clipsURL']) == len(data['clipsBytes']) \
 89 |             == len(data['su'])
 90 |         for new, clip, ck, in zip(data['su'], data['clipsURL'], data['ck']):
 91 |             clipURL = urlparse(clip).path
 92 |             urls.append(real_url(host, vid, tvid, new, clipURL, ck))
 93 | 
 94 |     print_info(site_info, title, 'mp4', size)
 95 |     if not info_only:
 96 |         download_urls(urls, title, 'mp4', size, refer=url, **kwargs)
 97 | 
 98 | 
 99 | download = sohu_download
100 | download_playlist = playlist_not_supported(site_info)
101 | 


--------------------------------------------------------------------------------
/lulu/extractors/imgur.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | 
  6 | from lulu.common import (
  7 |     match1,
  8 |     url_info,
  9 |     urls_size,
 10 |     get_content,
 11 | )
 12 | from lulu.extractor import VideoExtractor
 13 | 
 14 | 
 15 | class Imgur(VideoExtractor):
 16 |     name = 'Imgur'
 17 | 
 18 |     stream_types = [
 19 |         {'id': 'original'},
 20 |         {'id': 'thumbnail'},
 21 |     ]
 22 | 
 23 |     def prepare(self, **kwargs):
 24 |         if re.search(r'imgur\.com/a/', self.url):
 25 |             # album
 26 |             content = get_content(self.url)
 27 |             album = match1(content, r'album\s*:\s*({.*}),') or \
 28 |                 match1(content, r'image\s*:\s*({.*}),')
 29 |             album = json.loads(album)
 30 |             # count = album['album_images']['count']
 31 |             images = album['album_images']['images']
 32 |             ext = images[0]['ext']
 33 |             self.streams = {
 34 |                 'original': {
 35 |                     'src': [
 36 |                         'http://i.imgur.com/{}{}'.format(
 37 |                             i['hash'], ext
 38 |                         ) for i in images
 39 |                     ],
 40 |                     'size': sum([i['size'] for i in images]),
 41 |                     'container': ext[1:]
 42 |                 },
 43 |                 'thumbnail': {
 44 |                     'src': [
 45 |                         'http://i.imgur.com/{}s{}'.format(
 46 |                             i['hash'], '.jpg'
 47 |                         ) for i in images
 48 |                     ],
 49 |                     'container': 'jpg'
 50 |                 }
 51 |             }
 52 |             self.title = album['title']
 53 | 
 54 |         elif re.search(r'i\.imgur\.com/', self.url):
 55 |             # direct image
 56 |             _, container, size = url_info(self.url)
 57 |             self.streams = {
 58 |                 'original': {
 59 |                     'src': [self.url],
 60 |                     'size': size,
 61 |                     'container': container
 62 |                 }
 63 |             }
 64 |             self.title = match1(self.url, r'i\.imgur\.com/([^./]*)')
 65 | 
 66 |         else:
 67 |             # gallery image
 68 |             content = get_content(self.url)
 69 |             image = json.loads(match1(content, r'image\s*:\s*({.*}),'))
 70 |             ext = image['ext']
 71 |             self.streams = {
 72 |                 'original': {
 73 |                     'src': [
 74 |                         'http://i.imgur.com/{}{}'.format(image['hash'], ext)
 75 |                     ],
 76 |                     'size': image['size'],
 77 |                     'container': ext[1:]
 78 |                 },
 79 |                 'thumbnail': {
 80 |                     'src': [
 81 |                         'http://i.imgur.com/{}s{}'.format(
 82 |                             image['hash'], '.jpg'
 83 |                         )
 84 |                     ],
 85 |                     'container': 'jpg'
 86 |                 }
 87 |             }
 88 |             self.title = image['title'] or image['hash']
 89 | 
 90 |     def extract(self, **kwargs):
 91 |         if 'stream_id' in kwargs and kwargs['stream_id']:
 92 |             i = kwargs['stream_id']
 93 |             if 'size' not in self.streams[i]:
 94 |                 self.streams[i]['size'] = urls_size(self.streams[i]['src'])
 95 | 
 96 | 
 97 | site = Imgur()
 98 | download = site.download_by_url
 99 | download_playlist = site.download_by_url
100 | 


--------------------------------------------------------------------------------
/lulu/extractors/ckplayer.py:
--------------------------------------------------------------------------------
  1 | # coding=utf-8
  2 | 
  3 | from copy import copy
  4 | from xml.etree import cElementTree as ET
  5 | 
  6 | from lulu.common import (
  7 |     url_info,
  8 |     print_info,
  9 |     get_content,
 10 |     download_urls,
 11 |     playlist_not_supported,
 12 | )
 13 | 
 14 | 
 15 | __all__ = ['ckplayer_download']
 16 | site_info = 'CKPlayer General'
 17 | 
 18 | 
 19 | def ckplayer_get_info_by_xml(ckinfo):
 20 |     """str->dict
 21 |     Information for CKPlayer API content.
 22 |     """
 23 |     e = ET.XML(ckinfo)
 24 |     video_dict = {
 25 |         'title': '',
 26 |         'links': [],
 27 |         'size': 0,
 28 |         'flashvars': '',
 29 |     }
 30 |     dictified = dictify(e)['ckplayer']
 31 |     if 'info' in dictified:
 32 |         if '_text' in dictified['info'][0]['title'][0]:  # title
 33 |             video_dict['title'] = dictified['info'][0]['title'][0][
 34 |                 '_text'
 35 |             ].strip()
 36 | 
 37 |     if '_text' in dictified['video'][0]['size'][0]:  # size exists for 1 piece
 38 |         video_dict['size'] = sum(
 39 |             [int(i['size'][0]['_text']) for i in dictified['video']]
 40 |         )
 41 | 
 42 |     if '_text' in dictified['video'][0]['file'][0]:  # link exist
 43 |         video_dict['links'] = [
 44 |             i['file'][0]['_text'].strip() for i in dictified['video']
 45 |         ]
 46 | 
 47 |     if '_text' in dictified['flashvars'][0]:
 48 |         video_dict['flashvars'] = dictified['flashvars'][0]['_text'].strip()
 49 | 
 50 |     return video_dict
 51 | 
 52 | 
 53 | # helper
 54 | # https://stackoverflow.com/questions/2148119/how-to-convert-an-xml-string-to-a-dictionary-in-python  # noqa
 55 | def dictify(r, root=True):
 56 |     if root:
 57 |         return {r.tag: dictify(r, False)}
 58 |     d = copy(r.attrib)
 59 |     if r.text:
 60 |         d['_text'] = r.text
 61 |     for x in r.findall('./*'):
 62 |         if x.tag not in d:
 63 |             d[x.tag] = []
 64 |         d[x.tag].append(dictify(x, False))
 65 |     return d
 66 | 
 67 | 
 68 | def ckplayer_download_by_xml(
 69 |     ckinfo, output_dir='.', merge=False, info_only=False, **kwargs
 70 | ):
 71 |     # Info XML
 72 |     video_info = ckplayer_get_info_by_xml(ckinfo)
 73 | 
 74 |     try:
 75 |         title = kwargs['title']
 76 |     except Exception:
 77 |         title = ''
 78 |     type_ = ''
 79 |     size = 0
 80 | 
 81 |     if len(video_info['links']) > 0:  # has link
 82 |         # use 1st to determine type, ext
 83 |         type_, _ext, size = url_info(video_info['links'][0])
 84 | 
 85 |     if 'size' in video_info:
 86 |         size = int(video_info['size'])
 87 |     else:
 88 |         for i in video_info['links'][1:]:  # save 1st one
 89 |             size += url_info(i)[2]
 90 | 
 91 |     print_info(site_info, title, type_, size)
 92 |     if not info_only:
 93 |         download_urls(
 94 |             video_info['links'], title, _ext, size, output_dir=output_dir,
 95 |             merge=merge
 96 |         )
 97 | 
 98 | 
 99 | def ckplayer_download(
100 |     url, output_dir='.', merge=False, info_only=False, is_xml=True, **kwargs
101 | ):
102 |     if is_xml:  # URL is XML URL
103 |         try:
104 |             title = kwargs['title']
105 |         except Exception:
106 |             title = ''
107 |         try:
108 |             headers = kwargs['headers']  # headers provided
109 |             ckinfo = get_content(url, headers=headers)
110 |         except NameError:
111 |             ckinfo = get_content(url)
112 | 
113 |         ckplayer_download_by_xml(
114 |             ckinfo, output_dir, merge, info_only, title=title
115 |         )
116 | 
117 | 
118 | download = ckplayer_download
119 | download_playlist = playlist_not_supported(site_info)
120 | 


--------------------------------------------------------------------------------
/lulu/extractors/ixigua.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | import base64
  6 | import random
  7 | import binascii
  8 | from urllib import parse
  9 | 
 10 | from lulu.common import (
 11 |     match1,
 12 |     url_info,
 13 |     print_info,
 14 |     get_content,
 15 |     download_urls,
 16 | )
 17 | 
 18 | 
 19 | __all__ = ['ixigua_download', 'ixigua_download_playlist']
 20 | site_info = '西瓜视频 ixigua.com'
 21 | 
 22 | 
 23 | def get_r():
 24 |     return str(random.random())[2:]
 25 | 
 26 | 
 27 | def right_shift(val, n):
 28 |     return val >> n if val >= 0 else (val + 0x100000000) >> n
 29 | 
 30 | 
 31 | def get_s(text):
 32 |     """get video info
 33 |     """
 34 |     _id = match1(text, r"videoId: '(.*?)'")
 35 |     p = get_r()
 36 |     url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/{}'.format(_id)
 37 |     n = '{}?r={}'.format(parse.urlparse(url).path, p)
 38 |     c = binascii.crc32(n.encode('utf-8'))
 39 |     s = right_shift(c, 0)
 40 |     title = ''.join(re.findall(r"title: '(.*?)',", text))
 41 |     v_url = '{}?r={}&s={}'.format(url, p, s)
 42 |     return v_url, title
 43 | 
 44 | 
 45 | def get_moment(url, user_id, base_url, video_list):
 46 |     """Recursively obtaining a video list
 47 |     """
 48 |     video_list_data = json.loads(get_content(url))
 49 |     if not video_list_data['next']['max_behot_time']:
 50 |         return video_list
 51 |     [video_list.append(i['display_url']) for i in video_list_data['data']]
 52 |     max_behot_time = video_list_data['next']['max_behot_time']
 53 |     _param = {
 54 |         'user_id': user_id,
 55 |         'base_url': base_url,
 56 |         'video_list': video_list,
 57 |         'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
 58 |     }
 59 |     return get_moment(**_param)
 60 | 
 61 | 
 62 | def ixigua_download(url, output_dir='.', info_only=False, **kwargs):
 63 |     """Download a single video
 64 |     Sample URL: https://www.ixigua.com/a6487187567887254029/#mid=59051127876
 65 |     """
 66 |     if 'toutiao.com/group' in url:
 67 |         url = 'https://www.ixigua.com/a{}'.format(
 68 |             match1(url, r'.*?toutiao.com/group/(\d+)')
 69 |         )
 70 |     try:
 71 |         video_info_url, title = get_s(get_content(url))
 72 |         video_info = json.loads(get_content(video_info_url))
 73 |     except Exception:
 74 |         raise NotImplementedError(url)
 75 |     try:
 76 |         video_url = base64.b64decode(
 77 |             video_info['data']['video_list']['video_1']['main_url']
 78 |         ).decode()
 79 |     except Exception:
 80 |         raise NotImplementedError(url)
 81 |     filetype, ext, size = url_info(video_url)
 82 |     print_info(site_info, title, filetype, size)
 83 |     if not info_only:
 84 |         download_urls([video_url], title, ext, size, output_dir=output_dir)
 85 | 
 86 | 
 87 | def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs):
 88 |     """Download all video from the user's video list
 89 |     Sample URL: https://www.ixigua.com/c/user/71141690831/
 90 |     """
 91 |     if 'user' not in url:
 92 |         raise NotImplementedError(url)
 93 |     user_id = url.split('/')[-2]
 94 |     max_behot_time = 0
 95 |     if not user_id:
 96 |         raise NotImplementedError(url)
 97 |     base_url = (
 98 |         'https://www.ixigua.com/c/user/article/?user_id={user_id}&'
 99 |         'max_behot_time={max_behot_time}&max_repin_time=0&count=20&page_type=0'
100 |     )
101 |     _param = {
102 |         'user_id': user_id,
103 |         'base_url': base_url,
104 |         'video_list': [],
105 |         'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
106 |     }
107 |     for i in get_moment(**_param):
108 |         ixigua_download(i, output_dir, info_only, **kwargs)
109 | 
110 | 
111 | download = ixigua_download
112 | download_playlist = ixigua_download_playlist
113 | 


--------------------------------------------------------------------------------
/lulu/extractors/longzhu.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import json
  4 | 
  5 | from lulu.common import (
  6 |     match1,
  7 |     player,
  8 |     url_size,
  9 |     print_info,
 10 |     get_content,
 11 |     download_urls,
 12 |     general_m3u8_extractor,
 13 |     playlist_not_supported,
 14 | )
 15 | from lulu.util.parser import get_parser
 16 | 
 17 | 
 18 | __all__ = ['longzhu_download']
 19 | site_info = '龙珠直播 longzhu.com'
 20 | 
 21 | 
 22 | def longzhu_download(url, info_only=False, **kwargs):
 23 |     web_domain = url.split('/')[2]
 24 |     if (web_domain == 'star.longzhu.com') or (web_domain == 'y.longzhu.com'):
 25 |         domain = url.split('/')[3].split('?')[0]
 26 |         m_url = 'http://m.longzhu.com/{}'.format(domain)
 27 |         m_html = get_content(m_url)
 28 |         room_id_patt = r'var\s*roomId\s*=\s*(\d+);'
 29 |         room_id = match1(m_html, room_id_patt)
 30 |         json_url = (
 31 |             'http://liveapi.plu.cn/liveapp/roomstatus?roomId={}'.format(
 32 |                 room_id
 33 |             )
 34 |         )
 35 |         content = get_content(json_url)
 36 |         data = json.loads(content)
 37 |         streamUri = data['streamUri']
 38 |         if len(streamUri) <= 4:
 39 |             raise ValueError('The live stream is not online!')
 40 |         title = data['title']
 41 |         streamer = data['userName']
 42 |         title = '{}：{}'.format(streamer, title)
 43 | 
 44 |         steam_api_url = (
 45 |             'http://livestream.plu.cn/live/getlivePlayurl?roomId={}'.format(
 46 |                 room_id
 47 |             )
 48 |         )
 49 |         content = get_content(steam_api_url)
 50 |         data = json.loads(content)
 51 |         isonline = data.get('isTransfer')
 52 |         if isonline == '0':
 53 |             raise ValueError('The live stream is not online!')
 54 | 
 55 |         real_url = data['playLines'][0]['urls'][0]['securityUrl']
 56 | 
 57 |         print_info(site_info, title, 'flv', float('inf'))
 58 | 
 59 |         if not info_only:
 60 |             download_urls([real_url], title, 'flv', None, **kwargs)
 61 | 
 62 |     elif web_domain == 'replay.longzhu.com':
 63 |         videoid = match1(url, r'(\d+)$')
 64 |         json_url = (
 65 |             'http://liveapi.longzhu.com/livereplay/getreplayfordisplay?'
 66 |             'videoId={}'.format(videoid)
 67 |         )
 68 |         content = get_content(json_url)
 69 |         data = json.loads(content)
 70 | 
 71 |         username = data['userName']
 72 |         title = data['title']
 73 |         title = '{}：{}'.format(username, title)
 74 |         real_url = data['videoUrl']
 75 | 
 76 |         print_info(site_info, title, 'm3u8', 0)
 77 |         if player:
 78 |             download_urls([real_url], title, 'm3u8', 0, **kwargs)
 79 |         else:
 80 |             urls = general_m3u8_extractor(real_url)
 81 |             if not info_only:
 82 |                 download_urls(urls, title, 'ts', 0, **kwargs)
 83 | 
 84 |     elif web_domain == 'v.longzhu.com':
 85 |         page = get_content(url)
 86 |         parser = get_parser(page)
 87 |         title = parser.title.text
 88 |         media_id = match1(url, r'(\d+)$')
 89 |         # http://r.plures.net/ov/video/mobile/channel/video-1525da26174.js
 90 |         json_url = (
 91 |             'http://api.v.plu.cn/CloudMedia/GetInfoForPlayer?'
 92 |             'mediaId={}'.format(media_id)
 93 |         )
 94 |         content = get_content(json_url)
 95 |         data = json.loads(content)
 96 |         video = list(filter(lambda x: x['Ext'] == 'mp4', data['urls']))[0]
 97 |         video_url = video['SecurityUrl']
 98 |         ext = video['Ext']
 99 |         size = url_size(video_url)
100 | 
101 |         print_info(site_info, title, ext, size)
102 |         if not info_only:
103 |             download_urls([video_url], title, ext, size, **kwargs)
104 | 
105 |     else:
106 |         raise ValueError('Wrong url or unsupported link ... {}'.format(url))
107 | 
108 | 
109 | download = longzhu_download
110 | download_playlist = playlist_not_supported(site_info)
111 | 


--------------------------------------------------------------------------------
/lulu/extractors/bokecc.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import xml.etree.ElementTree as ET
  4 | 
  5 | from lulu.common import (
  6 |     url_info,
  7 |     get_content,
  8 | )
  9 | from lulu.util import log
 10 | from lulu.extractor import VideoExtractor
 11 | 
 12 | 
 13 | class BokeCC(VideoExtractor):
 14 |     name = 'BokeCC'
 15 | 
 16 |     # we do now know for now, as we have to check the output from the API
 17 |     stream_types = []
 18 | 
 19 |     API_ENDPOINT = 'http://p.bokecc.com/'
 20 | 
 21 |     def download_by_id(
 22 |         self, vid='', title=None, output_dir='.', merge=True, info_only=False,
 23 |         **kwargs
 24 |     ):
 25 |         """self, str->None
 26 | 
 27 |         Keyword arguments:
 28 |         self: self
 29 |         vid: The video ID for BokeCC cloud, something like
 30 |         FE3BB999594978049C33DC5901307461
 31 | 
 32 |         Calls the prepare() to download the video.
 33 | 
 34 |         If no title is provided, this method shall try to find a proper title
 35 |         with the information providin within the
 36 |         returned content of the API."""
 37 | 
 38 |         assert vid
 39 | 
 40 |         self.prepare(vid=vid, title=title, **kwargs)
 41 | 
 42 |         self.extract(**kwargs)
 43 | 
 44 |         self.download(
 45 |             output_dir=output_dir, merge=merge, info_only=info_only,
 46 |             **kwargs
 47 |         )
 48 | 
 49 |     def prepare(self, vid='', title=None, **kwargs):
 50 |         assert vid
 51 | 
 52 |         api_url = self.API_ENDPOINT + \
 53 |             'servlet/playinfo?vid={vid}&m=0'.format(vid=vid)  # return XML
 54 | 
 55 |         html = get_content(api_url)
 56 |         self.tree = ET.ElementTree(ET.fromstring(html))
 57 | 
 58 |         if self.tree.find('result').text != '1':
 59 |             log.wtf('API result says failed!')
 60 |             raise
 61 | 
 62 |         if title is None:
 63 |             self.title = '_'.join(
 64 |                 [i.text for i in self.tree.iterfind(
 65 |                     'video/videomarks/videomark/markdesc'
 66 |                 )]
 67 |             )
 68 |         else:
 69 |             self.title = title
 70 | 
 71 |         if not title:
 72 |             self.title = vid
 73 | 
 74 |         for i in self.tree.iterfind('video/quality'):
 75 |             quality = i.attrib['value']
 76 |             url = i[0].attrib['playurl']
 77 |             self.stream_types.append({
 78 |                 'id': quality, 'video_profile': i.attrib['desp']
 79 |             })
 80 |             self.streams[quality] = {
 81 |                 'url': url, 'video_profile': i.attrib['desp']
 82 |             }
 83 |             self.streams_sorted = [
 84 |                 dict(
 85 |                     [('id', stream_type['id'])] +
 86 |                     list(self.streams[stream_type['id']].items())
 87 |                 ) for stream_type in self.__class__.stream_types
 88 |                 if stream_type['id'] in self.streams
 89 |             ]
 90 | 
 91 |     def extract(self, **kwargs):
 92 |         for i in self.streams:
 93 |             s = self.streams[i]
 94 |             _, s['container'], s['size'] = url_info(s['url'])
 95 |             s['src'] = [s['url']]
 96 |         if 'stream_id' in kwargs and kwargs['stream_id']:
 97 |             # Extract the stream
 98 |             stream_id = kwargs['stream_id']
 99 | 
100 |             if stream_id not in self.streams:
101 |                 log.e('[Error] Invalid video format.')
102 |                 log.e(
103 |                     'Run \'-i\' command with no specific video format to '
104 |                     'view all available formats.'
105 |                 )
106 |                 exit(2)
107 |         else:
108 |             # Extract stream with the best quality
109 |             stream_id = self.streams_sorted[0]['id']
110 |             _, s['container'], s['size'] = url_info(s['url'])
111 |             s['src'] = [s['url']]
112 | 
113 | 
114 | site = BokeCC()
115 | 
116 | # I don't know how to call the player directly so I just put it here
117 | # just in case anyone touchs it -- Beining@Aug.24.2016
118 | # download = site.download_by_url
119 | # download_playlist = site.download_by_url
120 | 
121 | bokecc_download_by_id = site.download_by_id
122 | 


--------------------------------------------------------------------------------
/lulu/extractors/qie.py:
--------------------------------------------------------------------------------
  1 | # coding=utf-8
  2 | 
  3 | import re
  4 | import json
  5 | 
  6 | from lulu.common import (
  7 |     match1,
  8 |     get_content,
  9 |     playlist_not_supported,
 10 | )
 11 | from lulu.util import log
 12 | from lulu.extractor import VideoExtractor
 13 | 
 14 | 
 15 | site_info = '企鹅直播 live.qq.com'
 16 | 
 17 | 
 18 | class QiE(VideoExtractor):
 19 |     name = site_info
 20 | 
 21 |     # Last updated: 2015-11-24
 22 |     stream_types = [
 23 |         {'id': 'normal', 'container': 'flv', 'video_profile': '标清'},
 24 |         {'id': 'middle', 'container': 'flv', 'video_profile': '550'},
 25 |         {'id': 'middle2', 'container': 'flv', 'video_profile': '900'},
 26 |     ]
 27 | 
 28 |     id_dic = {i['video_profile']: (i['id']) for i in stream_types}
 29 | 
 30 |     api_endpoint = 'http://www.qie.tv/api/v1/room/{room_id}'
 31 |     game_ep = 'http://live.qq.com/game/game_details/get_game_details_info/'
 32 | 
 33 |     def get_room_id_from_url(self, match_id):
 34 |         meta = json.loads(get_content(self.game_ep + str(match_id)))
 35 |         if meta['error'] != 0:
 36 |             log.wtf('Error happens when accessing game_details api')
 37 |         rooms = meta['data']['anchor_data']
 38 |         for room in rooms:
 39 |             if room['is_use_room']:
 40 |                 return room['room_id']
 41 |         log.wtf('No room available for match {}'.format(match_id))
 42 | 
 43 |     def get_vid_from_url(self, url):
 44 |         """Extracts video ID from live.qq.com.
 45 |         """
 46 |         hit = re.search(r'live.qq.com/(\d+)', url)
 47 |         if hit is not None:
 48 |             return hit.group(1)
 49 |         hit = re.search(r'live.qq.com/directory/match/(\d+)', url)
 50 |         if hit is not None:
 51 |             return self.get_room_id_from_url(hit.group(1))
 52 |         html = get_content(url)
 53 |         room_id = match1(html, r'room_id\":(\d+)')
 54 |         if room_id is None:
 55 |             log.wtf('Unknown page {}'.format(url))
 56 |         return room_id
 57 | 
 58 |     def download_playlist_by_url(self, url, **kwargs):
 59 |         pass
 60 | 
 61 |     def prepare(self, **kwargs):
 62 |         if self.url:
 63 |             self.vid = self.get_vid_from_url(self.url)
 64 |         self.referer = self.url
 65 |         content = get_content(self.api_endpoint.format(room_id=self.vid))
 66 |         content = json.loads(content)
 67 |         self.title = content['data']['room_name']
 68 |         rtmp_url = content['data']['rtmp_url']
 69 |         # stream_avalable = [i['name'] for i in content['data']['stream']]
 70 |         stream_available = {}
 71 |         stream_available['normal'] = '{}/{}'.format(
 72 |             rtmp_url, content['data']['rtmp_live']
 73 |         )
 74 |         if len(content['data']['rtmp_multi_bitrate']) > 0:
 75 |             for k, v in content['data']['rtmp_multi_bitrate'].items():
 76 |                 stream_available[k] = rtmp_url + '/' + v
 77 | 
 78 |         for s in self.stream_types:
 79 |             if s['id'] in stream_available.keys():
 80 |                 quality_id = s['id']
 81 |                 url = stream_available[quality_id]
 82 |                 self.streams[quality_id] = {
 83 |                     'container': 'flv',
 84 |                     'video_profile': s['video_profile'],
 85 |                     'size': 0,
 86 |                     'url': url
 87 |                 }
 88 | 
 89 |     def extract(self, **kwargs):
 90 |         for i in self.streams:
 91 |             s = self.streams[i]
 92 |             s['src'] = [s['url']]
 93 |         if 'stream_id' in kwargs and kwargs['stream_id']:
 94 |             # Extract the stream
 95 |             stream_id = kwargs['stream_id']
 96 | 
 97 |             if stream_id not in self.streams:
 98 |                 log.e('[Error] Invalid video format.')
 99 |                 log.e(
100 |                     'Run \'-i\' command with no specific video format to '
101 |                     'view all available formats.'
102 |                 )
103 |                 exit(2)
104 |         else:
105 |             # Extract stream with the best quality
106 |             stream_id = self.streams_sorted[0]['id']
107 |             s['src'] = [s['url']]
108 | 
109 | 
110 | site = QiE()
111 | download = site.download_by_url
112 | download_playlist = playlist_not_supported(site_info)
113 | 


--------------------------------------------------------------------------------
/lulu/extractors/wanmen.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import json
  4 | 
  5 | from lulu.common import (
  6 |     match1,
  7 |     get_content,
  8 | )
  9 | from lulu.util import log
 10 | from lulu.extractors.bokecc import bokecc_download_by_id
 11 | 
 12 | 
 13 | __all__ = [
 14 |     'wanmen_download', 'wanmen_download_by_course',
 15 |     'wanmen_download_by_course_topic',
 16 |     'wanmen_download_by_course_topic_part'
 17 | ]
 18 | site_info = '万门 wanmen.org'
 19 | 
 20 | 
 21 | # Helper functions
 22 | def _wanmen_get_json_api_content_by_courseID(courseID):
 23 |     """int->JSON
 24 | 
 25 |     Return a parsed JSON tree of WanMen's API.
 26 |     """
 27 | 
 28 |     return json.loads(get_content(
 29 |         'http://api.wanmen.org/course/getCourseNested/{courseID}'.format(
 30 |             courseID=courseID)
 31 |         ))
 32 | 
 33 | 
 34 | def _wanmen_get_title_by_json_topic_part(json_content, tIndex, pIndex):
 35 |     """JSON, int, int, int->str
 36 | 
 37 |     Get a proper title with courseid+topicID+partID.
 38 |     """
 39 | 
 40 |     return '_'.join([
 41 |         json_content[0]['name'],
 42 |         json_content[0]['Topics'][tIndex]['name'],
 43 |         json_content[0]['Topics'][tIndex]['Parts'][pIndex]['name']
 44 |     ])
 45 | 
 46 | 
 47 | def _wanmen_get_boke_id_by_json_topic_part(json_content, tIndex, pIndex):
 48 |     """JSON, int, int, int->str
 49 | 
 50 |     Get one BokeCC video ID with courseid+topicID+partID."""
 51 | 
 52 |     return json_content[0]['Topics'][tIndex]['Parts'][pIndex]['ccVideoLink']
 53 | 
 54 | 
 55 | # Parsers
 56 | def wanmen_download_by_course(json_api_content, info_only=False, **kwargs):
 57 |     """int->None
 58 | 
 59 |     Download a WHOLE course.
 60 |     Reuse the API call to save time.
 61 |     """
 62 | 
 63 |     for tIndex in range(len(json_api_content[0]['Topics'])):
 64 |         for pIndex in range(
 65 |             len(json_api_content[0]['Topics'][tIndex]['Parts'])
 66 |         ):
 67 |             wanmen_download_by_course_topic_part(
 68 |                 json_api_content, tIndex, pIndex, info_only=info_only,
 69 |                 **kwargs
 70 |             )
 71 | 
 72 | 
 73 | def wanmen_download_by_course_topic(
 74 |     json_api_content, tIndex, info_only=False, **kwargs
 75 | ):
 76 |     """int, int->None
 77 | 
 78 |     Download a TOPIC of a course.
 79 |     Reuse the API call to save time.
 80 |     """
 81 | 
 82 |     for pIndex in range(len(json_api_content[0]['Topics'][tIndex]['Parts'])):
 83 |         wanmen_download_by_course_topic_part(
 84 |             json_api_content, tIndex, pIndex, info_only=info_only, **kwargs
 85 |         )
 86 | 
 87 | 
 88 | def wanmen_download_by_course_topic_part(
 89 |     json_api_content, tIndex, pIndex, info_only=False, **kwargs
 90 | ):
 91 |     """int, int, int->None
 92 | 
 93 |     Download ONE PART of the course.
 94 |     """
 95 | 
 96 |     html = json_api_content
 97 | 
 98 |     title = _wanmen_get_title_by_json_topic_part(
 99 |         html, tIndex, pIndex
100 |     )
101 |     bokeccID = _wanmen_get_boke_id_by_json_topic_part(
102 |         html, tIndex, pIndex
103 |     )
104 |     bokecc_download_by_id(
105 |         vid=bokeccID, title=title, info_only=info_only, **kwargs
106 |     )
107 | 
108 | 
109 | # Main entrance
110 | def wanmen_download(url, info_only=False, **kwargs):
111 | 
112 |     if 'wanmen.org' not in url:
113 |         log.wtf(
114 |             'You are at the wrong place dude. This is for WanMen University!'
115 |         )
116 | 
117 |     courseID = int(match1(url, r'course\/(\d+)'))
118 |     assert courseID > 0  # without courseID we cannot do anything
119 | 
120 |     tIndex = int(match1(url, r'tIndex=(\d+)'))
121 | 
122 |     pIndex = int(match1(url, r'pIndex=(\d+)'))
123 | 
124 |     json_api_content = _wanmen_get_json_api_content_by_courseID(courseID)
125 | 
126 |     if pIndex:  # only download ONE single part
127 |         assert tIndex >= 0
128 |         wanmen_download_by_course_topic_part(
129 |             json_api_content, tIndex, pIndex, info_only=info_only, **kwargs
130 |         )
131 |     elif tIndex:  # download a topic
132 |         wanmen_download_by_course_topic(
133 |             json_api_content, tIndex, info_only=info_only, **kwargs
134 |         )
135 |     else:  # download the whole course
136 |         wanmen_download_by_course(
137 |             json_api_content, info_only=info_only, **kwargs
138 |         )
139 | 
140 | 
141 | download = wanmen_download
142 | download_playlist = wanmen_download_by_course
143 | 


--------------------------------------------------------------------------------
/lulu/extractors/twitter.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | from urllib import parse
  6 | from html import unescape
  7 | 
  8 | from lulu.common import (
  9 |     match1,
 10 |     get_head,
 11 |     urls_size,
 12 |     print_info,
 13 |     get_content,
 14 |     download_urls,
 15 |     playlist_not_supported,
 16 | )
 17 | from lulu.extractors.vine import vine_download
 18 | 
 19 | 
 20 | __all__ = ['twitter_download']
 21 | site_info = 'Twitter twitter.com'
 22 | 
 23 | 
 24 | def extract_m3u(source):
 25 |     r1 = get_content(source)
 26 |     s1 = re.findall(r'(/ext_tw_video/.*)', r1)
 27 |     s1 += re.findall(r'(/amplify_video/.*)', r1)
 28 |     r2 = get_content('https://video.twimg.com{}'.format(s1[-1]))
 29 |     s2 = re.findall(r'(/ext_tw_video/.*)', r2)
 30 |     s2 += re.findall(r'(/amplify_video/.*)', r2)
 31 |     return ['https://video.twimg.com{}'.format(i) for i in s2]
 32 | 
 33 | 
 34 | def twitter_download(url, info_only=False, **kwargs):
 35 |     html = get_content(url)
 36 |     screen_name = match1(html, r'data-screen-name="([^"]*)"') or \
 37 |         match1(html, r'<meta name="twitter:title" content="([^"]*)"')
 38 |     item_id = match1(html, r'data-item-id="([^"]*)"') or \
 39 |         match1(html, r'<meta name="twitter:site:id" content="([^"]*)"')
 40 |     page_title = '{} [{}]'.format(screen_name, item_id)
 41 | 
 42 |     try:  # extract images
 43 |         urls = re.findall(
 44 |             r'property="og:image"\s*content="([^"]+:large)"', html
 45 |         )
 46 |         assert urls
 47 |         images = []
 48 |         for url in urls:
 49 |             url = ':'.join(url.split(':')[:-1]) + ':orig'
 50 |             filename = parse.unquote(url.split('/')[-1])
 51 |             title = '.'.join(filename.split('.')[:-1])
 52 |             ext = url.split(':')[-2].split('.')[-1]
 53 |             size = int(get_head(url)['Content-Length'])
 54 |             images.append({
 55 |                 'title': title,
 56 |                 'url': url,
 57 |                 'ext': ext,
 58 |                 'size': size
 59 |             })
 60 |         size = sum([image['size'] for image in images])
 61 |         print_info(site_info, page_title, images[0]['ext'], size)
 62 | 
 63 |         if not info_only:
 64 |             for image in images:
 65 |                 title = image['title']
 66 |                 ext = image['ext']
 67 |                 size = image['size']
 68 |                 url = image['url']
 69 |                 print_info(site_info, title, ext, size)
 70 |                 download_urls([url], title, ext, size, **kwargs)
 71 | 
 72 |     except Exception:  # extract video
 73 |         # always use i/cards or videos url
 74 |         if not re.match(r'https?://twitter.com/i/', url):
 75 |             url = match1(
 76 |                 html, r'<meta\s*property="og:video:url"\s*content="([^"]+)"'
 77 |             )
 78 |             if not url:
 79 |                 url = 'https://twitter.com/i/videos/{}'.format(item_id)
 80 |             html = get_content(url)
 81 | 
 82 |         data_config = match1(html, r'data-config="([^"]*)"') or \
 83 |             match1(html, r'data-player-config="([^"]*)"')
 84 |         i = json.loads(unescape(data_config))
 85 |         if 'video_url' in i:
 86 |             source = i['video_url']
 87 |             item_id = i['tweet_id']
 88 |             page_title = "{} [{}]".format(screen_name, item_id)
 89 |         elif 'playlist' in i:
 90 |             source = i['playlist'][0]['source']
 91 |             if not item_id:
 92 |                 page_title = i['playlist'][0]['contentId']
 93 |         elif 'vmap_url' in i:
 94 |             vmap_url = i['vmap_url']
 95 |             vmap = get_content(vmap_url)
 96 |             source = match1(vmap, r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>')
 97 |             item_id = i['tweet_id']
 98 |             page_title = '{} [{}]'.format(screen_name, item_id)
 99 |         elif 'scribe_playlist_url' in i:
100 |             scribe_playlist_url = i['scribe_playlist_url']
101 |             return vine_download(
102 |                 scribe_playlist_url, info_only=info_only, **kwargs
103 |             )
104 | 
105 |         try:
106 |             urls = extract_m3u(source)
107 |         except Exception:
108 |             urls = [source]
109 |         size = urls_size(urls)
110 |         mime, ext = 'video/mp4', 'mp4'
111 | 
112 |         print_info(site_info, page_title, mime, size)
113 |         if not info_only:
114 |             download_urls(urls, page_title, ext, size, **kwargs)
115 | 
116 | 
117 | download = twitter_download
118 | download_playlist = playlist_not_supported(site_info)
119 | 


--------------------------------------------------------------------------------
/lulu/extractors/ximalaya.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | import json
  5 | 
  6 | from lulu.common import (
  7 |     match1,
  8 |     url_info,
  9 |     print_info,
 10 |     get_content,
 11 |     download_urls,
 12 |     playlist_not_supported,
 13 | )
 14 | 
 15 | 
 16 | __all__ = [
 17 |     'ximalaya_download_playlist', 'ximalaya_download',
 18 |     'ximalaya_download_by_id'
 19 | ]
 20 | site_info = '喜马拉雅FM ximalaya.com'
 21 | 
 22 | 
 23 | stream_types = [
 24 |     {'itag': '1', 'container': 'm4a', 'bitrate': 'default'},
 25 |     {'itag': '2', 'container': 'm4a', 'bitrate': '32'},
 26 |     {'itag': '3', 'container': 'm4a', 'bitrate': '64'},
 27 | ]
 28 | 
 29 | 
 30 | def ximalaya_download_by_id(
 31 |     _id, title=None, info_only=False, stream_id=None, **kwargs
 32 | ):
 33 |     BASE_URL = 'http://www.ximalaya.com/tracks/'
 34 |     json_url = '{}{}.json'.format(BASE_URL, _id)
 35 |     json_data = json.loads(get_content(json_url))
 36 |     if 'res' in json_data:
 37 |         if json_data['res'] is False:
 38 |             raise ValueError('Server reported id {} is invalid'.format(_id))
 39 |     if 'is_paid' in json_data and json_data['is_paid']:
 40 |         if 'is_free' in json_data and not json_data['is_free']:
 41 |             raise ValueError('{} is paid item'.format(_id))
 42 |     if (not title) and 'title' in json_data:
 43 |         title = json_data['title']
 44 |     # no size data in the json. should it be calculated?
 45 |     size = 0
 46 |     url = json_data['play_path_64']
 47 |     if stream_id:
 48 |         if stream_id == '1':
 49 |             url = json_data['play_path_32']
 50 |         elif stream_id == '0':
 51 |             url = json_data['play_path']
 52 |     ext = 'm4a'
 53 |     urls = [url]
 54 |     print('Site:        %s' % site_info)
 55 |     print('title:       %s' % title)
 56 |     if info_only:
 57 |         if stream_id:
 58 |             print_stream_info(stream_id)
 59 |         else:
 60 |             for item in range(0, len(stream_types)):
 61 |                 print_stream_info(item)
 62 |     if not info_only:
 63 |         print('Type:        MPEG-4 audio m4a')
 64 |         print('Size:        N/A')
 65 |         download_urls(urls, title, ext, size, **kwargs)
 66 | 
 67 | 
 68 | def ximalaya_download(url, info_only=False, stream_id=None, **kwargs):
 69 |     if re.match(r'http://www\.ximalaya\.com/(\d+)/sound/(\d+)', url):
 70 |         _id = match1(url, r'http://www\.ximalaya\.com/\d+/sound/(\d+)')
 71 |     else:
 72 |         raise NotImplementedError(url)
 73 |     ximalaya_download_by_id(
 74 |         _id, info_only=info_only, stream_id=stream_id, **kwargs
 75 |     )
 76 | 
 77 | 
 78 | def ximalaya_download_page(
 79 |     playlist_url, info_only=False, stream_id=None, **kwargs
 80 | ):
 81 |     if re.match(r'http://www\.ximalaya\.com/(\d+)/album/(\d+)', playlist_url):
 82 |         page_content = get_content(playlist_url)
 83 |         pattern = re.compile(r'<li sound_id="(\d+)"')
 84 |         ids = pattern.findall(page_content)
 85 |         for _id in ids:
 86 |             try:
 87 |                 ximalaya_download_by_id(
 88 |                     _id, info_only=info_only, stream_id=stream_id, **kwargs
 89 |                 )
 90 |             except(ValueError):
 91 |                 print(
 92 |                     'Something wrong with {}, perhaps paid item?'.format(_id)
 93 |                 )
 94 |     else:
 95 |         raise NotImplementedError(playlist_url)
 96 | 
 97 | 
 98 | def ximalaya_download_playlist(url, info_only=False, stream_id=None, **kwargs):
 99 |     match_result = re.match(
100 |         r'http://www\.ximalaya\.com/(\d+)/album/(\d+)', url
101 |     )
102 |     if not match_result:
103 |         raise NotImplementedError(url)
104 |     pages = []
105 |     page_content = get_content(url)
106 |     if page_content.find('<div class="pagingBar_wrapper"') == -1:
107 |         pages.append(url)
108 |     else:
109 |         base_url = 'http://www.ximalaya.com/{}/album/{}'.format(
110 |             match_result.group(1), match_result.group(2)
111 |         )
112 |         html_str = '<a href=(\'|")\/{}\/album\/{}\?page='.format(
113 |             match_result.group(1), match_result.group(2)
114 |         )
115 |         count = len(re.findall(html_str, page_content))
116 |         for page_num in range(count):
117 |             pages.append('{}?page={}'.format(base_url, str(page_num+1)))
118 |             print(pages[-1])
119 |     for page in pages:
120 |         ximalaya_download_page(
121 |             page, info_only=info_only, stream_id=stream_id, **kwargs
122 |         )
123 | 
124 | 
125 | def print_stream_info(stream_id):
126 |     print('    - itag:        %s' % stream_id)
127 |     print('      container:   %s' % 'm4a')
128 |     print('      bitrate:     %s' % stream_types[int(stream_id)]['bitrate'])
129 |     print('      size:        %s' % 'N/A')
130 |     print('    # download-with: lulu --itag=%s [URL]' % stream_id)
131 | 
132 | 
133 | download = ximalaya_download
134 | download_playlist = ximalaya_download_playlist
135 | 


--------------------------------------------------------------------------------
/lulu/extractors/coub.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import os
  4 | import re
  5 | import json
  6 | 
  7 | from lulu.common import (
  8 |     match1,
  9 |     url_info,
 10 |     print_info,
 11 |     get_content,
 12 |     download_urls,
 13 |     get_output_filename,
 14 |     playlist_not_supported,
 15 | )
 16 | from lulu.processor import ffmpeg
 17 | from lulu.util.fs import legitimize
 18 | 
 19 | 
 20 | __all__ = ['coub_download']
 21 | site_info = 'Coub coub.com'
 22 | 
 23 | 
 24 | def coub_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 25 |     html = get_content(url)
 26 | 
 27 |     try:
 28 |         json_data = get_coub_data(html)
 29 |         title, video_url, audio_url = get_title_and_urls(json_data)
 30 |         video_file_name, video_file_path = get_file_path(
 31 |             merge, output_dir, title, video_url
 32 |         )
 33 |         audio_file_name, audio_file_path = get_file_path(
 34 |             merge, output_dir, title, audio_url
 35 |         )
 36 |         download_url(audio_url, merge, output_dir, title, info_only)
 37 |         download_url(video_url, merge, output_dir, title, info_only)
 38 |         if not info_only:
 39 |             try:
 40 |                 fix_coub_video_file(video_file_path)
 41 |                 audio_duration = float(
 42 |                     ffmpeg.ffprobe_get_media_duration(audio_file_path)
 43 |                 )
 44 |                 video_duration = float(
 45 |                     ffmpeg.ffprobe_get_media_duration(video_file_path)
 46 |                 )
 47 |                 loop_file_path = get_loop_file_path(title, output_dir)
 48 |                 single_file_path = audio_file_path
 49 |                 if audio_duration > video_duration:
 50 |                     write_loop_file(
 51 |                         int(audio_duration / video_duration), loop_file_path,
 52 |                         video_file_name
 53 |                     )
 54 |                 else:
 55 |                     single_file_path = video_file_path
 56 |                     write_loop_file(
 57 |                         int(video_duration / audio_duration), loop_file_path,
 58 |                         audio_file_name
 59 |                     )
 60 |                 ffmpeg.ffmpeg_concat_audio_and_video(
 61 |                     [loop_file_path, single_file_path], title + '_full', 'mp4'
 62 |                 )
 63 |                 cleanup_files(
 64 |                     [video_file_path, audio_file_path, loop_file_path]
 65 |                 )
 66 |             except EnvironmentError as err:
 67 |                 print('Error preparing full coub video. {}'.format(err))
 68 |     except Exception as err:
 69 |         print('Error while downloading files. {}'.format(err))
 70 | 
 71 | 
 72 | def write_loop_file(records_number, loop_file_path, file_name):
 73 |     with open(loop_file_path, 'a') as file:
 74 |         for i in range(records_number):
 75 |             file.write("file '{}'\n".format(file_name))
 76 | 
 77 | 
 78 | def download_url(url, merge, output_dir, title, info_only):
 79 |     mime, ext, size = url_info(url)
 80 |     print_info(site_info, title, mime, size)
 81 |     if not info_only:
 82 |         download_urls([url], title, ext, size, output_dir, merge=merge)
 83 | 
 84 | 
 85 | def fix_coub_video_file(file_path):
 86 |     with open(file_path, 'r+b') as file:
 87 |         file.seek(0)
 88 |         file.write(bytes(2))
 89 | 
 90 | 
 91 | def get_title_and_urls(json_data):
 92 |     title = legitimize(re.sub('[\s*]', '_', json_data['title']))
 93 |     video_info = json_data['file_versions']['html5']['video']
 94 |     if 'high' not in video_info:
 95 |         if 'med' not in video_info:
 96 |             video_url = video_info['low']['url']
 97 |         else:
 98 |             video_url = video_info['med']['url']
 99 |     else:
100 |         video_url = video_info['high']['url']
101 |     audio_info = json_data['file_versions']['html5']['audio']
102 |     if 'high' not in audio_info:
103 |         if 'med' not in audio_info:
104 |             audio_url = audio_info['low']['url']
105 |         else:
106 |             audio_url = audio_info['med']['url']
107 |     else:
108 |         audio_url = audio_info['high']['url']
109 |     return title, video_url, audio_url
110 | 
111 | 
112 | def get_coub_data(html):
113 |     coub_data = match1(
114 |         html,
115 |         r'<script id=\'coubPageCoubJson\' type=\'text/json\'>([^<]+)</script>'
116 |     )
117 |     json_data = json.loads(coub_data)
118 |     return json_data
119 | 
120 | 
121 | def get_file_path(merge, output_dir, title, url):
122 |     mime, ext, size = url_info(url)
123 |     file_name = get_output_filename([], title, ext, output_dir, merge)
124 |     file_path = os.path.join(output_dir, file_name)
125 |     return file_name, file_path
126 | 
127 | 
128 | def get_loop_file_path(title, output_dir):
129 |     return os.path.join(
130 |         output_dir, get_output_filename([], title, 'txt', None, False)
131 |     )
132 | 
133 | 
134 | def cleanup_files(files):
135 |     for file in files:
136 |         os.remove(file)
137 | 
138 | 
139 | download = coub_download
140 | download_playlist = playlist_not_supported(site_info)
141 | 


--------------------------------------------------------------------------------
/lulu/extractors/google.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import re
  4 | from html import unescape
  5 | from urllib import (
  6 |     parse,
  7 |     request,
  8 | )
  9 | 
 10 | from lulu.common import (
 11 |     match1,
 12 |     url_info,
 13 |     unicodize,
 14 |     print_info,
 15 |     get_content,
 16 |     get_location,
 17 |     download_urls,
 18 |     playlist_not_supported,
 19 | )
 20 | from lulu.config import YOUTUBE_CODECS
 21 | 
 22 | 
 23 | __all__ = ['google_download']
 24 | site_info = 'Google google.com'
 25 | 
 26 | 
 27 | fmt_level = dict(zip(
 28 |     [str(codec['itag']) for codec in YOUTUBE_CODECS],
 29 |     range(len(YOUTUBE_CODECS))
 30 | ))
 31 | 
 32 | 
 33 | def google_download(url, info_only=False, **kwargs):
 34 |     # Percent-encoding Unicode URL
 35 |     url = parse.quote(url, safe=':/+%?=')
 36 | 
 37 |     service = url.split('/')[2].split('.')[0]
 38 | 
 39 |     if service == 'plus':  # Google Plus
 40 | 
 41 |         # attempt to extract images first
 42 |         # TBD: posts with > 4 images
 43 |         # TBD: album links
 44 |         html = get_content(parse.unquote(url))
 45 |         real_urls = []
 46 |         for src in re.findall(r'src="([^"]+)"[^>]*itemprop="image"', html):
 47 |             t = src.split('/')
 48 |             t[0], t[-2] = t[0] or 'https:', 's0-d'
 49 |             u = '/'.join(t)
 50 |             real_urls.append(u)
 51 |         if not real_urls:
 52 |             real_urls = [
 53 |                 match1(html, r'<meta property="og:image" content="([^"]+)')
 54 |             ]
 55 |             real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
 56 |         post_date = match1(html, r'"?(20\d\d[-/]?[01]\d[-/]?[0123]\d)"?')
 57 |         post_id = match1(html, r'/posts/([^"]+)')
 58 |         title = '{}_{}'.format(post_date, post_id)
 59 | 
 60 |         try:
 61 |             url = 'https://plus.google.com/{}'.format(
 62 |                 match1(html, r'(photos/\d+/albums/\d+/\d+)\?authkey')
 63 |             )
 64 |             html = get_content(url)
 65 |             temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
 66 |             temp = sorted(temp, key=lambda x: fmt_level[x[0]])
 67 |             urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
 68 |             assert urls
 69 |             real_urls = urls  # Look ma, there's really a video!
 70 | 
 71 |             post_url = match1(
 72 |                 html, r'"(https://plus.google.com/[^/]+/posts/[^"]*)"'
 73 |             )
 74 |             post_author = match1(post_url, r'/\+([^/]+)/posts')
 75 |             if post_author:
 76 |                 post_url = 'https://plus.google.com/+{}/posts/{}'.format(
 77 |                     parse.quote(post_author), match1(post_url, r'posts/(.+)')
 78 |                 )
 79 |             post_html = get_content(post_url)
 80 |             title = match1(post_html, r'<title[^>]*>([^<\n]+)')
 81 |         except Exception:
 82 |             pass
 83 | 
 84 |         for i, real_url in enumerate(real_urls):
 85 |             title_i = '{}[{}]'.format(title, i) if len(real_urls) > 1 \
 86 |                 else title
 87 |             _, ext, size = url_info(real_url)
 88 |             if ext is None:
 89 |                 ext = 'mp4'
 90 | 
 91 |             print_info(site_info, title_i, ext, size)
 92 |             if not info_only:
 93 |                 download_urls([real_url], title_i, ext, size, **kwargs)
 94 | 
 95 |     elif service in ['docs', 'drive']:  # Google Docs
 96 |         html = get_content(url)
 97 | 
 98 |         title = match1(html, r'"title":"([^"]*)"') or match1(
 99 |             html, r'<meta itemprop="name" content="([^"]*)"'
100 |         )
101 |         if len(title.split('.')) > 1:
102 |             title = ".".join(title.split('.')[:-1])
103 | 
104 |         docid = match1(url, '/file/d/([^/]+)')
105 | 
106 |         request.install_opener(
107 |             request.build_opener(request.HTTPCookieProcessor())
108 |         )
109 | 
110 |         real_url = (
111 |             'https://docs.google.com/uc?export=download&confirm=no_antivirus&'
112 |             'id={}'.format(docid)
113 |         )
114 |         redirected_url = get_location(real_url)
115 |         if real_url != redirected_url:
116 |             # tiny file - get real url here
117 |             _, ext, size = url_info(redirected_url)
118 |             real_url = redirected_url
119 |         else:
120 |             # huge file - the real_url is a confirm page and real url is in it
121 |             confirm_page = get_content(real_url)
122 |             hrefs = re.findall(r'href="(.+?)"', confirm_page)
123 |             for u in hrefs:
124 |                 if u.startswith('/uc?export=download'):
125 |                     rel = unescape(u)
126 |             confirm_url = 'https://docs.google.com' + rel
127 |             real_url = get_location(confirm_url)
128 |             _, ext, size = url_info(real_url)
129 |             if size is None:
130 |                 size = 0
131 | 
132 |         print_info(site_info, title, ext, size)
133 |         if not info_only:
134 |             download_urls([real_url], title, ext, size, **kwargs)
135 | 
136 | 
137 | download = google_download
138 | download_playlist = playlist_not_supported(site_info)
139 | 


--------------------------------------------------------------------------------