├── .editorconfig ├── .github └── ISSUE_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── README.rst ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── src └── m3u8_dl │ ├── M3u8Downloader.py │ ├── __init__.py │ ├── cli.py │ ├── faker.py │ ├── myprint.py │ └── restore.py ├── tests ├── __init__.py ├── test_context.py └── test_m3u8_dl.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * m3u8_dl version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | # python project stuff 105 | .DS_Store 106 | __pycache__ 107 | deps 108 | docs 109 | .tags* 110 | .python-version 111 | .README.md 112 | .pypirc 113 | *.ts 114 | *.restore 115 | *.m3u8 116 | script.sh 117 | 118 | *.mp4 119 | *.json 120 | auto.py 121 | .idea 122 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | python: 5 | - 3.6 6 | - 3.5 7 | - 3.4 8 | - 2.7 9 | 10 | # Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 11 | install: pip install -U tox-travis 12 | 13 | # Command to run tests, e.g. python setup.py test 14 | script: tox 15 | 16 | # Assuming you have installed the travis-ci CLI tool, after you 17 | # create the Github repo and add it to Travis, run the 18 | # following command to finish PyPI deployment setup: 19 | # $ travis encrypt --add deploy.password 20 | deploy: 21 | provider: pypi 22 | distributions: sdist bdist_wheel 23 | user: kedpter 24 | password: 25 | secure: PLEASE_REPLACE_ME 26 | on: 27 | tags: true 28 | repo: kedpter/m3u8_dl 29 | python: 3.6 30 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * kedpter <790476448@qq.com> 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/kedpter/m3u8_dl/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" and "help 30 | wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | m3u8_dl could always use more documentation, whether as part of the 42 | official m3u8_dl docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/kedpter/m3u8_dl/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `m3u8_dl` for local development. 61 | 62 | 1. Fork the `m3u8_dl` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/m3u8_dl.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv m3u8_dl 70 | $ cd m3u8_dl/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass flake8 and the 80 | tests, including testing other Python versions with tox:: 81 | 82 | $ flake8 m3u8_dl tests 83 | $ python setup.py test or py.test 84 | $ tox 85 | 86 | To get flake8 and tox, just pip install them into your virtualenv. 87 | 88 | 6. Commit your changes and push your branch to GitHub:: 89 | 90 | $ git add . 91 | $ git commit -m "Your detailed description of your changes." 92 | $ git push origin name-of-your-bugfix-or-feature 93 | 94 | 7. Submit a pull request through the GitHub website. 95 | 96 | Pull Request Guidelines 97 | ----------------------- 98 | 99 | Before you submit a pull request, check that it meets these guidelines: 100 | 101 | 1. The pull request should include tests. 102 | 2. If the pull request adds functionality, the docs should be updated. Put 103 | your new functionality into a function with a docstring, and add the 104 | feature to the list in README.rst. 105 | 3. The pull request should work for Python 2.7, 3.4, 3.5 and 3.6, and for PyPy. Check 106 | https://travis-ci.org/kedpter/m3u8_dl/pull_requests 107 | and make sure that the tests pass for all supported Python versions. 108 | 109 | Tips 110 | ---- 111 | 112 | To run a subset of tests:: 113 | 114 | 115 | $ python -m unittest tests.test_m3u8_dl 116 | 117 | Deploying 118 | --------- 119 | 120 | A reminder for the maintainers on how to deploy. 121 | Make sure all your changes are committed (including an entry in HISTORY.rst). 122 | Then run:: 123 | 124 | $ bumpversion patch # possible: major / minor / patch 125 | $ git push 126 | $ git push --tags 127 | 128 | Travis will then deploy to PyPI if tests pass. 129 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 0.1.0 (2019-03-13) 6 | ------------------ 7 | 8 | * First release on PyPI. 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019, kedpter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: pypi-upload 2 | 3 | pypi-upload: 4 | rm -rf dist/* 5 | python setup.py sdist bdist_wheel 6 | twine upload dist/* 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### M3u8-dl 2 | 3 | ![alt text](https://img.shields.io/pypi/v/m3u8_dl.svg) 4 | ![alt text](https://img.shields.io/travis/kedpter/m3u8_dl.svg) 5 | ![alt text](https://readthedocs.org/projects/m3u8_dl/badge/?version=latest) 6 | 7 | M3u8-dl is a simple command-line util which downloads m3u8 file. 8 | 9 | 10 | ### Install 11 | 12 | ```bash 13 | pip install m3u8-dl 14 | ``` 15 | 16 | ### Usage 17 | 18 | Get the HLS Request infomation from web browser with `Developer Tools`. 19 | Such As `Request URL` and `Referer`. 20 | 21 | ```bash 22 | # HLS_URL -> Request URL 23 | # OUTPUT -> such as example.ts 24 | m3u8-dl HLS_URL OUTPUT 25 | # code below may not work since the website server may reject an out-of-date request 26 | m3u8-dl https://proxy-038.dc3.dailymotion.com/sec\(4pkX4jyJ09RyW9jaEyekktbBu55uix9cMXQu-o5e13EelVKd1csb9zYSD66hQl7PlA_V5ntIHivm_tuQqkANmQj8DbX33OMJ5Db-9n67_SQ\)/video/795/864/249468597_mp4_h264_aac.m3u8 example.ts -u https://proxy-038.dc3.dailymotion.com/sec\(4pkX4jyJ09RyW9jaEyekktbBu55uix9cMXQu-o5e13EelVKd1csb9zYSD66hQl7PlA_V5ntIHivm_tuQqkANmQj8DbX33OMJ5Db-9n67_SQ\)/video/795/864/ -r https://www.dailymotion.com/video/x44iz79 27 | 28 | # restore last session if the task was interrupted 29 | m3u8-dl --restore 30 | ``` 31 | 32 | If you are failed to download the stream, try it again with the options below: 33 | - Specify the Referer with `-r` when you're blocked by the website (403 forbidden). 34 | - Specify the base url with `-u` when `#EXTINF hls-720p0.ts` has no base url in `output.m3u8`. 35 | 36 | You can even make it run faster by using `-t`, which means how many threads you want to start. 37 | 38 | `--restore` will restore the last session. 39 | 40 | For more details, check `--help`. 41 | 42 | ### Notes 43 | - Version 0.2.0 44 | 1. Add support to local m3u8 files. For example 45 | ``` 46 | m3u8-dl file:///Users/username/Downloads/master.m3u8 example.ts 47 | ``` 48 | 49 | 2. Create a fake m3u8 file depends on ts range. For example 50 | ``` 51 | m3u8-dl -f master.m3u8 -r 1,100 --ts seg-@NUMBER-f1-v1-a1.ts?validfrom=1581996390&validto=1582003590&ip=89.187.161.206&hdl=-1&hash=rz91LEl6l%2FSZH83nXkv5BXzUhOQ%3D 52 | ``` 53 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | M3u8-dl 3 | ^^^^^^^ 4 | 5 | 6 | .. image:: https://img.shields.io/pypi/v/m3u8_dl.svg 7 | :target: https://img.shields.io/pypi/v/m3u8_dl.svg 8 | :alt: alt text 9 | 10 | 11 | .. image:: https://img.shields.io/travis/kedpter/m3u8_dl.svg 12 | :target: https://img.shields.io/travis/kedpter/m3u8_dl.svg 13 | :alt: alt text 14 | 15 | 16 | .. image:: https://readthedocs.org/projects/m3u8_dl/badge/?version=latest 17 | :target: https://readthedocs.org/projects/m3u8_dl/badge/?version=latest 18 | :alt: alt text 19 | 20 | 21 | M3u8-dl is a simple command-line util which downloads m3u8 file. 22 | 23 | Install 24 | ^^^^^^^ 25 | 26 | .. code-block:: bash 27 | 28 | pip install m3u8-dl 29 | 30 | Usage 31 | ^^^^^ 32 | 33 | Get the HLS Request infomation from web browser with ``Developer Tools``. 34 | Such As ``Request URL`` and ``Referer``. 35 | 36 | .. code-block:: bash 37 | 38 | # HLS_URL -> Request URL 39 | # OUTPUT -> such as example.ts 40 | m3u8-dl HLS_URL OUTPUT 41 | # restore last session if the task was interrupted 42 | m3u8-dl --restore 43 | 44 | If you are failed to download the stream, try it again with the options below: 45 | 46 | 47 | * Specify the Referer with ``-r`` when you're blocked by the website (403 forbidden). 48 | * Specify the base url with ``-u`` when ``#EXTINF hls-720p0.ts`` has no base url in ``output.m3u8``. 49 | 50 | You can even make it run faster by using ``-t``\ , which means how many threads you want to start. 51 | 52 | ``--restore`` will restore the last session. 53 | 54 | For more details, check ``--help``. 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | m3u8==0.3.7 2 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | bumpversion==0.5.3 2 | wheel==0.30.0 3 | watchdog==0.8.3 4 | tox==2.9.1 5 | coverage==4.5.1 6 | Sphinx==1.7.1 7 | 8 | 9 | 10 | autoflake>=1.2 11 | flake8>=3.5.0 12 | pytest>=3.6.2 13 | twine>=1.11.0 14 | yapf>=0.22.0 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version='{current_version}' 8 | replace = version='{new_version}' 9 | 10 | [bumpversion:file:m3u8_dl/__init__.py] 11 | search = __version__ = '{current_version}' 12 | replace = __version__ = '{new_version}' 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | 20 | [aliases] 21 | # Define setup.py command aliases here 22 | 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """The setup script.""" 4 | import re 5 | import codecs 6 | import os 7 | 8 | 9 | from setuptools import setup, find_packages 10 | 11 | with open('README.rst') as readme_file: 12 | readme = readme_file.read() 13 | 14 | with open('HISTORY.rst') as history_file: 15 | history = history_file.read() 16 | 17 | requirements = [ 18 | "m3u8>=0.3.7" 19 | ] 20 | 21 | setup_requirements = [] 22 | 23 | test_requirements = [] 24 | 25 | here = os.path.abspath(os.path.dirname(__file__)) 26 | 27 | 28 | def read(*parts): 29 | # intentionally *not* adding an encoding option to open, See: 30 | # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 31 | with codecs.open(os.path.join(here, *parts), 'r') as fp: 32 | return fp.read() 33 | 34 | 35 | def find_pakcage_info(info, *file_paths): 36 | info_file = read(*file_paths) 37 | 38 | match = re.search(r"^__" + re.escape(info) + r"__ = ['\"]([^'\"]*)['\"]", 39 | info_file, re.M) 40 | 41 | if match: 42 | return match.group(1) 43 | raise RuntimeError("Unable to find {} string.".format(info)) 44 | 45 | 46 | setup( 47 | author=find_pakcage_info('author', 'src', 'm3u8_dl', '__init__.py'), 48 | author_email=find_pakcage_info('email', 'src', 'm3u8_dl', '__init__.py'), 49 | version=find_pakcage_info('version', 'src', 'm3u8_dl', '__init__.py'), 50 | 51 | classifiers=[ 52 | 'Development Status :: 2 - Pre-Alpha', 53 | 'Intended Audience :: Developers', 54 | 'License :: OSI Approved :: MIT License', 55 | 'Natural Language :: English', 56 | "Programming Language :: Python :: 2", 57 | 'Programming Language :: Python :: 2.7', 58 | 'Programming Language :: Python :: 3', 59 | 'Programming Language :: Python :: 3.4', 60 | 'Programming Language :: Python :: 3.5', 61 | 'Programming Language :: Python :: 3.6', 62 | ], 63 | package_dir={"": "src"}, 64 | packages=find_packages( 65 | where="src", 66 | exclude=["contrib", "docs", "tests*", "tasks"], 67 | ), 68 | # package_data={'yourpackage': ['*.txt', 'path/to/resources/*.txt']}, 69 | description="Python downloader for HTTP Live Streaming (HLS), which is m3u8 file ", # noqa 70 | entry_points={ 71 | 'console_scripts': [ 72 | 'm3u8-dl=m3u8_dl.cli:main', 73 | ], 74 | }, 75 | install_requires=requirements, 76 | license="MIT license", 77 | long_description=readme + '\n\n' + history, 78 | include_package_data=True, 79 | keywords='m3u8_dl', 80 | name='m3u8_dl', 81 | setup_requires=setup_requirements, 82 | test_suite='tests', 83 | tests_require=test_requirements, 84 | url='https://github.com/kedpter/m3u8_dl', 85 | ) 86 | -------------------------------------------------------------------------------- /src/m3u8_dl/M3u8Downloader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import m3u8 4 | import requests 5 | from urllib.parse import urlparse, urljoin 6 | import os 7 | import shutil 8 | from threading import Thread, Lock 9 | import urllib3 10 | from m3u8_dl import myprint 11 | # to surpress InsecureRequestWarning 12 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 13 | 14 | 15 | def monitor_proc(proc_name): 16 | def monitor(func): 17 | def wrapper(*args, **kwargs): 18 | myprint.myprint('Executing: {} ...'.format(proc_name)) 19 | func(*args, **kwargs) 20 | myprint.myprint('Finished: {} '.format(proc_name)) 21 | return wrapper 22 | return monitor 23 | 24 | 25 | class DownloadFileNotValidException(Exception): 26 | pass 27 | 28 | 29 | class M3u8DownloaderNoStreamException(Exception): 30 | pass 31 | 32 | 33 | class M3u8DownloaderMaxTryException(Exception): 34 | pass 35 | 36 | 37 | def download_file(fileurl, headers, filename, check=None, verify=True): 38 | with requests.get(fileurl, headers=headers, stream=True, verify=verify) as r: # noqa 39 | if check and not check(r): 40 | myprint.myprint('Not a valid ts file') 41 | myprint.myprint(r.content) 42 | raise DownloadFileNotValidException() 43 | with open(filename, 'wb') as f: 44 | for chunk in r.iter_content(chunk_size=8192): 45 | if chunk: 46 | f.write(chunk) 47 | 48 | 49 | class M3u8File: 50 | 51 | def __init__(self, fileurl, headers, output_file, sslverify, finished=False): # noqa 52 | self.fileurl = fileurl 53 | self.headers = headers 54 | self.output_file = output_file 55 | self.finished = finished 56 | self.sslverify = sslverify 57 | 58 | def get_file(self): 59 | # check scheme (http or local) 60 | parsed_url = urlparse(self.fileurl) 61 | if parsed_url.scheme == "http" or parsed_url.scheme == "https": 62 | if not self.finished: 63 | download_file(self.fileurl, self.headers, 64 | self.output_file, verify=self.sslverify) 65 | elif parsed_url.scheme == "file": 66 | shutil.copy(parsed_url.path, self.output_file) 67 | else: 68 | raise Exception("Unspported url scheme") 69 | 70 | def parse_file(self): 71 | self.m3u8_obj = m3u8.load(self.output_file) 72 | return self.m3u8_obj 73 | 74 | def get_tssegments(self): 75 | return self.m3u8_obj.data['segments'] 76 | 77 | @staticmethod 78 | def get_path_by_url(url, folder): 79 | return os.path.join(folder, 80 | urlparse(url).path.split('/')[-1]) 81 | 82 | 83 | class TsFile(): 84 | def __init__(self, fileurl, headers, output_file, index, sslverify): 85 | self.fileurl = fileurl 86 | self.headers = headers 87 | self.output_file = output_file 88 | self.index = index 89 | self.finished = False 90 | self.sslverify = sslverify 91 | 92 | @staticmethod 93 | def check_valid(request): 94 | if request.headers['Content-Type'] == 'text/html': 95 | return False 96 | return True 97 | 98 | def get_file(self): 99 | download_file(self.fileurl, self.headers, self.output_file, 100 | self.check_valid, self.sslverify) 101 | self.finished = True 102 | 103 | 104 | class M3u8Context(object): 105 | rendering_attrs = ['file_url', 'base_url', 'referer', 'threads', 'output_file', 'sslverify', 106 | 'get_m3u8file_complete', 'downloaded_ts_urls', 'quiet'] 107 | 108 | def __init__(self, **kwargs): 109 | self._container = {} 110 | for key, value in kwargs.items(): 111 | self._container.setdefault(key, value) 112 | 113 | def __getitem__(self, item): 114 | return self._container[item] 115 | 116 | def __setitem__(self, key, value): 117 | self._container[key] = value 118 | 119 | def __iter__(self): 120 | return iter(self._container.items()) 121 | 122 | def __getstate__(self): 123 | obj_dict = {} 124 | for attr in self.rendering_attrs: 125 | if attr in self._container: 126 | obj_dict[attr] = self._container[attr] 127 | return obj_dict 128 | 129 | def __setstate__(self, obj): 130 | self._container = obj 131 | 132 | 133 | class M3u8Downloader: 134 | m3u8_filename = 'output.m3u8' 135 | ts_tmpfolder = '.tmpts' 136 | max_try = 10 137 | 138 | def __init__(self, context, on_progress_callback=None): 139 | self.context = context 140 | self.is_task_success = False 141 | 142 | self.fileurl = context['file_url'] 143 | self.base_url = context['base_url'] 144 | self.referer = context['referer'] 145 | self.threads = context['threads'] 146 | self.output_file = context['output_file'] 147 | self.sslverify = context['sslverify'] 148 | 149 | if context['quiet']: 150 | myprint.myprint = myprint.quiet_print 151 | 152 | self.headers = {'Referer': self.referer} 153 | self.tsfiles = [] 154 | 155 | self.ts_index = 0 156 | self.lock = Lock() 157 | 158 | self.on_progress = on_progress_callback 159 | 160 | if not os.path.isdir(self.ts_tmpfolder): 161 | os.mkdir(self.ts_tmpfolder) 162 | 163 | @monitor_proc('download m3u8 file') 164 | def get_m3u8file(self): 165 | self.m3u8file = M3u8File(self.fileurl, self.headers, 166 | self.m3u8_filename, self.sslverify, 167 | self.context['get_m3u8file_complete']) 168 | self.m3u8file.get_file() 169 | self.context['get_m3u8file_complete'] = True 170 | 171 | @monitor_proc('parse m3u8 file') 172 | def parse_m3u8file(self): 173 | self.m3u8file.parse_file() 174 | self.tssegments = self.m3u8file.get_tssegments() 175 | self.__all_tsseg_len = len(self.tssegments) 176 | if len(self.tssegments) == 0: 177 | myprint.myprint(self.m3u8file.m3u8_obj.data) 178 | raise M3u8DownloaderNoStreamException() 179 | 180 | @monitor_proc('download ts files') 181 | def get_tsfiles(self): 182 | """ 183 | start multiple threads to download ts files, 184 | threads will fetch links from the pool (ts segments) 185 | """ 186 | self.thread_pool = [] 187 | for i in range(self.threads): 188 | t = Thread(target=self._keep_download, args=( 189 | self.context['downloaded_ts_urls'], )) 190 | self.thread_pool.append(t) 191 | t.daemon = True 192 | t.start() 193 | for i in range(self.threads): 194 | self.thread_pool[i].join() 195 | pass 196 | 197 | def _keep_download(self, dd_ts): 198 | trycnt = 0 199 | while True: 200 | try: 201 | with self.lock: 202 | tsseg = self.tssegments.pop(0) 203 | self.ts_index += 1 204 | index = self.ts_index 205 | trycnt = 0 206 | except IndexError: 207 | self.is_task_success = True 208 | return 209 | try: 210 | self._download_ts(tsseg, index, dd_ts, trycnt) 211 | except M3u8DownloaderMaxTryException: 212 | break 213 | 214 | def _download_ts(self, tsseg, index, dd_ts, trycnt): 215 | uri = tsseg['uri'] 216 | if trycnt > self.max_try: 217 | raise M3u8DownloaderMaxTryException 218 | try: 219 | outfile = M3u8File.get_path_by_url(uri, 220 | self.ts_tmpfolder) 221 | url = urljoin(self.base_url, uri) 222 | tsfile = TsFile(url, self.headers, outfile, index, self.sslverify) 223 | 224 | 225 | if not uri in dd_ts: 226 | tsfile.get_file() 227 | dd_ts.append(uri) 228 | self.tsfiles.append(tsfile) 229 | 230 | self.on_progress(len(self.tsfiles), self.__all_tsseg_len) 231 | except DownloadFileNotValidException: 232 | trycnt = trycnt + 1 233 | self._download_ts(tsseg, index, dd_ts, trycnt) 234 | except Exception as e: 235 | myprint.myprint(e) 236 | myprint.myprint('Exception occurred, ignore ...') 237 | trycnt = trycnt + 1 238 | self._download_ts(tsseg, index, dd_ts, trycnt) 239 | 240 | @monitor_proc('merging ts files') 241 | def merge(self): 242 | # reorder 243 | self.tsfiles.sort(key=lambda x: x.index) 244 | with open(self.output_file, 'wb') as merged: 245 | for tsfile in self.tsfiles: 246 | myprint.myprint(tsfile.output_file) 247 | with open(tsfile.output_file, 'rb') as mergefile: 248 | shutil.copyfileobj(mergefile, merged) 249 | 250 | @monitor_proc('clean up') 251 | def cleanup(self): 252 | # clean 253 | shutil.rmtree(self.ts_tmpfolder) 254 | os.unlink(self.m3u8_filename) 255 | -------------------------------------------------------------------------------- /src/m3u8_dl/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for m3u8_dl.""" 4 | 5 | __author__ = """kedpter""" 6 | __email__ = '790476448@qq.com' 7 | __version__ = '0.2.2' 8 | 9 | from m3u8_dl.M3u8Downloader import M3u8Downloader, M3u8Context # noqa 10 | from m3u8_dl.restore import PickleContextRestore 11 | 12 | __all__ = ['M3u8Downloader', 'M3u8Context', 'PickleContextRestore'] 13 | -------------------------------------------------------------------------------- /src/m3u8_dl/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Console script for M3u8Downloader.""" 4 | from m3u8_dl import M3u8Downloader 5 | from m3u8_dl.faker import Faker 6 | import json 7 | import signal 8 | import sys 9 | import os 10 | import argparse 11 | from m3u8_dl import M3u8Context 12 | import pickle 13 | from m3u8_dl import PickleContextRestore 14 | 15 | 16 | 17 | def _show_progress_bar(downloaded, total): 18 | """ 19 | progress bar for command line 20 | """ 21 | htlen = 33 22 | percent = downloaded / total * 100 23 | # 20 hashtag(#) 24 | hashtags = int(percent / 100 * htlen) 25 | print('|' 26 | + '#' * hashtags + ' ' * (htlen - hashtags) + 27 | '|' + 28 | ' {0}/{1} '.format(downloaded, total) + 29 | ' {:.1f}'.format(percent).ljust(5) + ' %', end='\r', flush=True) # noqa 30 | 31 | 32 | def execute(restore, context): 33 | """ 34 | download ts file by restore object (dict) 35 | """ 36 | m = M3u8Downloader(context, on_progress_callback=_show_progress_bar) 37 | 38 | def signal_handler(sig, frame): 39 | print('\nCaptured Ctrl + C ! Saving Current Session ...') 40 | restore.dump(context) 41 | sys.exit(1) 42 | 43 | signal.signal(signal.SIGINT, signal_handler) 44 | 45 | m.get_m3u8file() 46 | print('m3u8: Saving as ' + M3u8Downloader.m3u8_filename) 47 | 48 | m.parse_m3u8file() 49 | m.get_tsfiles() 50 | if m.is_task_success: 51 | m.merge() 52 | 53 | # clean everything Downloader generates 54 | m.cleanup() 55 | # clean restore 56 | restore.cleanup() 57 | 58 | if not m.is_task_success: 59 | print('Download failed') 60 | print('Try it again with options --refer and --url') 61 | 62 | 63 | def main(): 64 | """ 65 | deal with the console 66 | """ 67 | parser = argparse.ArgumentParser() 68 | parser.add_argument("-u", "--url", default='', 69 | help="[0]base url for ts when downloading") 70 | parser.add_argument("-r", "--referer", default='', 71 | help="[0]the Referer in request header") 72 | parser.add_argument("-t", "--threads", type=int, default=10, 73 | help="[0]how many threads to start for download") 74 | parser.add_argument("--insecure", action="store_true", 75 | help="[0]ignore verifying the SSL certificate") 76 | parser.add_argument("--certfile", default='', 77 | help="[0]do not ignore SSL certificate, verify it with a file or directory with CAs") # noqa 78 | parser.add_argument("fileurl", nargs="?", 79 | help="[0]url [e.g.:http://example.com/xx.m3u8]") 80 | parser.add_argument("output", nargs="?", help="[0]file for saving [e.g.: example.ts]") # noqa 81 | parser.add_argument("--restore", action="store_true", 82 | help="[1]restore from last session") 83 | parser.add_argument("-f", "--fake", help="[2]fake a m3u8 file") 84 | parser.add_argument("--range", help="[2]ts range") 85 | parser.add_argument("--ts", help="[2]ts link") 86 | parser.add_argument("--quiet", action="store_true", 87 | help="suppress output") 88 | 89 | args = parser.parse_args() 90 | 91 | restore = PickleContextRestore() 92 | 93 | if args.fake: 94 | range = args.range.split(',') 95 | faker = Faker() 96 | faker.create_file(args.fake, args.ts, int(range[0]), int(range[1])+1) 97 | 98 | else: 99 | if args.restore: 100 | context = restore.load() 101 | else: 102 | if not args.fileurl or not args.output: 103 | print('error: [fileurl] and [output] are necessary if not in restore\n') # noqa 104 | parser.print_help() 105 | sys.exit(0) 106 | 107 | context = M3u8Context(file_url=args.fileurl, referer=args.referer, 108 | threads=args.threads, output_file=args.output, 109 | get_m3u8file_complete=False, downloaded_ts_urls=[], quiet=args.quiet) 110 | context["base_url"] = args.url \ 111 | if args.url .endswith('/') else args.url + '/' # noqa 112 | 113 | if args.insecure: 114 | context['sslverify'] = False 115 | if not args.insecure: 116 | if args.certfile == '': 117 | context['sslverify'] = True 118 | else: 119 | context['sslverify'] = args.certfile 120 | execute(restore, context) 121 | 122 | 123 | if __name__ == "__main__": 124 | main() 125 | -------------------------------------------------------------------------------- /src/m3u8_dl/faker.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | 3 | 4 | class Faker: 5 | headers = textwrap.dedent(""" 6 | #EXTM3U 7 | #EXT-X-TARGETDURATION:7 8 | #EXT-X-ALLOW-CACHE:YES 9 | #EXT-X-PLAYLIST-TYPE:VOD 10 | #EXT-X-VERSION:3 11 | #EXT-X-MEDIA-SEQUENCE:1""") 12 | 13 | def create_file(self, file, blink, rs, re): 14 | content = Faker.headers 15 | for i in range(rs, re): 16 | link = blink.replace("@NUMBER", str(i), 1) 17 | extinf = textwrap.dedent(""" 18 | #EXTINF:3.003, 19 | {0} 20 | """.format(link)) 21 | content = content + extinf 22 | 23 | with open(file, 'w') as f: 24 | f.write(content) 25 | -------------------------------------------------------------------------------- /src/m3u8_dl/myprint.py: -------------------------------------------------------------------------------- 1 | def quiet_print(msg): 2 | pass 3 | 4 | myprint = print 5 | -------------------------------------------------------------------------------- /src/m3u8_dl/restore.py: -------------------------------------------------------------------------------- 1 | from m3u8_dl import M3u8Context 2 | import pickle 3 | import os 4 | 5 | 6 | class PickleContextRestore: 7 | restore_file = 'm3u8_dl.restore' 8 | 9 | def __init__(self): 10 | pass 11 | 12 | def dump(self, context): 13 | with open(self.restore_file, 'wb') as f: 14 | pickle.dump(context, f) 15 | 16 | def load(self): 17 | with open(self.restore_file, 'rb') as f: 18 | return pickle.load(f) 19 | 20 | def cleanup(self): 21 | if os.path.isfile(self.restore_file): 22 | os.unlink(self.restore_file) 23 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for m3u8_dl.""" 4 | -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for `m3u8_dl` package.""" 5 | 6 | 7 | import unittest 8 | from click.testing import CliRunner 9 | 10 | from m3u8_dl import cli 11 | from m3u8_dl import M3u8Downloader, M3u8Context 12 | import pickle 13 | import tempfile 14 | import os 15 | 16 | 17 | class TestM3u8Context(unittest.TestCase): 18 | """Tests for `m3u8_dl` package.""" 19 | 20 | def setUp(self): 21 | """Set up test fixtures, if any.""" 22 | 23 | def tearDown(self): 24 | """Tear down test fixtures, if any.""" 25 | 26 | def test_context_to_use_like_dict(self): 27 | self.context = M3u8Context() 28 | self.context['abc'] = 1 29 | self.assertEquals(self.context['abc'], 1) 30 | 31 | def test_pickle_dump_and_load_success(self): 32 | self.context = M3u8Context() 33 | self.context['file_uri'] = '/folder/file' 34 | with tempfile.TemporaryDirectory() as tmpdirname: 35 | path = os.path.join(tmpdirname, '1.pickle') 36 | with open(path, 'wb') as f: 37 | pickle.dump(self.context, f) 38 | with open(path, 'rb') as f: 39 | new_context = pickle.load(f) 40 | # self.assertEquals(new_context['file_uri'], '/folder/file') 41 | for key, value in self.context: 42 | new_value = new_context[key] 43 | self.assertEquals(value, new_value) 44 | -------------------------------------------------------------------------------- /tests/test_m3u8_dl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for `m3u8_dl` package.""" 5 | 6 | 7 | import unittest 8 | from click.testing import CliRunner 9 | 10 | from m3u8_dl import cli 11 | 12 | 13 | class TestM3u8_dl(unittest.TestCase): 14 | """Tests for `m3u8_dl` package.""" 15 | 16 | def setUp(self): 17 | """Set up test fixtures, if any.""" 18 | 19 | def tearDown(self): 20 | """Tear down test fixtures, if any.""" 21 | 22 | def test_000_something(self): 23 | """Test something.""" 24 | 25 | def test_command_line_interface(self): 26 | """Test the CLI.""" 27 | # runner = CliRunner() 28 | # result = runner.invoke(cli.main) 29 | # assert result.exit_code == 0 30 | # assert 'm3u8_dl.cli.main' in result.output 31 | # help_result = runner.invoke(cli.main, ['--help']) 32 | # assert help_result.exit_code == 0 33 | # assert '--help Show this message and exit.' in help_result.output 34 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py36, flake8 3 | 4 | [travis] 5 | python = 6 | 3.6: py36 7 | 2.7: py27 8 | 9 | [testenv:flake8] 10 | basepython = python 11 | deps = flake8 12 | commands = flake8 easyenv 13 | 14 | [testenv] 15 | setenv = 16 | PYTHONPATH = {toxinidir} 17 | 18 | commands = pytest tests 19 | --------------------------------------------------------------------------------