├── .gitignore ├── .vim └── coc-settings.json ├── Makefile ├── README.md ├── demo.gif ├── m3_dl ├── D.py ├── __init__.py ├── __main__.py ├── logx │ ├── __init__.py │ ├── colored_handler.py │ ├── logging.yaml │ └── setup_logging.py ├── m3_dl.py └── progress2.py ├── main.py ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── common │ └── __init__.py ├── test_args.py └── test_m3u8.py ├── version └── version.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # PyCharm project file 132 | .idea -------------------------------------------------------------------------------- /.vim/coc-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.jediEnabled": true 3 | } 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | rm: 3 | find . -name '*.pyc' -exec rm -f {} + 4 | find . -name '*.pyo' -exec rm -f {} + 5 | find . -name '*~' -exec rm -f {} + 6 | find . -type d -iname '*egg-info' -exec rm -rdf {} + 7 | rm -f .coverage 8 | rm -rf htmlcov 9 | rm -rf dist 10 | rm -rf build 11 | rm -rf proxy.py.egg-info 12 | rm -rf .pytest_cache 13 | rm -rf .hypothesis 14 | rm -rdf assets 15 | 16 | 17 | test: rm 18 | pytest -s -v tests/ 19 | 20 | coverage-html: 21 | # --cov where you want to cover 22 | # tests where your test code is 23 | pytest --cov=m3_dl/ --cov-report=html tests/ 24 | open htmlcov/index.html 25 | 26 | coverage: 27 | pytest --cov=m3_dl/ tests/ 28 | 29 | install: uninstall 30 | pip3 install . 31 | 32 | uninstall: 33 | pip3 uninstall -y m3_dl 34 | 35 | dev: 36 | python3 -m m3_dl "https://edu.51cto.com//center/player/play/m3u8?lesson_id=348911&id=345643&dp=high&type=course&lesson_type=course" -d -k | mpv - 37 | 38 | run: 39 | python3 -m m3_dl ./index-v1-a1.m3u8 -k -w -o "./a.mp4" -p "socks5h://127.0.0.1:5993" -t 10 40 | 41 | stream: 42 | python3 -m m3_dl "https://doubanzyv1.tyswmp.com/2018/07/26/0vhyINWfXeWIkrJd/playlist.m3u8" -k | mpv - 43 | 44 | 45 | all: rm uninstall install run 46 | 47 | 48 | pure-all: env-rm rm env install test run 49 | 50 | 51 | 52 | upload-to-test: rm 53 | python3 setup.py bdist_wheel --universal 54 | twine upload --repository-url https://test.pypi.org/legacy/ dist/* 55 | 56 | 57 | upload-to-prod: rm auto_version 58 | python3 setup.py bdist_wheel --universal 59 | twine upload dist/* 60 | 61 | 62 | freeze-only: 63 | # pipreqs will find the module the project really depneds 64 | pipreqs . --force 65 | 66 | freeze: 67 | # pip3 will find all the module not belong to standard library 68 | pip3 freeze > requirements.txt 69 | 70 | 71 | env-rm: 72 | rm -rdf env 73 | 74 | 75 | env: 76 | python3 -m venv env 77 | . env/bin/activate 78 | 79 | 80 | convert: 81 | ffmpeg -i anjia12.mkv -codec copy anjia12_mp4.mp4 82 | 83 | 84 | 85 | auto_version: 86 | python version.py 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intro 2 | This is a pipe compatiable tool for downloading m3u8. 3 | 4 | 1. Fast download for m3u8. 5 | 2. Auto decrpt the key. 6 | 7 | ![](https://github.com/zk4/m3u8_dl/blob/master/demo.gif?raw=true) 8 | # Install 9 | since the name m3u8_dl is taken by other developer... 10 | 11 | short it to m3_dl 12 | ``` 13 | pip3 install m3_dl 14 | ``` 15 | > python version should be bigger than 3.7.1, my dev env is 3.7.1 16 | 17 | # Usage 18 | ``` 19 | m3_dl -o 20 | ``` 21 | 22 | ex: 23 | ``` 24 | # download to local file 25 | m3_dl http://aaa.com/a.m3u8 -o ./a.mp4 26 | 27 | # pipe it to mpv 28 | m3_dl https://you.tube-kuyun.com/20200210/1144_623a1fb3/index.m3u8 | mpv - 29 | 30 | # pipe it to mpv and save to local 31 | m3_dl https://you.tube-kuyun.com/20200210/1144_623a1fb3/index.m3u8 | tee > ./a.mp4 | mpv - 32 | ``` 33 | 34 | # Full Usage 35 | ``` 36 | usage: m3_dl [-h] [-o OUT_PATH] [-p PROXY] [-t THREADCOUNT] [-d] [-w] [-s] 37 | [--version] [-k] 38 | url 39 | 40 | positional arguments: 41 | url url 42 | 43 | optional arguments: 44 | -h, --help show this help message and exit 45 | -o OUT_PATH, --out_path OUT_PATH 46 | output path (default: None) 47 | -p PROXY, --proxy PROXY 48 | for example: socks5h://127.0.0.1:5992 (default: None) 49 | 50 | -e KEY custom decrypt key 51 | -t THREADCOUNT, --threadcount THREADCOUNT 52 | thread count (default: 2) 53 | -d, --debug debug info (default: False) 54 | -w, --overwrite overwrite existed file (default: False) 55 | --version show program's version number and exit 56 | -k, --ignore_certificate_verfication 57 | ignore certificate verfication, don`t use this option 58 | only if you know what you are doing! (default: False) 59 | ``` 60 | 61 | 62 | 63 | # TODO 64 | 1. enable redownlowd 65 | 1. make it mitm compatiable 66 | 67 | 68 | # QA 69 | **Why still can't I download m3u8 even though I got the correct m3u8 content?** 70 | 71 | Sometimes m3u8 file is not enough for m3_dl to donwload, cause the key is protected by user credential or whatever. 72 | 73 | You need to get the key somehow. and tell m3_dl by -e option. 74 | -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zk4/m3u8_dl/c559f56f08cc07d2ac0f34843da401b6ef7909c2/demo.gif -------------------------------------------------------------------------------- /m3_dl/D.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os 4 | from io import BytesIO 5 | import requests 6 | import traceback 7 | import logging 8 | import time 9 | import threading 10 | from .progress2 import pb2 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def userDefineVisual(tag, nowValue, fullValue, extrainfo): 16 | percent = float(nowValue) / fullValue 17 | icons = "🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚🕛✅" 18 | icons_len = len(icons) 19 | s = "%3d%%" % int(round(percent * 100)) if fullValue != nowValue else " " 20 | return f"{tag} {icons[nowValue%(icons_len-1)] if fullValue!=nowValue else icons[-1]} {s} {extrainfo.rjust(15)}" 21 | 22 | 23 | class D: 24 | def __init__( 25 | self, 26 | cookie=None, 27 | proxies=None, 28 | auth=None, 29 | headers=None, 30 | verify=True, 31 | debug=False, 32 | ignore_local=False, 33 | retry_times=9999999999, 34 | ) -> None: 35 | self.cookie = cookie 36 | self.proxies = proxies 37 | self.auth = auth 38 | self.headers = headers 39 | self.ignore_local = ignore_local 40 | self.retry_times = retry_times 41 | self.current_retry_times = 0 42 | self.verify = verify 43 | self.debug = debug 44 | super().__init__() 45 | 46 | def download(self, url, destFile, isAppend=True): 47 | try: 48 | if not os.path.isdir(os.path.dirname(destFile)): 49 | os.mkdir(os.path.dirname(destFile)) 50 | 51 | if os.path.exists(destFile): 52 | return True 53 | 54 | webSize = self.getWebFileSize(url) 55 | if webSize == 0: 56 | logger.debug("something went wrong, webSize is 0") 57 | return False 58 | 59 | localSize = 0 60 | if self.cookie: 61 | self.headers["cookie"] = self.cookie 62 | 63 | if isAppend: 64 | self.headers["Range"] = "bytes=%d-" % localSize 65 | else: 66 | os.remove(destFile) 67 | localSize = 0 68 | 69 | resp = requests.request( 70 | "GET", 71 | url, 72 | timeout=10, 73 | headers=self.headers, 74 | stream=True, 75 | proxies=self.proxies, 76 | allow_redirects=True, 77 | verify=self.verify, 78 | auth=self.auth, 79 | ) 80 | # if 300>resp.status_code >= 200: 81 | if resp.status_code >= 200: 82 | # logger.debug(f"stauts_code:{resp.status_code},destfile:{destFile}") 83 | 84 | name = threading.current_thread().getName() 85 | p = pb2.getSingleton() 86 | start = time.time() 87 | with open(destFile + ".tmp", "ab") as f: 88 | block_size = 1024 89 | wrote = localSize 90 | for data in resp.iter_content(block_size): 91 | if data: 92 | wrote = wrote + len(data) 93 | f.write(data) 94 | p.update( 95 | name.rjust(13, " "), 96 | wrote, 97 | webSize, 98 | str(int(wrote / (int(time.time() - start) + 1) / 1024)) 99 | + "kb/s", 100 | userDefineVisual, 101 | ) 102 | if wrote != webSize: 103 | logger.debug( 104 | f"ERROR, something went wrong wroteSize{wrote} != webSize{webSize}" 105 | ) 106 | return False 107 | 108 | os.rename(destFile + ".tmp", destFile) 109 | return True 110 | 111 | logger.debug(f"stauts_code:{resp.status_code},url:{resp.url}") 112 | raise Exception("status_code is not 200.") 113 | 114 | except Exception as e: 115 | if self.debug: 116 | logger.exception(e) 117 | # traceback.print_stack() 118 | return False 119 | 120 | def getWebFileSize(self, url): 121 | if self.cookie: 122 | self.headers["cookie"] = self.cookie 123 | 124 | rr = requests.get( 125 | url, 126 | headers=self.headers, 127 | stream=True, 128 | proxies=self.proxies, 129 | verify=self.verify, 130 | auth=self.auth, 131 | ) 132 | file_size = int(rr.headers["Content-Length"]) 133 | 134 | if 300 > rr.status_code >= 200: 135 | return file_size 136 | else: 137 | return 0 138 | -------------------------------------------------------------------------------- /m3_dl/__init__.py: -------------------------------------------------------------------------------- 1 | from .m3_dl import entry_point, main, createParse, m3u8_dl 2 | 3 | 4 | __all__ = ["entry_point", "createParse", "main", "m3u8_dl"] 5 | -------------------------------------------------------------------------------- /m3_dl/__main__.py: -------------------------------------------------------------------------------- 1 | from .m3_dl import entry_point 2 | 3 | if __name__ == "__main__": 4 | entry_point() 5 | -------------------------------------------------------------------------------- /m3_dl/logx/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from .setup_logging import setup_logging 3 | from .colored_handler import ColoredHandler 4 | -------------------------------------------------------------------------------- /m3_dl/logx/colored_handler.py: -------------------------------------------------------------------------------- 1 | from logging import Handler 2 | from termcolor import colored 3 | 4 | # class color: 5 | # W = '\033[0m' # white (normal) 6 | # R = '\033[31m' # red 7 | # G = '\033[32m' # green 8 | # O = '\033[33m' # orange 9 | # B = '\033[34m' # blue 10 | # P = '\033[35m' # purple 11 | # C = '\033[36m' # cyan 12 | # GR = '\033[37m' # gray 13 | 14 | 15 | class ColoredHandler(Handler): 16 | def emit(self, record): 17 | if record.levelname == "INFO": 18 | record.msg = colored(record.getMessage(), "green") 19 | if record.levelname == "ERROR": 20 | record.msg = colored(record.getMessage(), "cyan") 21 | if record.levelname == "CRITICAL": 22 | record.msg = colored(record.getMessage(), "red") 23 | if record.levelname == "WARNING": 24 | record.msg = colored(record.getMessage(), "blue") 25 | -------------------------------------------------------------------------------- /m3_dl/logx/logging.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | disable_existing_loggers: true 3 | 4 | formatters: 5 | standard: 6 | # format: '[%(levelname)-8s] %(filename)s:%(lineno)-5s %(message)s' 7 | format: "%(module)s:%(lineno)-4s %(message)s" 8 | error: 9 | format: "[%(asctime)s]:%(levelname)s %(name)s.%(funcName)s():\n %(message)s" 10 | 11 | handlers: 12 | console: 13 | class: logging.StreamHandler 14 | formatter: standard 15 | 16 | colored: 17 | class: m3_dl.logx.ColoredHandler 18 | 19 | root: 20 | level: INFO 21 | handlers: [colored,console] 22 | propagate: no 23 | 24 | loggers: 25 | logx: 26 | level: INFO 27 | handlers: [colored,console] 28 | propagate: no 29 | m3_dl: 30 | level: INFO 31 | handlers: [colored,console] 32 | propagate: no 33 | 34 | -------------------------------------------------------------------------------- /m3_dl/logx/setup_logging.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | import yaml 4 | import logging.config 5 | import logging 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def setup_logging( 11 | default_path="logging.yaml", default_level=logging.DEBUG, env_key="LOG_CFG" 12 | ): 13 | mydir = os.path.dirname(os.path.abspath(__file__)) 14 | path = default_path 15 | path = os.path.join(mydir, path) 16 | value = os.getenv(env_key, None) 17 | if value: 18 | path = value 19 | if os.path.exists(path): 20 | with open(path, "rt") as f: 21 | try: 22 | config = yaml.safe_load(f.read()) 23 | logging.config.dictConfig(config) 24 | except Exception as e: 25 | print(e) 26 | print("Error in Logging Configuration. Using default configs") 27 | else: 28 | logging.basicConfig(level=default_level) 29 | print(".Failed to load configuration file. Using default configs") 30 | -------------------------------------------------------------------------------- /m3_dl/m3_dl.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from concurrent.futures import ThreadPoolExecutor 4 | from os.path import join, basename, dirname 5 | from pathlib import Path 6 | from urllib.parse import urljoin 7 | import argparse 8 | import logging 9 | import os 10 | import random 11 | import queue 12 | import requests 13 | from requests.auth import HTTPBasicAuth 14 | import subprocess 15 | import threading 16 | import tempfile 17 | import uuid 18 | import time 19 | from Crypto.Cipher import AES 20 | import sys 21 | from .progress2 import pb2 22 | 23 | # don`t show verfication warning 24 | from urllib3.exceptions import InsecureRequestWarning 25 | 26 | requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) 27 | 28 | 29 | from .D import D, userDefineVisual 30 | from .logx import setup_logging 31 | 32 | # don`t remove this line 33 | setup_logging() 34 | 35 | logger = logging.getLogger(__name__) 36 | 37 | # proxies={"https":"socks5h://127.0.0.1:5992","http":"socks5h://127.0.0.1:5992"} 38 | headers = { 39 | "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", 40 | } 41 | 42 | 43 | def userDefineVisual2(tag, nowValue, fullValue, extrainfo): 44 | bar_length = 100 45 | percent = float(nowValue) / fullValue 46 | arrow = "#" * int(round(percent * bar_length) - 1) + "#" 47 | spaces = " " * (bar_length - len(arrow)) 48 | return "{2} [{0}] {1}%".format(arrow + spaces, int(round(percent * 100)), tag) 49 | 50 | 51 | class m3u8_dl(object): 52 | def __init__(self, url, out_path, proxy, auth, not_verify_ssl, custom_key, debug): 53 | pool_size = 10 54 | self.proxies = {"https": proxy, "http": proxy} 55 | self.verify = not not_verify_ssl 56 | self.url = url 57 | self.is_http_url = url.startswith("http") 58 | self.out_path = out_path 59 | self.session = self._get_http_session(pool_size, pool_size, 5, auth) 60 | self.m3u8_content = self.m3u8content(url) 61 | self.ts_list = [ 62 | urljoin(self.url, n.strip()) 63 | for n in self.m3u8_content.split("\n") 64 | if n and not n.startswith("#") 65 | ] 66 | self.length = len(self.ts_list) 67 | self.ts_list_pair = zip(self.ts_list, [n for n in range(len(self.ts_list))]) 68 | self.next_merged_id = 0 69 | self.ready_to_merged = set() 70 | self.downloadQ = queue.PriorityQueue() 71 | self.tempdir = tempfile.gettempdir() 72 | # self.tempdir = "/Users/zk/git/pythonPrj/m3u8_dl/temp/" 73 | self.tempname = str(uuid.uuid4()) 74 | self.debug = debug 75 | 76 | if self.out_path: 77 | outdir = dirname(out_path) 78 | if outdir and not os.path.isdir(outdir): 79 | os.makedirs(outdir) 80 | 81 | if self.out_path and os.path.isfile(self.out_path): 82 | os.remove(self.out_path) 83 | 84 | key = self.readkey() if not custom_key else custom_key 85 | 86 | self.cryptor = None 87 | if key: 88 | logger.debug(f"key: {key}") 89 | self.cryptor = AES.new(key, AES.MODE_CBC, key) 90 | 91 | def decode(self, content): 92 | if self.cryptor: 93 | return self.cryptor.decrypt(content) 94 | else: 95 | return content 96 | 97 | def readkey(self): 98 | tag_list = [ 99 | n.strip() for n in self.m3u8_content.split("\n") if n and n.startswith("#") 100 | ] 101 | for s in tag_list: 102 | if str.upper(s).startswith("#EXT-X-KEY"): 103 | logger.debug(f"{s} found") 104 | segments = s[len("#EXT-X-KEY") + 1 :] 105 | if segments == "NONE": 106 | return None 107 | 108 | logger.debug(f"segments:{segments}") 109 | segments_splited = segments.split(",") 110 | # [method,uri]=segments.split(",") 111 | method = segments_splited[0] 112 | uri = segments_splited[1] 113 | method, uri = method.split("=", 1)[1], uri.split("=", 1)[1][1:-1] 114 | 115 | logger.debug(f"request uri: {uri}") 116 | 117 | uri = urljoin(self.url, uri) 118 | 119 | r = self.session.get(uri, proxies=self.proxies) 120 | if r.status_code == 200: 121 | return r.content 122 | logger.fatal( 123 | f"Can`t download key url: {uri}, maybe you should use proxy" 124 | ) 125 | sys.exit(-1) 126 | 127 | def _get_http_session(self, pool_connections, pool_maxsize, max_retries, auth=None): 128 | session = requests.Session() 129 | adapter = requests.adapters.HTTPAdapter( 130 | pool_connections=pool_connections, 131 | pool_maxsize=pool_maxsize, 132 | max_retries=max_retries, 133 | ) 134 | session.mount("http://", adapter) 135 | session.mount("https://", adapter) 136 | 137 | if auth: 138 | username, password = auth 139 | session.auth = HTTPBasicAuth(username, password) 140 | 141 | return session 142 | 143 | def m3u8content(self, m3u8_url): 144 | logger.debug(f"m3u8_url {m3u8_url}") 145 | if m3u8_url.startswith("http"): 146 | r = self.session.get( 147 | m3u8_url, 148 | timeout=20, 149 | headers=headers, 150 | proxies=self.proxies, 151 | verify=self.verify, 152 | ) 153 | if r.ok: 154 | ts_list = [ 155 | urljoin(m3u8_url, n.strip()) 156 | for n in r.text.split("\n") 157 | if n and not n.startswith("#") 158 | ] 159 | if ts_list[0].endswith("m3u8"): 160 | self.url = urljoin(m3u8_url, ts_list[0]) 161 | return self.m3u8content(self.url) 162 | return r.text 163 | else: 164 | logger.debug(f"respnse:{r}") 165 | else: 166 | return Path(m3u8_url).read_text() 167 | 168 | raise Exception("read m3u8 content error.") 169 | 170 | def download(self, url, i): 171 | try: 172 | d = D( 173 | proxies=self.proxies, 174 | auth=self.session.auth, 175 | headers=headers, 176 | verify=self.verify, 177 | debug=self.debug, 178 | ) 179 | # logger.debug(f'url:{url}') 180 | pathname = join(self.tempdir, self.tempname, str(i)) 181 | # logger.debug(f'pathname:{pathname}') 182 | ret = d.download(url, pathname) 183 | if ret: 184 | # logger.info(f'{i} done') 185 | self.ready_to_merged.add(i) 186 | else: 187 | logger.debug(f"{i} download fails! re Q") 188 | self.downloadQ.put((i, url)) 189 | 190 | except Exception as e: 191 | if self.debug: 192 | logger.exception(e) 193 | 194 | def target(self): 195 | while self.next_merged_id < self.length: 196 | try: 197 | idx, url = self.downloadQ.get(timeout=3) 198 | if url: 199 | self.download(url, idx) 200 | except Exception as e: 201 | # logger.exception(e) 202 | pass 203 | 204 | def run(self, threadcount): 205 | if self.ts_list_pair: 206 | 207 | for i in range(threadcount): 208 | threading.Thread(target=self.target).start() 209 | 210 | threading.Thread(target=self.try_merge).start() 211 | 212 | for pair in self.ts_list_pair: 213 | self.downloadQ.put((pair[1], pair[0])) 214 | 215 | def try_merge(self): 216 | outfile = None 217 | 218 | pp = pb2.getSingleton() 219 | if self.out_path: 220 | outfile = open(self.out_path, "ab") 221 | while self.next_merged_id < self.length: 222 | dots = random.randint(0, 3) * "." 223 | 224 | # p.print(f'\r{self.next_merged_id}/{self.length} merged '+dots+(3-len(dots))*" ",file=sys.stderr,end="") 225 | pp.update( 226 | "total merged ", self.next_merged_id, self.length, "", userDefineVisual2 227 | ) 228 | pp.update( 229 | "block pending", 230 | self.next_merged_id + len(self.ready_to_merged), 231 | self.length, 232 | "", 233 | userDefineVisual2, 234 | ) 235 | oldidx = self.next_merged_id 236 | try: 237 | if self.next_merged_id in self.ready_to_merged: 238 | logger.debug(f"try merge {self.next_merged_id} ....") 239 | self.ready_to_merged.remove(self.next_merged_id) 240 | p = os.path.join( 241 | self.tempdir, self.tempname, str(self.next_merged_id) 242 | ) 243 | 244 | infile = open(p, "rb") 245 | o = self.decode(infile.read()) 246 | 247 | if self.out_path: 248 | outfile.write(o) 249 | outfile.flush() 250 | else: 251 | sys.stdout.buffer.write(o) 252 | sys.stdout.flush() 253 | 254 | infile.close() 255 | 256 | self.next_merged_id += 1 257 | 258 | os.remove(join(self.tempdir, self.tempname, str(oldidx))) 259 | else: 260 | time.sleep(1) 261 | logger.debug(f"waiting for {self.next_merged_id} to merge ") 262 | logger.debug( 263 | f"unmerged {self.ready_to_merged} active_thread:{threading.active_count()}" 264 | ) 265 | except Exception as e: 266 | logger.exception(e) 267 | try: 268 | self.next_merged_id = oldidx 269 | os.remove(join(self.tempdir, self.tempname, str(oldidx))) 270 | logger.error(f"{oldidx} merge error ,reput to thread") 271 | logger.exception(e) 272 | # print(self.ts_list[oldidx],oldidx) 273 | self.downloadQ.put((oldidx, self.ts_list[oldidx])) 274 | except Exception as e2: 275 | logger.exception(e) 276 | 277 | if self.out_path: 278 | outfile.close() 279 | 280 | 281 | def main(args): 282 | if args.debug: 283 | logger.setLevel("DEBUG") 284 | 285 | logger.debug(f"args.out_path:{args.out_path}") 286 | if args.out_path and os.path.exists(args.out_path) and not args.overwrite: 287 | logger.error(f"{args.out_path} exists! use -w if you want to overwrite it ") 288 | sys.exit(-1) 289 | 290 | m = m3u8_dl( 291 | args.url, 292 | args.out_path, 293 | args.proxy, 294 | args.auth.split(":", 1) if args.auth else None, 295 | args.ignore_certificate_verfication, 296 | args.key, 297 | args.debug, 298 | ) 299 | 300 | # must ensure 1 for merged thread 301 | threadcount = args.threadcount + 1 302 | m.run(threadcount) 303 | 304 | 305 | def entry_point(): 306 | parser = createParse() 307 | mainArgs = parser.parse_args() 308 | main(mainArgs) 309 | 310 | 311 | def createParse(): 312 | parser = argparse.ArgumentParser( 313 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="" 314 | ) 315 | parser.add_argument("url", help="url") 316 | parser.add_argument("-o", "--out_path", type=str, help="output path, ex: ./a.mp4") 317 | parser.add_argument( 318 | "-p", "--proxy", type=str, help="for example: socks5h://127.0.0.1:5992" 319 | ) 320 | parser.add_argument("-a", "--auth", type=str, help="for example: username:password") 321 | parser.add_argument("-e", "--key", type=str, help="custom decrypt key") 322 | parser.add_argument("-t", "--threadcount", type=int, help="thread count", default=2) 323 | parser.add_argument( 324 | "-d", "--debug", help="debug info", default=False, action="store_true" 325 | ) 326 | parser.add_argument( 327 | "-w", "--overwrite", help="overwrite existed file", action="store_true" 328 | ) 329 | mydir = os.path.dirname(os.path.abspath(__file__)) 330 | version = Path(join(mydir, "..", "version")).read_text() 331 | parser.add_argument("--version", action="version", version=version) 332 | parser.add_argument( 333 | "-k", 334 | "--ignore_certificate_verfication", 335 | help="ignore certificate verfication, don`t use this option only if you know what you are doing!", 336 | action="store_true", 337 | ) 338 | 339 | return parser 340 | -------------------------------------------------------------------------------- /m3_dl/progress2.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Author:zk 2018.7.19 3 | import collections 4 | import time 5 | import sys 6 | import threading 7 | from threading import Thread 8 | import logging 9 | 10 | logger = logging.getLogger(__name__) 11 | logger.setLevel(level=logging.WARN) 12 | 13 | 14 | class pb2: 15 | __od = collections.OrderedDict() 16 | __lock = threading.RLock() 17 | __dirty = False 18 | __instance = None 19 | __currentIconIdx = 0 20 | __istty = sys.stdout.isatty() 21 | if not __istty: 22 | logger.warning( 23 | "WARNNING:pb2 not in tty! only support one line. multi line not supported" 24 | ) 25 | 26 | def up(self): 27 | if self.__istty: 28 | sys.stdout.write("\x1b[1A") 29 | sys.stdout.flush() 30 | 31 | def down(self): 32 | if self.__istty: 33 | sys.stdout.write("\n") 34 | sys.stdout.flush() 35 | 36 | @classmethod 37 | def getSingleton(cls): 38 | if not cls.__instance: 39 | with cls.__lock: 40 | if not cls.__instance: 41 | cls.__instance = cls() 42 | return cls.__instance 43 | 44 | def __init__(self) -> None: 45 | super().__init__() 46 | self.___demon() 47 | 48 | def ___demon(self): 49 | t = Thread(target=self.paint) 50 | t.daemon = True 51 | t.start() 52 | 53 | def update(self, tag, nowValue, fullValue, extrainfo=None, customVisualbar=None): 54 | with self.__lock: 55 | self.__od[tag] = ( 56 | self.customVisualbar(tag, nowValue, fullValue, extrainfo) 57 | if customVisualbar is None 58 | else customVisualbar(tag, nowValue, fullValue, extrainfo) 59 | ) 60 | 61 | self.__dirty = True 62 | 63 | def customVisualbar(self, tag, nowValue, fullValue, extrainfo=""): 64 | bar_length = 100 65 | percent = float(nowValue) / fullValue 66 | arrow = "-" * int(round(percent * bar_length) - 1) + ">" 67 | spaces = " " * (bar_length - len(arrow)) 68 | return "{2} [{0}] {1}% {3}".format( 69 | arrow + spaces, int(round(percent * 100)), tag, extrainfo 70 | ) 71 | 72 | # print after bar 73 | def print(self, str): 74 | with self.__lock: 75 | self.__od[time.time()] = str 76 | self.__dirty = True 77 | 78 | # print before bar 79 | # def insert_print(self, str): 80 | # with self.__lock: 81 | # time_time = time.time() 82 | # columns, rows = shutil.get_terminal_size() 83 | # self.__od[time_time] = str+' '*(columns-len(str) ) 84 | # self.__od.move_to_end(time_time, last=False) 85 | # self.__dirty = True 86 | def start(self): 87 | 88 | with self.__lock: 89 | self.__paint() 90 | for i in range(len(self.__od)): 91 | self.down() 92 | self.__od.clear() 93 | 94 | def stop(self): 95 | self.start() 96 | 97 | def paint(self): 98 | while True: 99 | if not self.__dirty: 100 | time.sleep(0.1) 101 | continue 102 | self.__lock.acquire() 103 | self.__paint() 104 | self.__lock.release() 105 | 106 | def __paint(self): 107 | text = "" 108 | for _, v in self.__od.items(): 109 | if self.__istty: 110 | print(v) 111 | else: 112 | print("\r" + v, end="") 113 | for i in range(len(self.__od)): 114 | self.up() 115 | self.__dirty = False 116 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from m3_dl import entry_point 3 | 4 | if __name__ == "__main__": 5 | entry_point() 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | urllib3==1.25.8 2 | requests==2.23.0 3 | m3u8==0.5.4 4 | termcolor==1.1.0 5 | pycryptodome==3.9.8 6 | setuptools==40.6.3 7 | PyYAML==5.3.1 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | from pathlib import Path 4 | 5 | versionfile = Path("./version") 6 | version = versionfile.read_text().split("\n")[0] 7 | [mainv, modulev, minorv] = version.split(".") 8 | 9 | VERSION = (int(mainv), int(modulev), int(minorv)) 10 | __version__ = ".".join(map(str, VERSION[0:3])) 11 | __description__ = """this is a description""" 12 | __author__ = "zk" 13 | __author_email__ = "liuzq7@gmail.com" 14 | __homepage__ = "https://github.com/zk4/m3u8_dl" 15 | __download_url__ = "%s/archive/master.zip" % __homepage__ 16 | __license__ = "BSD" 17 | 18 | 19 | if __name__ == "__main__": 20 | setup( 21 | # used in pip install and uninstall 22 | # pip install modulename 23 | name="m3_dl", 24 | version=__version__, 25 | author=__author__, 26 | author_email=__author_email__, 27 | url=__homepage__, 28 | description=__description__, 29 | long_description=open("README.md", "r", encoding="utf-8").read().strip(), 30 | long_description_content_type="text/markdown", 31 | download_url=__download_url__, 32 | license=__license__, 33 | python_requires=">3.7.0", 34 | zip_safe=False, 35 | packages=find_packages(exclude=["tests", "tests.*"]), 36 | package_data={"m3_dl.logx": ["logging.yaml"], "m3_dl": ["../version"]}, 37 | install_requires=open("requirements.txt", "r").read().strip().split(), 38 | entry_points={"console_scripts": ["m3_dl = m3_dl:entry_point"]}, 39 | classifiers=[ 40 | "Development Status :: 5 - Production/Stable", 41 | "Environment :: Console", 42 | ], 43 | keywords=("best practice for python project"), 44 | ) 45 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zk4/m3u8_dl/c559f56f08cc07d2ac0f34843da401b6ef7909c2/tests/__init__.py -------------------------------------------------------------------------------- /tests/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zk4/m3u8_dl/c559f56f08cc07d2ac0f34843da401b6ef7909c2/tests/common/__init__.py -------------------------------------------------------------------------------- /tests/test_args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zk4/m3u8_dl/c559f56f08cc07d2ac0f34843da401b6ef7909c2/tests/test_args.py -------------------------------------------------------------------------------- /tests/test_m3u8.py: -------------------------------------------------------------------------------- 1 | import m3u8 2 | 3 | 4 | def test_load(): 5 | m3u8_obj = m3u8.load( 6 | "./index-v1-a1.m3u8" 7 | ) # this could also be an absolute filename 8 | print(m3u8_obj.segments) 9 | print(m3u8_obj.target_duration) 10 | print(m3u8_obj.keys) 11 | 12 | 13 | # if you already have the content as string, use 14 | 15 | 16 | def test_load_non_exist_file(): 17 | m3u8_obj = m3u8.load("./non_exist.m3u8") # this could also be an absolute filename 18 | print(m3u8_obj.segments) 19 | print(m3u8_obj.target_duration) 20 | print(m3u8_obj.keys) 21 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | 0.0.39 2 | 745aae801ae0d993d851297a0dceaf7ce004fd8e -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from pathlib import Path 3 | import subprocess 4 | import sys 5 | 6 | W = "\033[0m" # white (normal) 7 | R = "\033[31m" # red 8 | G = "\033[32m" # green 9 | 10 | try: 11 | head = subprocess.check_output(["git", "rev-parse", "HEAD"]) 12 | except Exception as e: 13 | print(R + "not git repository") 14 | print("make you first git commit!" + W) 15 | sys.exit(0) 16 | 17 | current_commit = head.strip().decode("utf-8") 18 | 19 | versionfile = Path("./version") 20 | 21 | lines = versionfile.read_text().split("\n") 22 | 23 | version = lines[0] 24 | commit = lines[1] 25 | [mainv, modulev, minorv] = version.split(".") 26 | if commit != current_commit: 27 | minorv = 1 + int(minorv) 28 | print("bump minor version") 29 | else: 30 | print("No git update ,no version change! bye!") 31 | sys.exit(0) 32 | 33 | newversion = f"{mainv}.{modulev}.{minorv}" 34 | versionfile.write_text(newversion + "\n" + current_commit) 35 | print(G + f"{version} -> {newversion}" + W) 36 | --------------------------------------------------------------------------------