├── tiktok_dl ├── extractors │ ├── __init__.py │ ├── extractor.py │ └── extractor_20200623.py ├── version.py ├── __init__.py ├── schema.py ├── logger.py ├── cli.py ├── validator.py ├── archive.py ├── tiktok_dl.py ├── schemas │ └── 2020-06-23.json ├── utils.py ├── options.py └── downloader.py ├── docs ├── authors.rst ├── history.rst ├── readme.rst ├── contributing.rst ├── usage.rst ├── index.rst ├── Makefile ├── make.bat ├── installation.rst └── conf.py ├── tests ├── __init__.py └── test_tiktok_dl.py ├── HISTORY.rst ├── requirements.txt ├── AUTHORS.rst ├── .coveragerc ├── MANIFEST.in ├── requirements_dev.txt ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── continuous-integration-pip.yml │ └── continuous-integration-publish.yml ├── setup.cfg ├── utils ├── readme.py └── template.rst ├── tox.ini ├── .readthedocs.yml ├── .pyup.yml ├── .pre-commit-config.yaml ├── LICENSE ├── setup.py ├── .gitignore ├── Makefile ├── CONTRIBUTING.rst └── README.rst /tiktok_dl/extractors/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Unit test package for tiktok_dl.""" 2 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Usage 3 | ===== 4 | 5 | To use tiktok-dl in a project:: 6 | 7 | import tiktok_dl 8 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 0.1.0 (2020-06-21) 6 | ------------------ 7 | 8 | * First release on PyPI. 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | colorama==0.4.3 2 | jsonschema==3.2.0 3 | loguru==0.5.1 4 | requests==2.24.0 5 | sanic==20.3.0 6 | tqdm==4.46.1 7 | -------------------------------------------------------------------------------- /tiktok_dl/version.py: -------------------------------------------------------------------------------- 1 | """Metadata for tiktok-dl.""" 2 | 3 | __author__ = """Aakash Gajjar""" 4 | __email__ = "skyqutip@gmail.com" 5 | __version__ = "0.1.0" 6 | -------------------------------------------------------------------------------- /tiktok_dl/__init__.py: -------------------------------------------------------------------------------- 1 | """Top-level package for tiktok-dl.""" 2 | # For relative imports to work in Python 3.6 3 | import os 4 | import sys 5 | 6 | sys.path.append(os.path.dirname(os.path.realpath(__file__))) 7 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Aakash Gajjar 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | def __repr__ 5 | if self.debug: 6 | if settings.DEBUG 7 | raise AssertionError 8 | raise NotImplementedError 9 | if 0: 10 | if __name__ == .__main__.: 11 | 12 | [run] 13 | omit = 14 | .eggs/* 15 | venv/* 16 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 12 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | black==19.10b0 2 | blacken-docs==1.7.0 3 | bump2version==1.0.0 4 | coverage==5.1 5 | pip-upgrader 6 | pre-commit==2.5.1 7 | pydocstyle==5.0.2 8 | pytest==5.4.3 9 | pytest-runner==5.2 10 | reorder_python_imports==2.3.1 11 | Sphinx==3.1.1 12 | tox==3.15.2 13 | twine==3.1.1 14 | watchdog==0.10.2 15 | wheel==0.34.2 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * tiktok-dl version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to tiktok-dl's documentation! 2 | ====================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | readme 9 | installation 10 | usage 11 | modules 12 | contributing 13 | authors 14 | history 15 | 16 | Indices and tables 17 | ================== 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:tiktok_dl/_version.py] 7 | search = __version__ = '{current_version}' 8 | replace = __version__ = '{new_version}' 9 | 10 | [bdist_wheel] 11 | universal = 1 12 | 13 | [flake8] 14 | exclude = docs 15 | 16 | [aliases] 17 | # Define setup.py command aliases here 18 | test = pytest 19 | 20 | [tool:pytest] 21 | collect_ignore = ['setup.py'] 22 | -------------------------------------------------------------------------------- /tiktok_dl/schema.py: -------------------------------------------------------------------------------- 1 | """Schema Collection for TikTok Video JSON.""" 2 | import json 3 | import os 4 | 5 | 6 | def parse_json(direcroy, filename): 7 | with open(os.path.join(direcroy, filename), "r", encoding="utf8") as f: 8 | return json.load(f) 9 | 10 | 11 | def schemas(): 12 | """Read Schemas from available schemas.""" 13 | directory = os.path.dirname(os.path.realpath(__file__)) 14 | return [ 15 | parse_json(directory, i) 16 | for i in os.listdir(os.path.join(directory, "schemas")) 17 | if ".json" in i 18 | ] 19 | -------------------------------------------------------------------------------- /utils/readme.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | with open("utils/template.rst", "r") as f: 4 | content = f.read() 5 | 6 | with open("README.rst", "w") as file: 7 | 8 | def strip_empty(e): 9 | if len(e) == 0: 10 | return e 11 | return " " + e 12 | 13 | cmd_out = subprocess.run( 14 | ["venv/Scripts/python.exe", "tiktok_dl\\cli.py", "--help"], 15 | stdout=subprocess.PIPE, 16 | ) 17 | args = [strip_empty(i) for i in cmd_out.stdout.decode("utf-8").split("\n")] 18 | file.write(content.replace("{{COMMAND_ARGS}}", "".join(args))) 19 | -------------------------------------------------------------------------------- /tests/test_tiktok_dl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Tests for `tiktok_dl` package.""" 3 | import pytest 4 | 5 | from tiktok_dl import tiktok_dl 6 | 7 | 8 | @pytest.fixture 9 | def response(): 10 | """Sample pytest fixture. 11 | 12 | See more at: http://doc.pytest.org/en/latest/fixture.html 13 | """ 14 | # import requests 15 | # return requests.get('https://github.com/audreyr/cookiecutter-pypackage') 16 | 17 | 18 | def test_content(response): 19 | """Sample pytest test function with the pytest fixture as an argument.""" 20 | # from bs4 import BeautifulSoup 21 | # assert 'GitHub' in BeautifulSoup(response.content).title.string 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py37, py38, black 3 | 4 | [travis] 5 | python = 6 | 3.8: py38 7 | 3.7: py37 8 | 3.6: py36 9 | 3.5: py35 10 | 11 | [testenv:black] 12 | basepython = python 13 | deps = black 14 | commands = black -t py38 setup.py tiktok_dl tests 15 | 16 | [testenv] 17 | setenv = 18 | PYTHONPATH = {toxinidir} 19 | deps = 20 | -r{toxinidir}/requirements_dev.txt 21 | ; If you want to make tox run the tests with the same versions, create a 22 | ; requirements.txt with the pinned versions and uncomment the following line: 23 | ; -r{toxinidir}/requirements.txt 24 | commands = 25 | pip install -U pip 26 | pytest --basetemp={envtmpdir} 27 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = tiktok_dl 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | 12 | # Build documentation with MkDocs 13 | #mkdocs: 14 | # configuration: mkdocs.yml 15 | 16 | # Optionally build your docs in additional formats such as PDF 17 | formats: 18 | - pdf 19 | 20 | # Optionally set the version of Python and requirements required to build your docs 21 | python: 22 | version: 3.8 23 | install: 24 | - requirements: requirements.txt 25 | system_packages: true 26 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # configure updates globally 2 | # default: all 3 | # allowed: all, insecure, False 4 | update: all 5 | 6 | # configure dependency pinning globally 7 | # default: True 8 | # allowed: True, False 9 | pin: True 10 | 11 | # set the default branch 12 | # default: empty, the default branch on GitHub 13 | # branch: dev 14 | 15 | # update schedule 16 | # default: empty 17 | # allowed: "every day", "every week", .. 18 | schedule: "every day" 19 | 20 | # search for requirement files 21 | # default: True 22 | # allowed: True, False 23 | search: True 24 | 25 | # Specify requirement files by hand, default is empty 26 | # default: empty 27 | # allowed: list 28 | requirements: 29 | - requirements.txt 30 | - requirements_dev.txt 31 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.4.0 6 | hooks: 7 | - id: detect-private-key 8 | - id: end-of-file-fixer 9 | - id: trailing-whitespace 10 | - id: requirements-txt-fixer 11 | - id: check-added-large-files 12 | - id: check-yaml 13 | - repo: https://github.com/asottile/reorder_python_imports 14 | rev: v2.3.0 15 | hooks: 16 | - id: reorder-python-imports 17 | - repo: https://github.com/psf/black 18 | rev: 19.10b0 19 | hooks: 20 | - id: black 21 | - repo: https://github.com/asottile/blacken-docs 22 | rev: v1.7.0 23 | hooks: 24 | - id: blacken-docs 25 | additional_dependencies: [black==19.10b0] 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=tiktok_dl 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /tiktok_dl/extractors/extractor.py: -------------------------------------------------------------------------------- 1 | """Extractor for extracting data from JSON.""" 2 | from tiktok_dl.extractors.extractor_20200623 import Extractor20200623 3 | 4 | 5 | class ExtractorError(Exception): 6 | """No Extractor found. 7 | 8 | Args: 9 | Exception (Exception): If no Extractor found. 10 | """ 11 | 12 | pass 13 | 14 | 15 | class Extractor: 16 | """Extract TikTok Video JSON.""" 17 | 18 | def extract(self, json_data: dict, version: str): 19 | """Extract data from json_data with json schema version. 20 | 21 | Args: 22 | json_data (dict): Data to be extracted from. 23 | version (str): Version of the json schema to use. 24 | 25 | Returns: 26 | dict: Extracted json data. 27 | """ 28 | for extractor in [Extractor20200623()]: 29 | if extractor.__class__.version() == version: 30 | return (version, extractor.__class__.extract(json_data)) 31 | 32 | raise ExtractorError("Unable to extract from json_data") 33 | -------------------------------------------------------------------------------- /.github/workflows/continuous-integration-pip.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: [3.8] 12 | 13 | steps: 14 | - uses: actions/cache@v2 15 | with: 16 | path: ~/.cache/pip 17 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 18 | restore-keys: | 19 | ${{ runner.os }}-pip- 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | pip install pytest 29 | pip install pytest-cov 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | - name: Test with pytest 32 | run: | 33 | pytest --cov=./ --cov-report=xml 34 | - name: Upload coverage to Codecov 35 | uses: codecov/codecov-action@v1 36 | -------------------------------------------------------------------------------- /tiktok_dl/logger.py: -------------------------------------------------------------------------------- 1 | """Logger Class.""" 2 | from loguru import logger 3 | 4 | 5 | class Logger: 6 | """Wrapper for logger.""" 7 | 8 | def __init__( 9 | self, no_warnings=False, quiet=False, verbose=True, 10 | ): 11 | """Set class options. 12 | 13 | Args: 14 | no_warnings (bool, optional): Do not log any warnings. Defaults to False. 15 | quiet (bool, optional): Do not print anything. Defaults to False. 16 | verbose (bool, optional): Be verbose. Defaults to True. 17 | """ 18 | 19 | self.no_warnings = no_warnings 20 | self.quiet = quiet 21 | self.verbose = verbose 22 | 23 | def debug(self, *args): 24 | if self.verbose: 25 | logger.debug(*args) 26 | 27 | def info(self, *args): 28 | if self.verbose: 29 | logger.info(*args) 30 | 31 | def warning(self, *args): 32 | if not self.no_warnings or not self.quiet: 33 | logger.warning(*args) 34 | 35 | def error(self, *args): 36 | if self.verbose or not self.quiet: 37 | logger.error(*args) 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020, Aakash Gajjar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/continuous-integration-publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.*' 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: [3.8] 15 | 16 | steps: 17 | - uses: actions/cache@v2 18 | with: 19 | path: ~/.cache/pip 20 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 21 | restore-keys: | 22 | ${{ runner.os }}-pip- 23 | - uses: actions/checkout@v2 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install setuptools 32 | pip install wheel 33 | pip install twine 34 | - name: Build and publish 35 | env: 36 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 37 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 38 | run: | 39 | python setup.py sdist bdist_wheel 40 | twine upload dist/* 41 | -------------------------------------------------------------------------------- /tiktok_dl/cli.py: -------------------------------------------------------------------------------- 1 | """Console script for tiktok_dl.""" 2 | import os 3 | import sys 4 | 5 | from loguru import logger 6 | 7 | from tiktok_dl.options import options_parser 8 | from tiktok_dl.tiktok_dl import TikTokDownloader 9 | 10 | 11 | def main(): 12 | """Console script for tiktok_dl.""" 13 | parser = options_parser() 14 | args = parser.parse_args() 15 | 16 | if len(args.urls) == 0 and args.batch_file is None: 17 | parser.error("URL or file containing list of URLs (--batch-file) is required.") 18 | 19 | if args.batch_file is not None and os.path.isfile(args.batch_file): 20 | with open(args.batch_file, "r") as f: 21 | for url in f.read().split("\n"): 22 | if len(url.strip()) > 0: 23 | args.urls.append(url.strip()) 24 | 25 | logger.info("Downloading {} urls", len(args.urls)) 26 | 27 | tiktok = TikTokDownloader(args) 28 | tiktok.process_urls() 29 | 30 | print("Arguments: " + str(args)) 31 | print("Replace this message by putting your code into " "tiktok_dl.cli.main") 32 | return 0 33 | 34 | 35 | if __name__ == "__main__": 36 | sys.exit(main()) # pragma: no cover 37 | -------------------------------------------------------------------------------- /utils/template.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | tiktok-dl: TikTok Video Downloader 3 | ================================== 4 | 5 | 6 | .. image:: https://img.shields.io/pypi/v/tiktok_dl.svg 7 | :target: https://pypi.python.org/pypi/tiktok_dl 8 | 9 | .. image:: https://img.shields.io/travis/skyme5/tiktok_dl.svg 10 | :target: https://travis-ci.com/skyme5/tiktok_dl 11 | 12 | .. image:: https://readthedocs.org/projects/tiktok-dl/badge/?version=latest 13 | :target: https://tiktok-dl.readthedocs.io/en/latest/?badge=latest 14 | :alt: Documentation Status 15 | 16 | .. image:: https://pyup.io/repos/github/skyme5/tiktok_dl/shield.svg 17 | :target: https://pyup.io/repos/github/skyme5/tiktok_dl/ 18 | :alt: Updates 19 | 20 | 21 | Usage 22 | ----- 23 | 24 | .. code-block:: text 25 | 26 | {{COMMAND_ARGS}} 27 | 28 | 29 | * Free software: MIT license 30 | * Documentation: https://tiktok-dl.readthedocs.io. 31 | 32 | 33 | Features 34 | -------- 35 | 36 | * TODO 37 | 38 | Credits 39 | ------- 40 | 41 | This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. 42 | 43 | .. _Cookiecutter: https://github.com/audreyr/cookiecutter 44 | .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage 45 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Installation 5 | ============ 6 | 7 | 8 | Stable release 9 | -------------- 10 | 11 | To install tiktok-dl, run this command in your terminal: 12 | 13 | .. code-block:: console 14 | 15 | $ pip install tiktok-dl 16 | 17 | This is the preferred method to install tiktok-dl, as it will always install the most recent stable release. 18 | 19 | If you don't have `pip`_ installed, this `Python installation guide`_ can guide 20 | you through the process. 21 | 22 | .. _pip: https://pip.pypa.io 23 | .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ 24 | 25 | 26 | From sources 27 | ------------ 28 | 29 | The sources for tiktok-dl can be downloaded from the `Github repo`_. 30 | 31 | You can either clone the public repository: 32 | 33 | .. code-block:: console 34 | 35 | $ git clone git://github.com/skyme5/tiktok-dl 36 | 37 | Or download the `tarball`_: 38 | 39 | .. code-block:: console 40 | 41 | $ curl -OJL https://github.com/skyme5/tiktok-dl/tarball/master 42 | 43 | Once you have a copy of the source, you can install it with: 44 | 45 | .. code-block:: console 46 | 47 | $ python setup.py install 48 | 49 | 50 | .. _Github repo: https://github.com/skyme5/tiktok-dl 51 | .. _tarball: https://github.com/skyme5/tiktok-dl/tarball/master 52 | -------------------------------------------------------------------------------- /tiktok_dl/validator.py: -------------------------------------------------------------------------------- 1 | """Schema Validator for TikTok Video JSON.""" 2 | from jsonschema import validate 3 | from jsonschema import ValidationError 4 | from loguru import logger 5 | 6 | from tiktok_dl.schema import schemas 7 | 8 | 9 | class AwemeValidator: 10 | """Validate Schema for TikTok Video JSON.""" 11 | 12 | def __init__(self): 13 | """Initialize schemas.""" 14 | self.schemas = schemas() 15 | 16 | def validate(self, json_data: dict): 17 | """Validate json_data. 18 | 19 | This will try validating from collection of schema. 20 | 21 | Args: 22 | str (dict): json data to validate. 23 | 24 | Returns: 25 | bool: True If Scheme validation was a success. 26 | str : Version of the Schema that was validated. 27 | """ 28 | for json_schema in self.schemas.reverse(): 29 | try: 30 | validate(instance=json_data, schema=json_schema.get("SCHEMA")) 31 | logger.debug( 32 | "Schema Validation success with " + json_schema.get("VERSION") 33 | ) 34 | 35 | return (True, json_schema.get("VERSION")) 36 | except ValidationError as e: 37 | pass 38 | logger.warning("No valid schema exists for given json data") 39 | 40 | return (False, None) 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """The setup script.""" 3 | from setuptools import find_packages 4 | from setuptools import setup 5 | 6 | from tiktok_dl.version import __version__ 7 | 8 | with open("README.rst") as readme_file: 9 | readme = readme_file.read() 10 | 11 | with open("HISTORY.rst") as history_file: 12 | history = history_file.read() 13 | 14 | requirements = [] 15 | 16 | setup_requirements = [ 17 | "pytest-runner", 18 | ] 19 | 20 | test_requirements = [ 21 | "pytest>=3", 22 | ] 23 | 24 | setup( 25 | name="tiktok-dl", 26 | version=__version__, 27 | author="Aakash Gajjar", 28 | author_email="skyqutip@gmail.com", 29 | url="https://github.com/skyme5/tiktok-dl", 30 | description="TikTok Video Downloader", 31 | long_description=readme + "\n\n" + history, 32 | entry_points={"console_scripts": ["tiktok-dl=tiktok_dl.cli:main",],}, 33 | include_package_data=True, 34 | packages=find_packages(include=["tiktok_dl", "tiktok_dl.*"]), 35 | install_requires=requirements, 36 | setup_requires=setup_requirements, 37 | test_suite="tests", 38 | tests_require=test_requirements, 39 | keywords="tiktok-dl", 40 | python_requires=">=3.5", 41 | classifiers=[ 42 | "Development Status :: 2 - Pre-Alpha", 43 | "Intended Audience :: Developers", 44 | "License :: OSI Approved :: MIT License", 45 | "Natural Language :: English", 46 | "Programming Language :: Python :: 3", 47 | "Programming Language :: Python :: 3.5", 48 | "Programming Language :: Python :: 3.6", 49 | "Programming Language :: Python :: 3.7", 50 | "Programming Language :: Python :: 3.8", 51 | ], 52 | license="MIT license", 53 | zip_safe=False, 54 | ) 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | # IDE settings 105 | .vscode/ 106 | -------------------------------------------------------------------------------- /tiktok_dl/archive.py: -------------------------------------------------------------------------------- 1 | """Archive Manager for tiktok_dl.""" 2 | 3 | 4 | class ArchiveManager: 5 | """Manage Archive file containing ids of downloaded TikTok Videos.""" 6 | 7 | def __init__(self, download_archive=None): 8 | """Initialize Archive Manager. 9 | 10 | 1. Read archive list from disc. 11 | 2. Open file object for archive list 12 | 13 | Args: 14 | download_archive (str, optional): File path of the local archive. Defaults to None. 15 | """ 16 | self.archive_path = download_archive 17 | self.enable_archive = download_archive is not None 18 | self.archive_file = self._open() 19 | self.list = self._init_archive() 20 | 21 | def _open(self): 22 | """Open archive file for reading+writing.""" 23 | if self.enable_archive: 24 | return open(self.archive_path, "r+", encoding="utf-8") 25 | return None 26 | 27 | def _init_archive(self): 28 | """Read local archive file.""" 29 | return self.archive_file.read().strip().split("\n") 30 | 31 | def _update_archive(self, video_id: str): 32 | self.list.append(video_id) 33 | if self.enable_archive: 34 | self.archive_file.write("%s\n" % video_id) 35 | 36 | def recorded(self, video_id: str): 37 | """Check if the video_id exists in the Archive?. 38 | 39 | Args: 40 | video_id (str): id of the TikTok Video 41 | 42 | Returns: 43 | bool: True if exists in archive 44 | """ 45 | return video_id in self.list 46 | 47 | def append(self, video_id: str): 48 | """Record video_id to the archive. 49 | 50 | Args: 51 | video_id (str): id of the TikTok Video 52 | """ 53 | self._update_archive(video_id) 54 | 55 | def close(self): 56 | """Close Archive Manager.""" 57 | if self.enable_archive: 58 | self.archive_file.close() 59 | -------------------------------------------------------------------------------- /tiktok_dl/tiktok_dl.py: -------------------------------------------------------------------------------- 1 | """Main module.""" 2 | from tiktok_dl.downloader import Downloader 3 | from tiktok_dl.extractors.extractor import Extractor 4 | from tiktok_dl.logger import Logger 5 | from tiktok_dl.validator import AwemeValidator 6 | 7 | 8 | class TikTokDownloader: 9 | """TikTok Downloader Class.""" 10 | 11 | def __init__(self, options): 12 | """Initialize validator, extractor, logger and downloader. 13 | 14 | Args: 15 | options (dict): Dictionary of command-line options. 16 | """ 17 | self.options = options 18 | self.validator = AwemeValidator() 19 | self.extractor = Extractor() 20 | self.logger = Logger( 21 | no_warnings=self.options.no_warnings, 22 | quiet=self.options.quiet, 23 | verbose=self.options.verbose, 24 | ) 25 | self.downloader = Downloader( 26 | validator=self.validator, 27 | extractor=self.extractor, 28 | logger=self.logger, 29 | directory_prefix=self.options.directory_prefix, 30 | dump_json=self.options.dump_json, 31 | max_sleep_interval=self.options.max_sleep_interval, 32 | no_check_certificate=self.options.no_check_certificate, 33 | no_overwrite=self.options.no_overwrite, 34 | no_write_json=self.options.no_write_json, 35 | output_template=self.options.output_template, 36 | print_json=self.options.print_json, 37 | simulate=self.options.simulate, 38 | skip_download=self.options.skip_download, 39 | sleep_interval=self.options.sleep_interval, 40 | write_description=self.options.write_description, 41 | write_thumbnail=self.options.write_thumbnail, 42 | ) 43 | 44 | def download(self, url): 45 | def queue(url): 46 | pass 47 | 48 | queue(url) 49 | 50 | def process_urls(self): 51 | for url in self.options.urls: 52 | self.download(url) 53 | -------------------------------------------------------------------------------- /tiktok_dl/schemas/2020-06-23.json: -------------------------------------------------------------------------------- 1 | { 2 | "NAME": "2020-06-23", 3 | "VERSION": "0.0.1", 4 | "SCHEMA": { 5 | "type": "object", 6 | "properties": { 7 | "id": { "type": "string" }, 8 | "play_urls": { "type": "array" }, 9 | "ext": { "type": "string" }, 10 | "width": { "type": "number" }, 11 | "height": { "type": "number" }, 12 | "duration": { "type": "number" }, 13 | "thumbnails": { "type": "array" }, 14 | "comment_count": { "type": "number" }, 15 | "digg_count": { "type": "number" }, 16 | "share_count": { "type": "number" }, 17 | "play_count": { "type": "number" }, 18 | "create_time": { "type": "number" }, 19 | "upload_date": { "type": "string" }, 20 | "title": { "type": "string" }, 21 | "description": { "type": "string" }, 22 | "nick_name": { "type": "string" }, 23 | "unique_id": { "type": "string" }, 24 | "sec_uid": { "type": "string" }, 25 | "user_id": { "type": "string" }, 26 | "user_url": { "type": "string" }, 27 | "profile_pics": { "type": "array" }, 28 | "webpage_url": { "type": "string" }, 29 | "follower_count": { "type": "number" }, 30 | "heart_total": { "type": "string" }, 31 | "challenge_list": { "type": "array" }, 32 | "duet_info": { "type": "string" }, 33 | "text_extra": { "type": "array" }, 34 | "music_id": { "type": "string" }, 35 | "music_title": { "type": "string" }, 36 | "music_artist": { "type": "string" }, 37 | "music_covers": { "type": "array" } 38 | }, 39 | "required": [ 40 | "challenge_list", 41 | "comment_count", 42 | "create_time", 43 | "description", 44 | "digg_count", 45 | "duet_info", 46 | "duration", 47 | "ext", 48 | "follower_count", 49 | "heart_total", 50 | "height", 51 | "id", 52 | "music_artist", 53 | "music_covers", 54 | "music_id", 55 | "music_title", 56 | "nick_name", 57 | "play_count", 58 | "play_urls", 59 | "profile_pics", 60 | "sec_uid", 61 | "share_count", 62 | "text_extra", 63 | "thumbnails", 64 | "title", 65 | "unique_id", 66 | "upload_date", 67 | "user_id", 68 | "user_url", 69 | "webpage_url", 70 | "width" 71 | ] 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | 4 | define BROWSER_PYSCRIPT 5 | import os, webbrowser, sys 6 | 7 | from urllib.request import pathname2url 8 | 9 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 10 | endef 11 | export BROWSER_PYSCRIPT 12 | 13 | define PRINT_HELP_PYSCRIPT 14 | import re, sys 15 | 16 | for line in sys.stdin: 17 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 18 | if match: 19 | target, help = match.groups() 20 | print("%-20s %s" % (target, help)) 21 | endef 22 | export PRINT_HELP_PYSCRIPT 23 | 24 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 25 | 26 | help: 27 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 28 | 29 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 30 | 31 | clean-build: ## remove build artifacts 32 | rm -fr build/ 33 | rm -fr dist/ 34 | rm -fr .eggs/ 35 | find . -name '*.egg-info' -exec rm -rf {} + 36 | find . -name '*.egg' -exec rm -rf {} + 37 | 38 | clean-pyc: ## remove Python file artifacts 39 | find . -name '*.pyc' -exec rm -f {} + 40 | find . -name '*.pyo' -exec rm -f {} + 41 | find . -name '*~' -exec rm -f {} + 42 | find . -name '__pycache__' -exec rm -fr {} + 43 | 44 | clean-test: ## remove test and coverage artifacts 45 | rm -fr .tox/ 46 | rm -f .coverage 47 | rm -fr htmlcov/ 48 | rm -fr .pytest_cache 49 | 50 | commit: 51 | @NO_STAG=$$(git status -s | grep AM | wc -l) && \ 52 | if [ "$$NO_STAG" -gt "0" ]; then\ 53 | echo -e "\033[1;33mUnstaged files detected, please stag these files.\033[0m";\ 54 | git status -s | grep --color=always AM;\ 55 | else\ 56 | pre-commit run;\ 57 | fi 58 | 59 | commit-add: 60 | git add . 61 | pre-commit run 62 | 63 | lint: ## check style with black 64 | black -t py38 setup.py tiktok_dl tests 65 | 66 | up-dep: 67 | pip-upgrade requirements.txt 68 | 69 | up-dev: 70 | pip-upgrade requirements_dev.txt 71 | 72 | up-all: 73 | pip-upgrade requirements.txt requirements_dev.txt 74 | 75 | test: ## run tests quickly with the default Python 76 | python utils/readme.py 77 | pytest 78 | 79 | test-all: ## run tests on every Python version with tox 80 | python utils/readme.py 81 | tox 82 | 83 | coverage: ## check code coverage quickly with the default Python 84 | coverage run --source tiktok_dl -m pytest 85 | coverage report -m 86 | coverage html 87 | $(BROWSER) htmlcov/index.html 88 | 89 | docs: ## generate Sphinx HTML documentation, including API docs 90 | rm -f docs/tiktok_dl.rst 91 | rm -f docs/modules.rst 92 | sphinx-apidoc -o docs/ tiktok_dl 93 | $(MAKE) -C docs clean 94 | $(MAKE) -C docs html 95 | $(BROWSER) docs/_build/html/index.html 96 | 97 | servedocs: docs ## compile the docs watching for changes 98 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 99 | 100 | release: dist ## package and upload a release 101 | twine upload dist/* 102 | 103 | dist: clean ## builds source and wheel package 104 | python setup.py sdist 105 | python setup.py bdist_wheel 106 | ls -l dist 107 | twine check dist/* 108 | 109 | install: clean ## install the package to the active Python's site-packages 110 | python setup.py install 111 | -------------------------------------------------------------------------------- /tiktok_dl/extractors/extractor_20200623.py: -------------------------------------------------------------------------------- 1 | """JSON Extractor Class.""" 2 | import datetime 3 | 4 | from tiktok_dl.utils import int_or_none 5 | from tiktok_dl.utils import str_or_none 6 | from tiktok_dl.utils import try_get 7 | 8 | 9 | class Extractor20200623: 10 | """JSON Extractor for 20200623.""" 11 | 12 | def version(self): 13 | return "0.0.1" 14 | 15 | def extract(self, video_data: dict): 16 | video_info = try_get(video_data, lambda x: x["videoData"]["itemInfos"], dict) 17 | author_info = try_get(video_data, lambda x: x["videoData"]["authorInfos"], dict) 18 | share_info = try_get(video_data, lambda x: x["shareMeta"], dict) 19 | music_info = try_get(video_data, lambda x: x["videoData"]["musicInfos"], dict) 20 | author_stats = try_get( 21 | video_data, lambda x: x["videoData"]["authorStats"], dict 22 | ) 23 | 24 | unique_id = str_or_none(author_info.get("uniqueId")) 25 | timestamp = try_get(video_info, lambda x: int(x["createTime"]), int) 26 | date = datetime.utcfromtimestamp(timestamp).strftime("%Y%m%d") 27 | 28 | height = try_get(video_info, lambda x: x["video"]["videoMeta"]["height"], int) 29 | width = try_get(video_info, lambda x: x["video"]["videoMeta"]["width"], int) 30 | 31 | return { 32 | "id": str_or_none(video_info.get("id")), 33 | "play_urls": try_get(video_info, lambda x: x["video"]["urls"], list), 34 | "ext": "mp4", 35 | "width": width, 36 | "height": height, 37 | "duration": try_get( 38 | video_info, lambda x: x["video"]["videoMeta"]["duration"], int 39 | ), 40 | "thumbnails": try_get(video_info, lambda x: x["covers"], list), 41 | "comment_count": int_or_none(video_info.get("commentCount")), 42 | "digg_count": int_or_none(video_info.get("diggCount")), 43 | "share_count": int_or_none(video_info.get("shareCount")), 44 | "play_count": int_or_none(video_info.get("playCount")), 45 | "create_time": timestamp, 46 | "upload_date": date, 47 | "title": "{} on TikTok".format(str_or_none(author_info.get("nickName"))), 48 | "description": str_or_none(share_info.get("desc")), 49 | "nick_name": str_or_none(author_info.get("nickName")), 50 | "unique_id": unique_id, 51 | "sec_uid": str_or_none(author_info.get("secUid")), 52 | "user_id": str_or_none(author_info.get("userId")), 53 | "user_url": "https://www.tiktok.com/@" + unique_id, 54 | "profile_pics": try_get(author_info, lambda x: x["covers"], list), 55 | "webpage_url": "https://www.tiktok.com/@{}/video/{}?source=h5_t".format( 56 | str_or_none(author_info.get("uniqueId")), 57 | str_or_none(video_info.get("id")), 58 | ), 59 | "follower_count": int_or_none(author_stats.get("followerCount")), 60 | "heart_total": str_or_none(author_stats.get("heartCount")), 61 | "challenge_list": try_get( 62 | video_data, lambda x: x["videoData"]["challengeInfoList"], list 63 | ), 64 | "duet_info": try_get(video_data, lambda x: x["videoData"]["duetInfo"], str), 65 | "text_extra": try_get( 66 | video_data, lambda x: x["videoData"]["textExtra"], list 67 | ), 68 | "music_id": str_or_none(music_info.get("musicId")), 69 | "music_title": str_or_none(music_info.get("musicName")), 70 | "music_artist": str_or_none(music_info.get("authorName")), 71 | "music_covers": try_get(music_info, lambda x: x["covers"], list), 72 | } 73 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/skyme5/tiktok-dl/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" and "help 30 | wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | tiktok-dl could always use more documentation, whether as part of the 42 | official tiktok-dl docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/skyme5/tiktok-dl/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `tiktok-dl` for local development. 61 | 62 | 1. Fork the `tiktok-dl` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/tiktok-dl.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv tiktok-dl 70 | $ cd tiktok-dl/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass black and the 80 | tests, including testing other Python versions with tox:: 81 | 82 | $ black tiktok-dl tests 83 | $ python setup.py test or pytest 84 | $ tox 85 | 86 | To get black and tox, just pip install them into your virtualenv. 87 | 88 | 6. Commit your changes and push your branch to GitHub:: 89 | 90 | $ git add . 91 | $ git commit -m "Your detailed description of your changes." 92 | $ git push origin name-of-your-bugfix-or-feature 93 | 94 | 7. Submit a pull request through the GitHub website. 95 | 96 | Pull Request Guidelines 97 | ----------------------- 98 | 99 | Before you submit a pull request, check that it meets these guidelines: 100 | 101 | 1. The pull request should include tests. 102 | 2. If the pull request adds functionality, the docs should be updated. Put 103 | your new functionality into a function with a docstring, and add the 104 | feature to the list in README.rst. 105 | 3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. Check 106 | https://travis-ci.com/skyme5/tiktok-dl/pull_requests 107 | and make sure that the tests pass for all supported Python versions. 108 | 109 | Tips 110 | ---- 111 | 112 | To run a subset of tests:: 113 | 114 | $ pytest tests.test_tiktok_dl 115 | 116 | 117 | Deploying 118 | --------- 119 | 120 | A reminder for the maintainers on how to deploy. 121 | Make sure all your changes are committed (including an entry in HISTORY.rst). 122 | Then run:: 123 | 124 | $ bump2version patch # possible: major / minor / patch 125 | $ git push 126 | $ git push --tags 127 | 128 | Travis will then deploy to PyPI if tests pass. 129 | -------------------------------------------------------------------------------- /tiktok_dl/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import datetime 3 | 4 | from loguru import logger 5 | from requests.exceptions import InvalidURL 6 | 7 | 8 | def format_utctime(time: int, fmt: str) -> str: 9 | """Format unixtimestamp to custom time format string. 10 | 11 | Args: 12 | time (int): unixtimestamp. 13 | fmt (str): time format string. 14 | 15 | Returns: 16 | str: unixtimestamp formatted to custom fmt. 17 | """ 18 | return datetime.utcfromtimestamp(time).strftime(fmt) 19 | 20 | 21 | def search_regex( 22 | pattern, string: str, name: str, default=object(), fatal=True, flags=0, group=None 23 | ): 24 | """Perform a regex search on the given string, using a single or a list of patterns returning the first matching group. 25 | 26 | In case of failure return a default value or raise a WARNING or a 27 | RegexNotFoundError, depending on fatal, specifying the field name. 28 | """ 29 | if isinstance(pattern, (str, type(re.compile("")))): 30 | mobj = re.search(pattern, string, flags) 31 | else: 32 | for p in pattern: 33 | mobj = re.search(p, string, flags) 34 | if mobj: 35 | break 36 | 37 | if mobj: 38 | if group is None: 39 | # return the first matching group 40 | return next(g for g in mobj.groups() if g is not None) 41 | else: 42 | return mobj.group(group) 43 | elif default is not default: 44 | return default 45 | elif fatal: 46 | raise re.error("Unable to extract %s" % name) 47 | else: 48 | logger.error("unable to extract {}", name) 49 | return None 50 | 51 | 52 | def valid_url_re(): 53 | """TikTok URL RegExp. 54 | 55 | Captures id of the TikTok Video. 56 | """ 57 | return re.compile( 58 | r"https?://www\.tiktokv?\.com/(?:@[\w\._]+|share)/video/(?P\d+)" 59 | ) 60 | 61 | 62 | def match_id(url: str, valid_re): 63 | """Get id of the TikTok Video. 64 | 65 | Args: 66 | url (str): TikTok Video URL. 67 | valid_re (re): Instance of re. 68 | 69 | Raises: 70 | InvalidURL: Given url is Invalid. 71 | re.error: RegExp was unable to extract any id. 72 | 73 | Returns: 74 | str: id of the TikTok Video. 75 | """ 76 | m = valid_re.match(url) 77 | if m is None: 78 | raise InvalidURL("Url is invalid {}".format(url)) 79 | if m.group("id") is None: 80 | raise re.error("unable to find video id {}".format(url)) 81 | 82 | return str(m.group("id")) 83 | 84 | 85 | def try_get(src, getter, expected_type=None): 86 | """Getter for Object with type checking. 87 | 88 | Args: 89 | src (object): Object for getter. 90 | getter (lambda): Lambda expression for getting item from Object. 91 | expected_type (type, optional): Expected type from the getter. Defaults to None. 92 | 93 | Returns: 94 | expected_type: Value of getter for Object. 95 | """ 96 | if not isinstance(getter, (list, tuple)): 97 | getter = [getter] 98 | for get in getter: 99 | try: 100 | v = get(src) 101 | except (AttributeError, KeyError, TypeError, IndexError): 102 | pass 103 | else: 104 | if expected_type is None or isinstance(v, expected_type): 105 | return v 106 | 107 | 108 | def str_or_none(v, default=None): 109 | """Check if str.""" 110 | return default if v is None else str(v) 111 | 112 | 113 | def int_or_none(v, default=None, get_attr=None): 114 | """Check if input is int. 115 | 116 | Args: 117 | v (int): Input to check. 118 | default (type, optional): Expected type of get_attr. Defaults to None. 119 | get_attr (getter, optional): Getter to use. Defaults to None. 120 | 121 | Returns: 122 | int or None: Return int if valid or None. 123 | """ 124 | if get_attr: 125 | if v is not None: 126 | v = getattr(v, get_attr, None) 127 | if v == "": 128 | v = None 129 | if v is None: 130 | return default 131 | try: 132 | return int(v) 133 | except (ValueError, TypeError): 134 | return default 135 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | tiktok-dl: TikTok Video Downloader 3 | ================================== 4 | 5 | 6 | .. image:: https://img.shields.io/pypi/v/tiktok_dl.svg 7 | :target: https://pypi.python.org/pypi/tiktok_dl 8 | 9 | .. image:: https://img.shields.io/travis/skyme5/tiktok_dl.svg 10 | :target: https://travis-ci.com/skyme5/tiktok_dl 11 | 12 | .. image:: https://readthedocs.org/projects/tiktok-dl/badge/?version=latest 13 | :target: https://tiktok-dl.readthedocs.io/en/latest/?badge=latest 14 | :alt: Documentation Status 15 | 16 | .. image:: https://pyup.io/repos/github/skyme5/tiktok_dl/shield.svg 17 | :target: https://pyup.io/repos/github/skyme5/tiktok_dl/ 18 | :alt: Updates 19 | 20 | 21 | Usage 22 | ----- 23 | 24 | .. code-block:: text 25 | 26 | usage: cli.py [-h] [-V] [--download-archive DOWNLOAD_ARCHIVE] [-d] [-p CONCURRENT_COUNT] [-a FILENAME] [-o OUTPUT_TEMPLATE] [-w] [--write-description] [--no-write-json] [-P DIRECTORY_PREFIX] [--write-thumbnail] [-q] [--no-warnings] [-s] [--skip-download] [-g] [-e] [--get-id] [--get-thumbnail] [--get-description] [--get-duration] [--get-filename] [-j] [--print-json] [-v] [--no-check-certificate] [--sleep-interval SLEEP_INTERVAL] [--max-sleep-interval MAX_SLEEP_INTERVAL] [URL [URL ...]] TikTok Video downloader positional arguments: URL URL of the video optional arguments: -h, --help show this help message and exit -V, --version Print program version and exit Video Selection: --download-archive DOWNLOAD_ARCHIVE Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. Parallel Download: -d, --daemon Run as daemon. -p CONCURRENT_COUNT, --concurrent-count CONCURRENT_COUNT Download videos in parallel. Filesystem Options: -a FILENAME, --batch-file FILENAME File containing URLs to download ('-' for stdin), one URL per line. Lines starting with '#', ';' or ']' are considered as comments and ignored. -o OUTPUT_TEMPLATE, --output-template OUTPUT_TEMPLATE Output filename template, see the "OUTPUT TEMPLATE" for all the info. -w, --no-overwrite Do not overwrite files --write-description Write video description to a .description file. --no-write-json Write video metadata to a .json file. -P DIRECTORY_PREFIX, --directory-prefix DIRECTORY_PREFIX Directory prefix. Thumbnail images: --write-thumbnail Write thumbnail image to disk. Verbosity / Simulation Options:: -q, --quiet Activate quiet mode. --no-warnings Ignore warnings. -s, --simulate Do not download the video and do not write anything to disk. --skip-download Do not download the video. -g, --get-url Simulate, quiet but print URL. -e, --get-title Simulate, quiet but print title. --get-id Simulate, quiet but print id. --get-thumbnail Simulate, quiet but print thumbnail URL. --get-description Simulate, quiet but print video description. --get-duration Simulate, quiet but print video length. --get-filename Simulate, quiet but print output filename. -j, --dump-json Simulate, quiet but print JSON information. See the "OUTPUT TEMPLATE" for a description of available keys. --print-json Be quiet and print the video information as JSON (video is still being downloaded). -v, --verbose Print various debugging information. Workarounds: --no-check-certificate Suppress HTTPS certificate validation. --sleep-interval SLEEP_INTERVAL Number of seconds to sleep before each download when used alone or a lower bound of a range for randomized sleep before each download (minimum possible number of seconds to sleep) when used along with --max-sleep- interval. --max-sleep-interval MAX_SLEEP_INTERVAL Upper bound of a range for randomized sleep before each download (maximum possible number of seconds to sleep). Must only be used along with --min-sleep- interval. 27 | 28 | 29 | * Free software: MIT license 30 | * Documentation: https://tiktok-dl.readthedocs.io. 31 | 32 | 33 | Features 34 | -------- 35 | 36 | * TODO 37 | 38 | Credits 39 | ------- 40 | 41 | This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. 42 | 43 | .. _Cookiecutter: https://github.com/audreyr/cookiecutter 44 | .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage 45 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # tiktok_dl documentation build configuration file, created by 4 | # sphinx-quickstart on Fri Jun 9 13:47:02 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | # If extensions (or modules to document with autodoc) are in another 15 | # directory, add these directories to sys.path here. If the directory is 16 | # relative to the documentation root, use os.path.abspath to make it 17 | # absolute, like shown here. 18 | # 19 | import os 20 | import sys 21 | 22 | sys.path.insert(0, os.path.abspath("..")) 23 | 24 | import tiktok_dl 25 | 26 | # -- General configuration --------------------------------------------- 27 | 28 | # If your documentation needs a minimal Sphinx version, state it here. 29 | # 30 | # needs_sphinx = '1.0' 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 34 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ["_templates"] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = ".rst" 44 | 45 | # The master toctree document. 46 | master_doc = "index" 47 | 48 | # General information about the project. 49 | project = "tiktok-dl" 50 | copyright = "2020, Aakash Gajjar" 51 | author = "Aakash Gajjar" 52 | 53 | # The version info for the project you're documenting, acts as replacement 54 | # for |version| and |release|, also used in various other places throughout 55 | # the built documents. 56 | # 57 | # The short X.Y version. 58 | version = tiktok_dl.__version__ 59 | # The full version, including alpha/beta/rc tags. 60 | release = tiktok_dl.__version__ 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = "sphinx" 76 | 77 | # If true, `todo` and `todoList` produce output, else they produce nothing. 78 | todo_include_todos = False 79 | 80 | 81 | # -- Options for HTML output ------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = "alabaster" 87 | 88 | # Theme options are theme-specific and customize the look and feel of a 89 | # theme further. For a list of options available for each theme, see the 90 | # documentation. 91 | # 92 | # html_theme_options = {} 93 | 94 | # Add any paths that contain custom static files (such as style sheets) here, 95 | # relative to this directory. They are copied after the builtin static files, 96 | # so a file named "default.css" will overwrite the builtin "default.css". 97 | html_static_path = ["_static"] 98 | 99 | 100 | # -- Options for HTMLHelp output --------------------------------------- 101 | 102 | # Output file base name for HTML help builder. 103 | htmlhelp_basename = "tiktok_dldoc" 104 | 105 | 106 | # -- Options for LaTeX output ------------------------------------------ 107 | 108 | latex_elements = { 109 | # The paper size ('letterpaper' or 'a4paper'). 110 | # 111 | # 'papersize': 'letterpaper', 112 | # The font size ('10pt', '11pt' or '12pt'). 113 | # 114 | # 'pointsize': '10pt', 115 | # Additional stuff for the LaTeX preamble. 116 | # 117 | # 'preamble': '', 118 | # Latex figure (float) alignment 119 | # 120 | # 'figure_align': 'htbp', 121 | } 122 | 123 | # Grouping the document tree into LaTeX files. List of tuples 124 | # (source start file, target name, title, author, documentclass 125 | # [howto, manual, or own class]). 126 | latex_documents = [ 127 | (master_doc, "tiktok_dl.tex", "tiktok-dl Documentation", "Aakash Gajjar", "manual"), 128 | ] 129 | 130 | 131 | # -- Options for manual page output ------------------------------------ 132 | 133 | # One entry per manual page. List of tuples 134 | # (source start file, name, description, authors, manual section). 135 | man_pages = [(master_doc, "tiktok_dl", "tiktok-dl Documentation", [author], 1)] 136 | 137 | 138 | # -- Options for Texinfo output ---------------------------------------- 139 | 140 | # Grouping the document tree into Texinfo files. List of tuples 141 | # (source start file, target name, title, author, 142 | # dir menu entry, description, category) 143 | texinfo_documents = [ 144 | ( 145 | master_doc, 146 | "tiktok_dl", 147 | "tiktok-dl Documentation", 148 | author, 149 | "tiktok_dl", 150 | "One line description of project.", 151 | "Miscellaneous", 152 | ), 153 | ] 154 | -------------------------------------------------------------------------------- /tiktok_dl/options.py: -------------------------------------------------------------------------------- 1 | """Command-line Options for tiktok_dl.""" 2 | import argparse 3 | 4 | from tiktok_dl.version import __version__ 5 | 6 | 7 | def options_parser(): 8 | """Parser Command-line Options.""" 9 | parser = argparse.ArgumentParser(description="TikTok Video downloader",) 10 | 11 | parser.add_argument( 12 | "-V", 13 | "--version", 14 | action="version", 15 | version=__version__, 16 | help="Print program version and exit", 17 | ) 18 | 19 | parser.add_argument( 20 | "urls", metavar="URL", nargs="*", type=str, help="URL of the video" 21 | ) 22 | 23 | video_selection_group = parser.add_argument_group("Video Selection") 24 | video_selection_group.add_argument( 25 | "-a", 26 | "--download-archive", 27 | metavar="DOWNLOAD_ARCHIVE", 28 | type=str, 29 | default=None, 30 | help="Download only videos not listed in the archive file. " 31 | "Record the IDs of all downloaded videos in it.", 32 | ) 33 | 34 | parallel_download_group = parser.add_argument_group("Parallel Download") 35 | parallel_download_group.add_argument( 36 | "-d", "--daemon", action="store_true", dest="daemon", help="Run as daemon.", 37 | ) 38 | parallel_download_group.add_argument( 39 | "-j", 40 | "--concurrent-count", 41 | metavar="CONCURRENT_COUNT", 42 | type=int, 43 | default=2, 44 | help="Download videos in parallel.", 45 | ) 46 | 47 | filesystem_group = parser.add_argument_group("Filesystem Options") 48 | filesystem_group.add_argument( 49 | "-i", 50 | "--batch-file", 51 | metavar="FILENAME", 52 | type=str, 53 | default=None, 54 | help="File containing URLs to download ('-' for stdin), one URL per line. " 55 | "Lines starting with '#', ';' or ']' are considered as comments and ignored.", 56 | ) 57 | filesystem_group.add_argument( 58 | "-o", 59 | "--output-template", 60 | metavar="OUTPUT_TEMPLATE", 61 | type=str, 62 | default="{Y}-{d}-{m}_{H}-{M}-{S} {id}_{user_id}", 63 | help='Output filename template, see the "OUTPUT TEMPLATE" for all the info.', 64 | ) 65 | filesystem_group.add_argument( 66 | "-n", 67 | "--no-overwrite", 68 | action="store_true", 69 | default=False, 70 | help="Do not overwrite files", 71 | ) 72 | filesystem_group.add_argument( 73 | "--write-description", 74 | action="store_true", 75 | help="Write video description to a .description file.", 76 | ) 77 | filesystem_group.add_argument( 78 | "--no-write-json", 79 | action="store_true", 80 | default=False, 81 | help="Write video metadata to a .json file.", 82 | ) 83 | filesystem_group.add_argument( 84 | "-P", 85 | "--directory-prefix", 86 | metavar="DIRECTORY_PREFIX", 87 | type=str, 88 | default=None, 89 | help="Directory prefix.", 90 | ) 91 | 92 | thumbnail_group = parser.add_argument_group("Thumbnail images") 93 | thumbnail_group.add_argument( 94 | "--write-thumbnail", 95 | action="store_true", 96 | default=True, 97 | help="Write thumbnail image to disk.", 98 | ) 99 | 100 | simulation_group = parser.add_argument_group("Verbosity / Simulation Options:") 101 | simulation_group.add_argument( 102 | "-q", 103 | "--quiet", 104 | action="store_true", 105 | default=False, 106 | help="Activate quiet mode.", 107 | ) 108 | simulation_group.add_argument( 109 | "--no-warnings", action="store_true", default=False, help="Ignore warnings.", 110 | ) 111 | simulation_group.add_argument( 112 | "-s", 113 | "--simulate", 114 | action="store_true", 115 | default=False, 116 | help="Do not download the video and do not write anything to disk.", 117 | ) 118 | simulation_group.add_argument( 119 | "--skip-download", 120 | action="store_true", 121 | default=False, 122 | help="Do not download the video.", 123 | ) 124 | simulation_group.add_argument( 125 | "--dump-json", 126 | action="store_true", 127 | default=False, 128 | help="Simulate, quiet but print JSON information. " 129 | 'See the "OUTPUT TEMPLATE" for a description of available keys.', 130 | ) 131 | simulation_group.add_argument( 132 | "--print-json", 133 | action="store_true", 134 | default=False, 135 | help="Be quiet and print the video information as JSON (video is still being downloaded).", 136 | ) 137 | simulation_group.add_argument( 138 | "-v", 139 | "--verbose", 140 | action="store_false", 141 | default=True, 142 | help="Print various debugging information.", 143 | ) 144 | 145 | workarounds_group = parser.add_argument_group("Workarounds") 146 | workarounds_group.add_argument( 147 | "--no-check-certificate", 148 | action="store_true", 149 | default=False, 150 | help="Suppress HTTPS certificate validation.", 151 | ) 152 | workarounds_group.add_argument( 153 | "--sleep-interval", 154 | metavar="SLEEP_INTERVAL", 155 | type=float, 156 | default=0.2, 157 | help="Number of seconds to sleep before each download.", 158 | ) 159 | workarounds_group.add_argument( 160 | "--max-sleep-interval", 161 | metavar="MAX_SLEEP_INTERVAL", 162 | type=float, 163 | default=0, 164 | help="Maximum possible number of seconds to sleep.", 165 | ) 166 | parser.set_defaults( 167 | batch_file=None, 168 | concurrent_count=1, 169 | daemon=False, 170 | directory_prefix=None, 171 | download_archive=None, 172 | dump_json=False, 173 | get_description=False, 174 | get_duration=False, 175 | get_filename=False, 176 | get_id=True, 177 | get_thumbnail=False, 178 | get_title=False, 179 | get_url=False, 180 | max_sleep_interval=0, 181 | no_check_certificate=False, 182 | no_overwrite=False, 183 | no_warnings=False, 184 | no_write_json=False, 185 | output_template="{Y}-{d}-{m}_{H}-{M}-{S} {id}_{user_id}", 186 | print_json=False, 187 | quiet=False, 188 | simulate=False, 189 | skip_download=False, 190 | sleep_interval=0.2, 191 | urls=[], 192 | verbose=True, 193 | write_description=False, 194 | write_thumbnail=True, 195 | ) 196 | 197 | return parser 198 | -------------------------------------------------------------------------------- /tiktok_dl/downloader.py: -------------------------------------------------------------------------------- 1 | """TikTok Video Downloader""" 2 | import json 3 | import os 4 | import re 5 | import time 6 | 7 | import requests 8 | import urllib3 9 | 10 | from tiktok_dl.utils import format_utctime 11 | from tiktok_dl.utils import match_id 12 | from tiktok_dl.utils import search_regex 13 | from tiktok_dl.utils import try_get 14 | from tiktok_dl.utils import valid_url_re 15 | 16 | 17 | class URLExistsInArchive(Exception): 18 | """URL Recorded in the Archive. 19 | 20 | Args: 21 | Exception (Exception): If URL is recorded in the archive. 22 | """ 23 | 24 | pass 25 | 26 | 27 | class Downloader: 28 | """Downloader for TikTok Videos.""" 29 | 30 | def __init__( 31 | self, 32 | validator, 33 | extractor, 34 | logger, 35 | directory_prefix=None, 36 | dump_json=False, 37 | max_sleep_interval=0, 38 | no_check_certificate=True, 39 | no_overwrite=False, 40 | no_write_json=False, 41 | output_template="{Y}-{d}-{m}_{H}-{M}-{S} {id}_{user_id}", 42 | print_json=False, 43 | simulate=False, 44 | skip_download=False, 45 | sleep_interval=0.2, 46 | write_description=False, 47 | write_thumbnail=True, 48 | ): 49 | """Class for handling file downloads. 50 | 51 | Args: 52 | validator: Instance of AwemeValidator class. 53 | extractor: Instance of Extractor class. 54 | self.logger: Instance of self.logger class. 55 | directory_prefix (str, optional): Working directory for Downloader. Defaults to None. 56 | dump_json (bool, optional): Dump TikTok Video JSON and exit. Defaults to False. 57 | max_sleep_interval (int, optional): Maximum amount of seconds to sleep between downloads. Defaults to 0 (no sleeping). 58 | no_check_certificate (bool, optional): Do not validate server ssl certificates. Defaults to False. 59 | no_overwrite (bool, optional): Do not overwrite any file. Defaults to False. 60 | no_write_json (bool, optional): Do not create `.info.json` file. Defaults to False. 61 | output_template (str, optional): Output file template. Defaults to "{Y}-{d}-{m}_{H}-{M}-{S} {id}_{user_id}". 62 | print_json (bool, optional): Pretty print JSON when used with dump_json. Defaults to False. 63 | simulate (bool, optional): Simulate only do not write and download anything. Defaults to False. 64 | skip_download (bool, optional): Do not download any media. Defaults to False. 65 | sleep_interval (float, optional): Number of seconds to sleep between each requests. Defaults to 0.2. 66 | write_description (bool, optional): Create seperate .description file for description. Defaults to False. 67 | write_thumbnail (bool, optional): Download Video Thumbnail. Defaults to True. 68 | """ 69 | self.directory_prefix = directory_prefix 70 | self.dump_json = dump_json 71 | self.max_sleep_interval = max_sleep_interval 72 | self.no_check_certificate = no_check_certificate 73 | self.no_overwrite = no_overwrite 74 | self.no_write_json = no_write_json 75 | self.output_template = output_template 76 | self.print_json = print_json 77 | self.simulate = simulate 78 | self.skip_download = skip_download 79 | self.sleep_interval = sleep_interval 80 | self.write_description = write_description 81 | self.write_thumbnail = write_thumbnail 82 | 83 | self.validator = validator 84 | self.extractor = extractor 85 | self.logger = logger 86 | 87 | self.headers = { 88 | "user-agent": ( 89 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " 90 | "AppleWebKit/537.36 (KHTML, like Gecko) " 91 | "Chrome/83.0.4103.44 Safari/537.36" 92 | ) 93 | } 94 | self.reaponse_ok = requests.codes.get("ok") 95 | 96 | if self.no_check_certificate: 97 | urllib3.disable_warnings() 98 | 99 | def _parse_json(self, json_string: str, video_id: str, fatal=True): 100 | try: 101 | return json.loads(json_string) 102 | except ValueError as ve: 103 | errmsg = "{}: Failed to parse JSON ".format(video_id) 104 | if fatal: 105 | raise Exception(errmsg, cause=ve) 106 | else: 107 | self.logger.error(errmsg + str(ve)) 108 | 109 | def _download_webpage(self, url: str, video_id: str, note="Downloading webpage"): 110 | self.logger.debug("{} {}", note, video_id) 111 | r = requests.get(url, verify=self.no_check_certificate, headers=self.headers) 112 | return r.text 113 | 114 | def _fetch_data(self, url: str): 115 | video_id = match_id(url, valid_url_re) 116 | 117 | webpage = self._download_webpage( 118 | url, video_id, note="Downloading video webpage" 119 | ) 120 | json_string = search_regex( 121 | r"id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P[^<]+)", 122 | webpage, 123 | "json_string", 124 | group="json_string_id", 125 | ) 126 | json_data = self._parse_json(json_string, video_id) 127 | aweme_data = try_get( 128 | json_data, lambda x: x["props"]["pageProps"], expected_type=dict 129 | ) 130 | 131 | if aweme_data.get("statusCode") != 0: 132 | raise FileNotFoundError("Video not available " + video_id) 133 | 134 | extract_version, extract_data = self.extractor.extract(json_data=aweme_data) 135 | 136 | return { 137 | "video_data": extract_data, 138 | "aweme_data": aweme_data, 139 | "tiktok-dl": extract_version, 140 | "timestamp": int(time.time()), 141 | } 142 | 143 | def _expand_path(self, path): 144 | if self.directory_prefix is None: 145 | return path 146 | return os.path.join(self.directory_prefix, path) 147 | 148 | def _output_format(self, json_data: dict): 149 | def enhance_json_data(json_data): 150 | data = dict(json_data) 151 | timestamp = data.get("create_time") 152 | data["Y"] = format_utctime(time=timestamp, fmt="%Y") 153 | data["m"] = format_utctime(time=timestamp, fmt="%m") 154 | data["d"] = format_utctime(time=timestamp, fmt="%d") 155 | data["H"] = format_utctime(time=timestamp, fmt="%H") 156 | data["M"] = format_utctime(time=timestamp, fmt="%M") 157 | data["S"] = format_utctime(time=timestamp, fmt="%S") 158 | return data 159 | 160 | enhanced = enhance_json_data(json_data) 161 | return self.output_template.format(**enhanced) 162 | 163 | def _save_json(self, data: dict, dest: str): 164 | if not os.path.exists(os.path.dirname(dest)): 165 | os.makedirs(os.path.dirname(dest)) 166 | 167 | with open(dest, "w", encoding="utf-8") as f: 168 | json.dump(data, f, ensure_ascii=False) 169 | 170 | def _download_url(self, url: str, dest: str): 171 | if not os.path.exists(os.path.dirname(dest)): 172 | os.makedirs(os.path.dirname(dest), exist_ok=True) 173 | 174 | try: 175 | if os.path.getsize(dest) == 0: 176 | os.remove(dest) 177 | except FileNotFoundError: 178 | pass 179 | 180 | try: 181 | with open(dest, "xb") as handle: 182 | response = requests.get(url, stream=True, timeout=160) 183 | if response.status_code != self.reaponse_ok: 184 | response.raise_for_status() 185 | 186 | self.logger.debug("Downloading to {}".format(dest)) 187 | for data in response.iter_content(chunk_size=4194304): 188 | handle.write(data) 189 | handle.close() 190 | except FileExistsError: 191 | pass 192 | except requests.exceptions.RequestException: 193 | self.logger.error("File {} not found on Server {}".format(dest, url)) 194 | pass 195 | 196 | if os.path.getsize(dest) == 0: 197 | os.remove(dest) 198 | 199 | def _download_media(self, video_data: dict, filepath: str): 200 | video_url = video_data["play_urls"][0] 201 | self._download_url(video_url, self._expand_path(filepath + ".mp4")) 202 | cover_url = video_data["thumbnails"][0] 203 | self._download_url(cover_url, self._expand_path(filepath + ".jpg")) 204 | 205 | def download(self, url: str): 206 | try: 207 | data = self._fetch_data(url) 208 | self.validator.validate(data.get("video_data")) 209 | filepath = self._output_format(data.get("video_data")) 210 | self._download_media(data.get("video_data"), filepath) 211 | self._save_json(data, self._expand_path(filepath + ".json")) 212 | except requests.exceptions.InvalidURL as e: 213 | self.logger.error(e) 214 | pass 215 | except ConnectionError as e: 216 | self.logger.error(e) 217 | pass 218 | except re.error as e: 219 | self.logger.error(e) 220 | pass 221 | except FileNotFoundError as e: 222 | self.logger.warning(e) 223 | pass 224 | --------------------------------------------------------------------------------