├── tests
    ├── __init__.py
    ├── test_user.py
    ├── test_utils.py
    ├── test_captcha.py
    └── captcha_examples.json
├── examples
    ├── __init__.py
    ├── check_tool.py
    ├── ms_token_example.py
    ├── hashtag_example.py
    ├── video_bytes_example .py
    ├── comments_example.py
    ├── video_example.py
    ├── network_info_example.py
    └── user_example.py
├── .gitattributes
├── pytok
    ├── __init__.py
    ├── api
    │   ├── trending.py
    │   ├── __init__.py
    │   ├── sound.py
    │   ├── search.py
    │   ├── hashtag.py
    │   ├── user.py
    │   ├── base.py
    │   └── video.py
    ├── exceptions.py
    ├── helpers.py
    ├── tiktok.py
    ├── captcha_solver.py
    └── utils.py
├── requirements.txt
├── .gitignore
├── CITATION.cff
├── setup.py
└── README.md


/tests/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/examples/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 | 


--------------------------------------------------------------------------------
/pytok/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | .. include:: ../README.md
3 | """
4 | __docformat__ = "restructuredtext"
5 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | requests
 2 | playwright
 3 | pyvirtualdisplay
 4 | opencv-python
 5 | brotli
 6 | pandas
 7 | tqdm
 8 | patchright
 9 | pyclick
10 | TikTokApi


--------------------------------------------------------------------------------
/examples/check_tool.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | async def main():
 7 |     async with PyTok(browser="chromium") as api:
 8 |         await api._page.goto("https://www.browserscan.net/")
 9 |         pass
10 | 
11 | if __name__ == "__main__":
12 |     asyncio.run(main())
13 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | __pycache__/*
 2 | bmp.log
 3 | geckodriver.log
 4 | server.log
 5 | browsermob-proxy/*
 6 | myScripts/*
 7 | test.py
 8 | debug.log
 9 | res.html
10 | tmp/*
11 | dist/*
12 | *.egg-info
13 | tmp/
14 | tmp
15 | .pytest_cache/*
16 | test.mp4
17 | test.txt
18 | .pytest_cache/*
19 | tests/__pycache__/*
20 | *.pyc
21 | acrawl.js
22 | test2.py
23 | build
24 | MANIFEST
25 | src
26 | .vscode
27 | .env
28 | tests/data


--------------------------------------------------------------------------------
/examples/ms_token_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | async def main():
 7 |     async with PyTok(headless=True) as api:
 8 |         user = api.user(username="therock")
 9 |         # get random user to load page
10 |         user_data = await user.info()
11 |         ms_tokens = await api.get_ms_tokens()
12 |         print(ms_tokens)
13 | 
14 | if __name__ == "__main__":
15 |     asyncio.run(main())
16 | 


--------------------------------------------------------------------------------
/CITATION.cff:
--------------------------------------------------------------------------------
 1 | cff-version: 1.2.0
 2 | message: "If you use this software, please cite it as below."
 3 | authors:
 4 | - family-names: "Steel"
 5 |   given-names: "Ben"
 6 |   orcid: "https://orcid.org/0009-0006-3845-1394"
 7 | - family-names: "Abrahams"
 8 |   given-names: "Alexei"
 9 |   orcid: "https://orcid.org/0000-0002-6547-072X"
10 | title: "PyTok"
11 | version: 0.1.0
12 | doi: 10.5281/zenodo.12802714
13 | date-released: 2024-07-23
14 | url: "https://github.com/networkdynamics/pytok"


--------------------------------------------------------------------------------
/tests/test_user.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import os
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | # username = "brianjordanalvarez"
 7 | username = 'marierenaudstab'
 8 | 
 9 | 
10 | async def test_user_videos():
11 |     async with PyTok(headless=True) as api:
12 |         user = api.user(username=username)
13 |         user_data = await user.info()
14 |         count = 0
15 |         async for video in api.user(username=username).videos(count=100):
16 |             count += 1
17 | 
18 |         assert count >= 120
19 | 
20 | 
21 | if __name__ == '__main__':
22 |     asyncio.run(test_user_videos())


--------------------------------------------------------------------------------
/examples/hashtag_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | hashtag_name = 'fyp'
 7 | 
 8 | async def main():
 9 |     async with PyTok(manual_captcha_solves=True) as api:
10 |         hashtag = api.hashtag(name=hashtag_name)
11 | 
12 |         videos = []
13 |         async for video in hashtag.videos(count=1000):
14 |             video_info = await video.info()
15 |             videos.append(video_info)
16 | 
17 |         with open("out.json", "w") as out_file:
18 |             json.dump(videos, out_file)
19 | 
20 | if __name__ == "__main__":
21 |     asyncio.run(main())


--------------------------------------------------------------------------------
/examples/video_bytes_example .py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | username = 'therock'
 7 | id = '7296444945991224622'
 8 | 
 9 | async def main():
10 |     async with PyTok() as api:
11 |         video = api.video(username=username, id=id)
12 | 
13 |         # Bytes of the TikTok video
14 |         video_data = await video.info()
15 |         video_bytes = await video.bytes()
16 | 
17 |         with open("out.json", "w") as out_file:
18 |             json.dump(video_data, out_file)
19 | 
20 |         with open("out.mp4", "wb") as out_file:
21 |             out_file.write(video_bytes)
22 | 
23 | if __name__ == "__main__":
24 |     asyncio.run(main())
25 | 
26 | 


--------------------------------------------------------------------------------
/examples/comments_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | videos = [
 7 |     {
 8 |         'id': '7058106162235100462',
 9 |         'author': {
10 |             'uniqueId': 'charlesmcbryde'
11 |         }
12 |     }
13 | ]
14 | 
15 | async def main():
16 |     async with PyTok(headless=False) as api:
17 |         for video in videos:
18 |             comments = []
19 |             async for comment in api.video(id=video['id'], username=video['author']['uniqueId']).comments(count=1000):
20 |                 comments.append(comment)
21 | 
22 |             assert len(comments) > 0, "No comments found"
23 |             with open("out.json", "w") as f:
24 |                 json.dump(comments, f)
25 | 
26 | if __name__ == "__main__":
27 |     asyncio.run(main())
28 | 


--------------------------------------------------------------------------------
/pytok/api/trending.py:
--------------------------------------------------------------------------------
 1 | from __future__ import annotations
 2 | 
 3 | import logging
 4 | from urllib.parse import urlencode
 5 | 
 6 | from .video import Video
 7 | from .sound import Sound
 8 | from .user import User
 9 | from .hashtag import Hashtag
10 | 
11 | from typing import TYPE_CHECKING, Iterator
12 | 
13 | if TYPE_CHECKING:
14 |     from ..tiktok import PyTok
15 | 
16 | 
17 | class Trending:
18 |     """Contains static methods related to trending."""
19 | 
20 |     parent: PyTok
21 | 
22 |     @staticmethod
23 |     def videos(count=30, **kwargs) -> Iterator[Video]:
24 |         """
25 |         Returns Videos that are trending on TikTok.
26 | 
27 |         - Parameters:
28 |             - count (int): The amount of videos you want returned.
29 |         """
30 | 
31 |         raise NotImplementedError()
32 | 


--------------------------------------------------------------------------------
/examples/video_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | username = 'therock'
 7 | id = '7296444945991224622'
 8 | 
 9 | async def main():
10 |     async with PyTok() as api:
11 |         video = api.video(username=username, id=id)
12 | 
13 |         # Bytes of the TikTok video
14 |         video_data = await video.info()
15 |         related_videos = []
16 |         async for related_video in video.related_videos():
17 |             related_videos.append(related_video)
18 | 
19 |         with open("out.json", "w") as out_file:
20 |             json.dump(video_data, out_file)
21 | 
22 |         with open("related.json", "w") as out_file:
23 |             json.dump(list(related_videos), out_file)
24 | 
25 | if __name__ == "__main__":
26 |     asyncio.run(main())
27 | 
28 | 


--------------------------------------------------------------------------------
/examples/network_info_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | 
 4 | from pytok.tiktok import PyTok
 5 | 
 6 | username = 'therock'
 7 | id = '7296444945991224622'
 8 | 
 9 | async def main():
10 |     async with PyTok() as api:
11 |         video = api.video(username=username, id=id)
12 | 
13 |         # Bytes of the TikTok video
14 |         video_data = await video.info()
15 |         network_data = await video.network_info()
16 |         bytes_network_data = await video.bytes_network_info()
17 | 
18 |         all_data = {
19 |             "video_data": video_data,
20 |             "network_data": network_data,
21 |             "bytes_network_data": bytes_network_data
22 |         }
23 | 
24 |         with open("out.json", "w") as out_file:
25 |             json.dump(all_data, out_file)
26 | 
27 | if __name__ == "__main__":
28 |     asyncio.run(main())
29 | 
30 | 


--------------------------------------------------------------------------------
/tests/test_utils.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | 
 3 | import pytest
 4 | 
 5 | from pytok import utils
 6 | 
 7 | @pytest.mark.parametrize("json_file_path", [os.path.join(".", "tests", "data", "20230915-200856_error_videos.json")])
 8 | def test_get_video_df(json_file_path):
 9 |     csv_file_path = json_file_path.replace(".json", ".csv")
10 |     video_df = utils.try_load_video_df_from_file(csv_file_path, file_paths=[json_file_path])
11 | 
12 |     assert video_df is not None
13 |     assert len(video_df) > 0
14 | 
15 | @pytest.mark.parametrize("json_file_path", [os.path.join(".", "tests", "data", "20230915-200856_error_users.json")])
16 | def test_get_user_df(json_file_path):
17 |     csv_file_path = json_file_path.replace(".json", ".csv")
18 |     user_df = utils.try_load_user_df_from_file(csv_file_path, file_paths=[json_file_path])
19 | 
20 |     assert user_df is not None
21 |     assert len(user_df) > 0
22 | 
23 | if __name__ == "__main__":
24 |     pytest.main([__file__])


--------------------------------------------------------------------------------
/examples/user_example.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import json
 3 | import logging
 4 | 
 5 | from pytok.tiktok import PyTok
 6 | 
 7 | # Enable debug logging
 8 | logging.basicConfig(
 9 |     level=logging.DEBUG,
10 |     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
11 | )
12 | 
13 | async def main():
14 |     users = ['therock']
15 |     async with PyTok(logging_level=logging.DEBUG, manual_captcha_solves=True, log_captcha_solves=True) as api:
16 |         for username in users:
17 |             user = api.user(username=username)
18 |             user_data = await user.info()
19 | 
20 |             videos = []
21 |             videos_bytes = []
22 |             async for video in user.videos():
23 |                 video_data = await video.info()
24 |                 videos.append(video_data)
25 | 
26 |             assert len(videos) > 0, "No videos found"
27 |             with open("out.json", "w") as f:
28 |                 json.dump(videos, f)
29 | 
30 | if __name__ == "__main__":
31 |     asyncio.run(main())
32 | 


--------------------------------------------------------------------------------
/pytok/api/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 | This module contains classes that all represent different types of data sent back by the TikTok servers.
 3 | 
 4 | The files within in module correspond to what type of object is described and all have different methods associated with them.
 5 | 
 6 | 
 7 | ### How To Interpret TikTok Data
 8 | There are quite a few ambigious keys in the JSON that TikTok returns so here's a section that tries to document some of them.
 9 | 
10 | **Note**: These are incomplete, if you get confused about something feel free to add it here as a PR once you figure it out.
11 | 
12 | | JSON Key         | Description |
13 | |------------------|-------------|
14 | | createTime | The [unix epoch](https://docs.python.org/3/library/datetime.html#datetime.date.fromtimestamp) of creation, all other time fields are also unix epochs.  |
15 | | secUid & (userId or id) | Two different unique attributes that are used in conjunction to reference a specific account, so if you're storing users somewhere in a database, you should store both secUid & userId. |
16 | | id | A unique attribute used to reference a non-user object like video, hashtag, etc |
17 | | diggCount | The likes for a specific video. |
18 | | digged | Used to check if the current user has liked/digged a video, this will always be false since this package doesn't support logged-in user functions. |
19 | """
20 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from distutils.core import setup
 2 | import os.path
 3 | import setuptools
 4 | 
 5 | with open("README.md", "r", encoding="utf-8") as fh:
 6 |     long_description = fh.read()
 7 | 
 8 | setuptools.setup(
 9 |     name="pytok",
10 |     packages=setuptools.find_packages(),
11 |     version="0.0.1",
12 |     license="MIT",
13 |     description="Playwright based version of The Unofficial TikTok API Wrapper in Python 3.",
14 |     author="Ben Steel",
15 |     author_email="bendavidsteel@gmail.com",
16 |     url="https://github.com/networkdynamics/pytok",
17 |     long_description=long_description,
18 |     long_description_content_type="text/markdown",
19 |     keywords=["tiktok", "python3", "api", "unofficial", "tiktok-api", "tiktok api"],
20 |     install_requires=["requests", "playwright", "undetected_playwright", "pyvirtualdisplay", "tqdm", "opencv-python", "brotli", "patchright", "pyclick", "TikTokApi"],
21 |     classifiers=[
22 |         "Development Status :: 3 - Alpha",
23 |         "Intended Audience :: Developers",
24 |         "Topic :: Software Development :: Build Tools",
25 |         "License :: OSI Approved :: MIT License",
26 |         "Programming Language :: Python :: 3.7",
27 |         "Programming Language :: Python :: 3.8",
28 |         "Programming Language :: Python :: 3.9",
29 |         "Programming Language :: Python :: 3.10",
30 |     ],
31 | )
32 | 


--------------------------------------------------------------------------------
/pytok/exceptions.py:
--------------------------------------------------------------------------------
 1 | class TikTokException(Exception):
 2 |     """Generic exception that all other TikTok errors are children of."""
 3 | 
 4 |     def __init__(self, *args, **kwargs):
 5 |         super().__init__(*args, **kwargs)
 6 | 
 7 | 
 8 | class CaptchaException(TikTokException):
 9 |     """TikTok is showing captcha"""
10 | 
11 | 
12 | class NotFoundException(TikTokException):
13 |     """TikTok indicated that this object does not exist."""
14 | 
15 | 
16 | class EmptyResponseException(TikTokException):
17 |     """TikTok sent back an empty response."""
18 | 
19 | 
20 | class SoundRemovedException(TikTokException):
21 |     """This TikTok sound has no id from being removed by TikTok."""
22 | 
23 | 
24 | class InvalidJSONException(TikTokException):
25 |     """TikTok returned invalid JSON."""
26 | 
27 | 
28 | class NotAvailableException(TikTokException):
29 |     """The requested object is not available in this region."""
30 | 
31 | class NoContentException(TikTokException):
32 |     """TikTok returned no content"""
33 | 
34 | class TimeoutException(TikTokException):
35 |     """Timed out trying to get content from TikTok"""
36 | 
37 | class ApiFailedException(TikTokException):
38 |     """TikTok API is failing"""
39 | 
40 | class FewerVideosThanExpectedException(TikTokException):
41 |     """TikTok is returning fewer videos for this user than their metadata led us to expect"""
42 | 
43 | class AccountPrivateException(TikTokException):
44 |     """This TikTok account is private and cannot be scraped"""


--------------------------------------------------------------------------------
/tests/test_captcha.py:
--------------------------------------------------------------------------------
 1 | import base64
 2 | import json
 3 | import os
 4 | 
 5 | import cv2
 6 | import matplotlib.pyplot as plt
 7 | import numpy as np
 8 | 
 9 | from pytok import captcha_solver
10 | 
11 | def main():
12 |     this_dir_path = os.path.dirname(os.path.realpath(__file__))
13 |     with open(os.path.join(this_dir_path, 'captcha_examples.json'), 'r') as f:
14 |         data = json.load(f)
15 |     for type, examples in data.items():
16 |         for example in examples:
17 |             puzzle_b64 = example['puzzle'].strip("b'")
18 |             piece_b64 = example['piece'].strip("b'")
19 | 
20 |             best_angle = captcha_solver.whirl_solver(puzzle_b64, piece_b64)
21 |             puzzle, piece, puzzle_edge, piece_edge = captcha_solver._get_images_and_edges(puzzle_b64, piece_b64)
22 | 
23 |             solved_puzzle = puzzle.copy()
24 |             puzzle_r = (piece.shape[0] / 2) - 1
25 |             for y in range(solved_puzzle.shape[1]):
26 |                 for x in range(solved_puzzle.shape[0]):
27 |                     if (x - solved_puzzle.shape[0] / 2) ** 2 + (y - solved_puzzle.shape[1] / 2) ** 2 < puzzle_r ** 2:
28 |                         theta = np.arctan2(y - solved_puzzle.shape[1] / 2, x - solved_puzzle.shape[0] / 2)
29 |                         theta -= (best_angle / piece_edge.shape[0]) * 2 * np.pi
30 |                         r = np.sqrt((x - solved_puzzle.shape[0] / 2) ** 2 + (y - solved_puzzle.shape[1] / 2) ** 2)
31 |                         solved_puzzle[x, y] = piece[int(piece.shape[0] / 2 + r * np.cos(theta)), int(piece.shape[1] / 2 + r * np.sin(theta))]
32 | 
33 |             matches = np.zeros(puzzle_edge.shape[0])
34 |             for angle in range(puzzle_edge.shape[0]):
35 |                 match = np.sum(puzzle_edge * np.roll(piece_edge, angle, axis=0))
36 |                 matches[angle] = match
37 | 
38 |             # save the best match
39 |             fig, ax = plt.subplots(nrows=7)
40 |             ax[0].imshow(puzzle)
41 |             ax[1].imshow(piece)
42 |             ax[2].imshow(solved_puzzle)
43 |             ax[3].imshow(np.repeat(puzzle_edge[np.newaxis, :, :] / 255, 50, axis=0))
44 |             ax[4].imshow(np.repeat(piece_edge[np.newaxis, :, :] / 255, 50, axis=0))
45 |             ax[5].imshow(np.repeat(np.roll(piece_edge / 255, best_angle, axis=0)[np.newaxis, :, :], 50, axis=0))
46 |             ax[6].plot(matches)
47 |             plt.show()
48 |             
49 | 
50 | if __name__ == '__main__':
51 |     main()


--------------------------------------------------------------------------------
/pytok/helpers.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | from urllib import parse as url_parsers
 3 | 
 4 | import requests
 5 | 
 6 | from .exceptions import *
 7 | 
 8 | 
 9 | def extract_tag_contents(html):
10 |     if isinstance(html, bytes):
11 |         html = html.decode("utf-8")
12 |     data_json_match = re.search(r"""<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application\/json">([^\>]+)<\/script>""", html)
13 |     if data_json_match:
14 |         return data_json_match.group(1)
15 |     else:
16 |         next_json = re.search(
17 |             r"id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P<next_data>[^<]+)",
18 |             html,
19 |         )
20 |         if next_json:
21 |             nonce_start = '<head nonce="'
22 |             nonce_end = '">'
23 |             nonce = html.split(nonce_start)[1].split(nonce_end)[0]
24 |             j_raw = html.split(
25 |                 '<script id="__NEXT_DATA__" type="application/json" nonce="%s" crossorigin="anonymous">'
26 |                 % nonce
27 |             )[1].split("</script>")[0]
28 |             return j_raw
29 |         else:
30 |             sigi_json = re.search('<script id="SIGI_STATE" type="application\/json">(.*?)<\/script>', html)
31 |             #sigi_json = re.search(
32 |                 #r'>\s*window\[[\'"]SIGI_STATE[\'"]\]\s*=\s*(?P<sigi_state>{.+});', html
33 |             #)
34 |             if sigi_json:
35 |                 return sigi_json.group(1)
36 |             else:
37 |                 raise NotAvailableException("Could not find the tag contents")
38 |                 # not a reliable way to check for captchas
39 |                 # raise CaptchaException(
40 |                 #    "TikTok blocks this request displaying a Captcha \nTip: Consider using a proxy or a custom_verify_fp as method parameters"
41 |                 # )
42 | 
43 | 
44 | def extract_video_id_from_url(url):
45 |     url = requests.head(url=url, allow_redirects=True).url
46 |     if "@" in url and "/video/" in url:
47 |         return url.split("/video/")[1].split("?")[0]
48 |     else:
49 |         raise TypeError(
50 |             "URL format not supported. Below is an example of a supported url.\n"
51 |             "https://www.tiktok.com/@therock/video/6829267836783971589"
52 |         )
53 | 
54 | def extract_user_id_from_url(url):
55 |     url = requests.head(url=url, allow_redirects=True).url
56 |     if "@" in url and "/video/" in url:
57 |         return url.split("/video/")[0].split("@")[1]
58 |     else:
59 |         raise TypeError(
60 |             "URL format not supported. Below is an example of a supported url.\n"
61 |             "https://www.tiktok.com/@therock/video/6829267836783971589"
62 |         )
63 | 
64 | def add_if_not_replace(text, pat, replace, add):
65 |     if re.search(pat, text):
66 |         return re.sub(pat, replace, text)
67 |     else:
68 |         text += add
69 |         return text
70 |     
71 | def edit_url(url, new_params):
72 |     url_parsed = url_parsers.urlparse(url)
73 |     params = url_parsers.parse_qs(url_parsed.query, keep_blank_values=True)
74 |     for k, v in new_params.items():
75 |         params[k] = [v]
76 |     # url encode params chosen to match the tiktok url encoding method
77 |     return f"{url_parsed.scheme}://{url_parsed.netloc}{url_parsed.path}?{url_parsers.urlencode(params, doseq=True, safe='=', quote_via=url_parsers.quote)}"
78 | 
79 | 
80 | 


--------------------------------------------------------------------------------
/pytok/api/sound.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | from os import path
  3 | 
  4 | import json
  5 | 
  6 | from urllib.parse import quote, urlencode
  7 | 
  8 | from ..helpers import extract_tag_contents
  9 | from ..exceptions import *
 10 | 
 11 | from typing import TYPE_CHECKING, ClassVar, Iterator, Optional
 12 | 
 13 | if TYPE_CHECKING:
 14 |     from ..tiktok import PyTok
 15 |     from .user import User
 16 |     from .video import Video
 17 | 
 18 | 
 19 | class Sound:
 20 |     """
 21 |     A TikTok Sound/Music/Song.
 22 | 
 23 |     Example Usage
 24 |     ```py
 25 |     song = api.song(id='7016547803243022337')
 26 |     ```
 27 |     """
 28 | 
 29 |     parent: ClassVar[PyTok]
 30 | 
 31 |     id: str
 32 |     """TikTok's ID for the sound"""
 33 |     title: Optional[str]
 34 |     """The title of the song."""
 35 |     author: Optional[User]
 36 |     """The author of the song (if it exists)"""
 37 | 
 38 |     def __init__(self, id: Optional[str] = None, data: Optional[str] = None):
 39 |         """
 40 |         You must provide the id of the sound or it will not work.
 41 |         """
 42 |         if data is not None:
 43 |             self.as_dict = data
 44 |             self.__extract_from_data()
 45 |         elif id is None:
 46 |             raise TypeError("You must provide id parameter.")
 47 |         else:
 48 |             self.id = id
 49 | 
 50 |     def info(self, use_html=False, **kwargs) -> dict:
 51 |         """
 52 |         Returns a dictionary of TikTok's Sound/Music object.
 53 | 
 54 |         - Parameters:
 55 |             - use_html (bool): If you want to perform an HTML request or not.
 56 |                 Defaults to False to use an API call, which shouldn't get detected
 57 |                 as often as an HTML request.
 58 | 
 59 | 
 60 |         Example Usage
 61 |         ```py
 62 |         sound_data = api.sound(id='7016547803243022337').info()
 63 |         ```
 64 |         """
 65 |         raise NotImplementedError()
 66 | 
 67 |     def info_full(self, **kwargs) -> dict:
 68 |         """
 69 |         Returns all the data associated with a TikTok Sound.
 70 | 
 71 |         This makes an API request, there is no HTML request option, as such
 72 |         with Sound.info()
 73 | 
 74 |         Example Usage
 75 |         ```py
 76 |         sound_data = api.sound(id='7016547803243022337').info_full()
 77 |         ```
 78 |         """
 79 |         raise NotImplementedError()
 80 | 
 81 |     def videos(self, count=30, offset=0, **kwargs) -> Iterator[Video]:
 82 |         """
 83 |         Returns Video objects of videos created with this sound.
 84 | 
 85 |         - Parameters:
 86 |             - count (int): The amount of videos you want returned.
 87 |             - offset (int): The offset of videos you want returned.
 88 | 
 89 |         Example Usage
 90 |         ```py
 91 |         for video in api.sound(id='7016547803243022337').videos():
 92 |             # do something
 93 |         ```
 94 |         """
 95 |         raise NotImplementedError()
 96 | 
 97 |     def __extract_from_data(self):
 98 |         data = self.as_dict
 99 |         keys = data.keys()
100 | 
101 |         self.id = data.get("id")
102 |         self.title = data.get("title")
103 | 
104 |         if data.get("authorName") is not None:
105 |             self.author = self.parent.user(username=data["authorName"])
106 | 
107 |         if self.id is None:
108 |             Sound.parent.logger.error(
109 |                 f"Failed to create Sound with data: {data}\nwhich has keys {data.keys()}"
110 |             )
111 | 
112 |     def __ensure_valid(self):
113 |         if self.id == "":
114 |             raise SoundRemovedException("This sound has been removed!")
115 | 
116 |     def __repr__(self):
117 |         return self.__str__()
118 | 
119 |     def __str__(self):
120 |         return f"PyTok.sound(id='{self.id}')"
121 | 
122 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | 
  2 | [![DOI](https://zenodo.org/badge/555492190.svg)](https://zenodo.org/doi/10.5281/zenodo.12802713)
  3 | 
  4 | # pytok
  5 | 
  6 | This is a Playwright based version of David Teacher's unofficial api wrapper for TikTok.com in python. It re-implements a currently limited set of the features of the original library, with a shifted focus on using browser automation to allow automatic captcha solves with a hopefully minor trade-off in performance.
  7 | 
  8 | ## Installation
  9 | 
 10 | ```bash
 11 | pip install git+https://github.com/networkdynamics/pytok.git@master
 12 | ```
 13 | 
 14 | ## Quick Start Guide
 15 | 
 16 | Here's a quick bit of code to get the videos from a particular hashtag on TikTok. There's more examples in the [examples](https://github.com/networkdynamics/pytok/tree/master/examples) directory.
 17 | 
 18 | ```py
 19 | import asyncio
 20 | 
 21 | from pytok.tiktok import PyTok
 22 | 
 23 | async def main():
 24 |     async with PyTok() as api:
 25 |         user = api.user(username="therock")
 26 |         user_data = await user.info()
 27 |         print(user_data)
 28 | 
 29 |         videos = []
 30 |         async for video in user.videos():
 31 |             video_data = video.info()
 32 |             print(video_data)
 33 | 
 34 | if __name__ == "__main__":
 35 |     asyncio.run(main())
 36 | ```
 37 | 
 38 | 
 39 | Please note pulling data from TikTok takes a while! We recommend leaving the scripts running on a server for a while for them to finish downloading everything. Feel free to play around with the delay constants to either speed up the process or avoid TikTok rate limiting, like so: `PyTok(request_delay=10)`
 40 | 
 41 | Please do not hesitate to make an issue in this repo to get our help with this!
 42 | 
 43 | ## Citation
 44 | 
 45 | If you use this library in your research, please cite it using the following BibTeX entry:
 46 | 
 47 | ```bibtex
 48 | @software{ben_steel_2024_12802714,
 49 |   author       = {Ben Steel and
 50 |                   Alexei Abrahams},
 51 |   title        = {{networkdynamics/pytok: Initial working version of 
 52 |                    library}},
 53 |   month        = jul,
 54 |   year         = 2024,
 55 |   publisher    = {Zenodo},
 56 |   version      = {v0.1.0},
 57 |   doi          = {10.5281/zenodo.12802714},
 58 |   url          = {https://doi.org/10.5281/zenodo.12802714}
 59 | }
 60 | ```
 61 | 
 62 | ## Format and Schema
 63 | 
 64 | The JSONable dictionary returned by the `info()` methods contains all of the data that the TikTok API returns. We have provided helper functions to parse that data into Pandas DataFrames, `utils.get_comment_df()`, `utils.get_video_df()` and `utils.get_user_df()` for the data from comments, videos, and users respectively.
 65 | 
 66 | The video dataframe will contain the following columns:
 67 | |Field name | Description |
 68 | |----------|----------|
 69 | |`video_id`| Unique video ID |
 70 | |`createtime`| UTC datetime of video creation time in YYYY-MM-DD HH:MM:SS format |
 71 | |`author_name`| Unique author name |
 72 | |`author_id`| Unique author ID |
 73 | |`desc`| The full video description from the author |
 74 | |`hashtags`| A list of hashtags used in the video description |
 75 | |`share_video_id`| If the video is sharing another video, this is the video ID of that original video, else empty |
 76 | |`share_video_user_id`| If the video is sharing another video, this the user ID of the author of that video, else empty |
 77 | |`share_video_user_name`| If the video is sharing another video, this is the user name of the author of that video, else empty |
 78 | |`share_type`| If the video is sharing another video, this is the type of the share, stitch, duet etc. |
 79 | |`mentions`| A list of users mentioned in the video description, if any |
 80 | |`digg_count`| The number of likes on the video |
 81 | |`share_count`| The number of times the video was shared |
 82 | |`comment_count`| The number of comments on the video |
 83 | |`play_count`| The number of times the video was played |
 84 | 
 85 | The comment dataframe will contain the following columns:
 86 | |Field name | Description |
 87 | |----------|-----------|
 88 | |`comment_id`| Unique comment ID |
 89 | |`createtime`| UTC datetime of comment creation time in YYYY-MM-DD HH:MM:SS format |
 90 | |`author_name`| Unique author name |
 91 | |`author_id`| Unique author ID |
 92 | |`text`| Text of the comment |
 93 | |`mentions`| A list of users that are tagged in the comment |
 94 | |`video_id`| The ID of the video the comment is on |
 95 | |`comment_language`| The language of the comment, as predicted by the TikTok API |
 96 | |`digg_count`| The number of likes the comment got |
 97 | |`reply_comment_id`| If the comment is replying to another comment, this is the ID of that comment |
 98 | 
 99 | The user dataframe will contain the following columns:
100 | |Field name | Description |
101 | |----------|-----------|
102 | |`id`| Unique author ID |
103 | |`unique_id`| Unique user name |
104 | |`nickname`| Display user name, changeable |
105 | |`signature`| Short user description |
106 | |`verified`| Whether or not the user is verified |
107 | |`num_following`| How many other accounts the user is following |
108 | |`num_followers`| How many followers the user has |
109 | |`num_videos`| How many videos the user has made |
110 | |`num_likes`| How many total likes the user has had |
111 | |`createtime`| When the user account was made. This is derived from the `id` field, and can occasionally be incorrect with a very low unix epoch such as 1971 |
112 | 
113 | 


--------------------------------------------------------------------------------
/pytok/api/search.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | import json
  4 | import time
  5 | from typing import TYPE_CHECKING, Iterator, Type
  6 | from urllib.parse import urlencode
  7 | import re
  8 | 
  9 | from .user import User
 10 | from .hashtag import Hashtag
 11 | from .video import Video
 12 | from .base import Base
 13 | from ..exceptions import *
 14 | 
 15 | if TYPE_CHECKING:
 16 |     from ..tiktok import PyTok
 17 | 
 18 | import requests
 19 | from playwright.async_api import TimeoutError
 20 | 
 21 | class Search(Base):
 22 |     """Contains static methods about searching."""
 23 | 
 24 |     parent: PyTok
 25 | 
 26 |     def __init__(self, search_term):
 27 |         self.search_term = search_term
 28 | 
 29 |     def videos(self, count=28, offset=0, **kwargs) -> Iterator[Video]:
 30 |         """
 31 |         Searches for Videos
 32 | 
 33 |         - Parameters:
 34 |             - search_term (str): The phrase you want to search for.
 35 |             - count (int): The amount of videos you want returned.
 36 |             - offset (int): The offset of videos from your data you want returned.
 37 | 
 38 |         Example Usage
 39 |         ```py
 40 |         for video in api.search.videos('therock'):
 41 |             # do something
 42 |         ```
 43 |         """
 44 |         return self.search_type(
 45 |             "item", count=count, offset=offset, **kwargs
 46 |         )
 47 | 
 48 |     def users(self, count=28, offset=0, **kwargs) -> Iterator[User]:
 49 |         """
 50 |         Searches for users using an alternate endpoint than Search.users
 51 | 
 52 |         - Parameters:
 53 |             - search_term (str): The phrase you want to search for.
 54 |             - count (int): The amount of videos you want returned.
 55 | 
 56 |         Example Usage
 57 |         ```py
 58 |         for user in api.search.users_alternate('therock'):
 59 |             # do something
 60 |         ```
 61 |         """
 62 |         return self.search_type(
 63 |             "user", count=count, offset=offset, **kwargs
 64 |         )
 65 | 
 66 |     async def search_type(self, obj_type, count=28, offset=0, **kwargs) -> Iterator:
 67 |         """
 68 |         Searches for users using an alternate endpoint than Search.users
 69 | 
 70 |         - Parameters:
 71 |             - search_term (str): The phrase you want to search for.
 72 |             - count (int): The amount of videos you want returned.
 73 |             - obj_type (str): user | item
 74 | 
 75 |         Just use .video & .users
 76 |         ```
 77 |         """
 78 | 
 79 |         if obj_type == "user":
 80 |             subdomain = "www"
 81 |             subpath = "user"
 82 |         elif obj_type == "item":
 83 |             subdomain = "www"
 84 |             subpath = "video"
 85 |         else:
 86 |             raise TypeError("invalid obj_type")
 87 | 
 88 |         page = self.parent._page
 89 | 
 90 |         url = f"https://{subdomain}.tiktok.com/search/{subpath}?q={self.search_term}"
 91 |         await page.goto(url)
 92 | 
 93 |         is_visible = await page.locator('[data-e2e="search_video-item"]').is_visible()
 94 |         is_visible2 = await page.locator('[data-e2e="search_video-item-list"]').is_visible()
 95 |         is_visible3 = await page.locator('[id="column-item-video-container-0"]').is_visible()
 96 | 
 97 |         await self.wait_for_content_or_captcha('[id="column-item-video-container-0"]')
 98 | 
 99 |         processed_urls = []
100 |         amount_yielded = 0
101 |         pull_method = 'browser'
102 |         
103 |         path = f"api/search/{obj_type}"
104 | 
105 |         while amount_yielded < count:
106 |             await self.check_and_wait_for_captcha()
107 |             await self.parent.request_delay()
108 |             await self.slight_scroll_up()
109 |             await self.parent.request_delay()
110 |             await self.scroll_down(30000, speed=12)
111 | 
112 |             if pull_method == 'browser':
113 |                 search_responses = self.get_responses(path)
114 |                 search_responses = [response for response in search_responses if response.url not in processed_urls]
115 |                 for response in search_responses:
116 |                     processed_urls.append(response.url)
117 |                     body = await self.get_response_body(response)
118 |                     if not body:
119 |                         continue
120 |                     res = json.loads(body)
121 |                     if res.get('type') == 'verify':
122 |                         # this is the captcha denied response
123 |                         continue
124 | 
125 |                     # When I move to 3.10+ support make this a match switch.
126 |                     if obj_type == "user":
127 |                         for result in res.get("user_list", []):
128 |                             yield User(data=result)
129 |                             amount_yielded += 1
130 | 
131 |                     if obj_type == "item":
132 |                         for result in res.get("item_list", []):
133 |                             yield Video(data=result)
134 |                             amount_yielded += 1
135 | 
136 |                     if res.get("has_more", 0) == 0:
137 |                         Search.parent.logger.info(
138 |                             "TikTok is not sending videos beyond this point."
139 |                         )
140 |                         return
141 | 
142 |                 # try:
143 |                 #     load_more_button = await self.wait_for_content_or_captcha('search-load-more')
144 |                 # except TimeoutError:
145 |                 #     return
146 | 
147 |                 # await load_more_button.click()
148 | 
149 |             
150 |             elif pull_method == 'requests':
151 |                 cursor = res["cursor"]
152 |                 next_url = re.sub("offset=([0-9]+)", f"offset={cursor}", request.url)
153 |                 cookies = self.parent._context.cookies()
154 |                 cookies = {cookie['name']: cookie['value'] for cookie in cookies}
155 |                 r = requests.get(next_url, headers=request.headers, cookies=cookies)
156 |                 res = r.json()
157 | 
158 |                 if res.get('type') == 'verify':
159 |                     pull_method = 'browser'
160 |                     continue
161 | 
162 |                 if obj_type == "user":
163 |                     for result in res.get("user_list", []):
164 |                         yield User(data=result)
165 |                         amount_yielded += 1
166 | 
167 |                 if obj_type == "item":
168 |                     for result in res.get("item_list", []):
169 |                         yield Video(data=result)
170 |                         amount_yielded += 1
171 | 
172 |                 if res.get("has_more", 0) == 0:
173 |                     self.parent.logger.info(
174 |                         "TikTok is not sending videos beyond this point."
175 |                     )
176 |                     return
177 | 


--------------------------------------------------------------------------------
/pytok/tiktok.py:
--------------------------------------------------------------------------------
  1 | import logging
  2 | import os
  3 | import re
  4 | import time
  5 | from typing import Optional
  6 | 
  7 | from patchright.async_api import async_playwright
  8 | from TikTokApi import TikTokApi
  9 | 
 10 | from .api.sound import Sound
 11 | from .api.user import User
 12 | from .api.search import Search
 13 | from .api.hashtag import Hashtag
 14 | from .api.video import Video
 15 | from .api.trending import Trending
 16 | 
 17 | from .exceptions import *
 18 | from .utils import LOGGER_NAME
 19 | from .captcha_solver import CaptchaSolver
 20 | from dataclasses import dataclass
 21 | 
 22 | os.environ["no_proxy"] = "127.0.0.1,localhost"
 23 | 
 24 | BASE_URL = "https://m.tiktok.com/"
 25 | DESKTOP_BASE_URL = "https://www.tiktok.com/"
 26 | 
 27 | 
 28 | class PyTok:
 29 |     _is_context_manager = False
 30 |     user = User
 31 |     search = Search
 32 |     sound = Sound
 33 |     hashtag = Hashtag
 34 |     video = Video
 35 |     trending = Trending
 36 |     logger = logging.getLogger(LOGGER_NAME)
 37 | 
 38 |     def __init__(
 39 |             self,
 40 |             logging_level: int = logging.WARNING,
 41 |             request_delay: Optional[int] = 0,
 42 |             headless: Optional[bool] = False,
 43 |             browser: Optional[str] = "chromium",
 44 |             manual_captcha_solves: Optional[bool] = False,
 45 |             log_captcha_solves: Optional[bool] = False,
 46 |             num_sessions: int = 3,
 47 |     ):
 48 |         """The PyTok class. Used to interact with TikTok. This is a singleton
 49 |             class to prevent issues from arising with playwright
 50 | 
 51 |         ##### Parameters
 52 |         * logging_level: The logging level you want the program to run at, optional
 53 |             These are the standard python logging module's levels.
 54 | 
 55 |         * request_delay: The amount of time in seconds to wait before making a request, optional
 56 |             This is used to throttle your own requests as you may end up making too
 57 |             many requests to TikTok for your IP.
 58 | 
 59 |         * num_sessions: Number of browser sessions to create (used by TikTok-Api), optional
 60 | 
 61 |         * **kwargs
 62 |             Parameters that are passed on to basically every module and methods
 63 |             that interact with this main class. These may or may not be documented
 64 |             in other places.
 65 |         """
 66 | 
 67 |         self._headless = headless
 68 |         self._request_delay = request_delay
 69 |         self._browser = browser
 70 |         self._manual_captcha_solves = manual_captcha_solves
 71 |         self._log_captcha_solves = log_captcha_solves
 72 |         self._num_sessions = num_sessions
 73 | 
 74 |         self.logger.setLevel(logging_level)
 75 | 
 76 |         # Add classes from the api folder
 77 |         User.parent = self
 78 |         Search.parent = self
 79 |         Sound.parent = self
 80 |         Hashtag.parent = self
 81 |         Video.parent = self
 82 |         Trending.parent = self
 83 | 
 84 |         self.request_cache = {}
 85 | 
 86 |         # Create TikTokApi instance for API requests
 87 |         self.tiktok_api = TikTokApi(logging_level=logging_level)
 88 | 
 89 |         if self._headless:
 90 |             from pyvirtualdisplay import Display
 91 |             self._display = Display()
 92 |             self._display.start()
 93 | 
 94 |         # options = uc.ChromeOptions()
 95 |         # options.add_argument('--ignore-ssl-errors=yes')
 96 |         # options.add_argument('--ignore-certificate-errors')
 97 |         # # options.page_load_strategy = 'eager'
 98 | 
 99 |     async def __aenter__(self):
100 |         # Create TikTok-Api sessions
101 |         suppress_resource_load_types = ['document', 'stylesheet', 'image', 'media', 'font', 'script', 'textrack', 'xhr', 'fetch', 'eventsource', 'websocket', 'manifest', 'other']
102 |         await self.tiktok_api.create_sessions(
103 |             num_sessions=self._num_sessions,
104 |             headless=self._headless,
105 |             browser=self._browser,
106 |             suppress_resource_load_types=suppress_resource_load_types,
107 |             starting_url='https://www.tiktok.com/@therock',
108 |             allow_partial_sessions=True,
109 |         )
110 | 
111 |         # Use TikTok-Api's browser session objects directly
112 |         session = self.tiktok_api.sessions[0]
113 |         self._playwright = self.tiktok_api.playwright
114 |         self._browser = self.tiktok_api.browser
115 |         self._context = session.context
116 |         self._page = session.page
117 | 
118 |         # move mouse to 0, 0 to have known mouse start position
119 |         await self._page.mouse.move(0, 0)
120 | 
121 |         self._requests = []
122 |         self._responses = []
123 | 
124 |         self._page.on("request", lambda request: self._requests.append(request))
125 | 
126 |         async def save_responses_and_body(response):
127 |             self._responses.append(response)
128 |             try:
129 |                 response._body = await response.body()
130 |             except Exception:
131 |                 pass
132 | 
133 |         self._page.on("response", save_responses_and_body)
134 | 
135 |         self._user_agent = await self._page.evaluate("() => navigator.userAgent")
136 |         self._is_context_manager = True
137 |         return self
138 | 
139 |     async def request_delay(self):
140 |         if self._request_delay is not None:
141 |             await self._page.wait_for_timeout(self._request_delay * 1000)
142 | 
143 |     def __del__(self):
144 |         """A basic cleanup method, called automatically from the code"""
145 |         if not self._is_context_manager:
146 |             self.logger.debug(
147 |                 "PyTok was shutdown improperlly. Ensure the instance is terminated with .shutdown()"
148 |             )
149 |             self.shutdown()
150 |         return
151 | 
152 |     #
153 |     # PRIVATE METHODS
154 |     #
155 | 
156 |     def r1(self, pattern, text):
157 |         m = re.search(pattern, text)
158 |         if m:
159 |             return m.group(1)
160 | 
161 |     async def shutdown(self) -> None:
162 |         try:
163 |             # Close TikTok-Api sessions (which closes browser, contexts, and playwright)
164 |             await self.tiktok_api.close_sessions()
165 |         except Exception:
166 |             pass
167 |         finally:
168 |             if self._headless:
169 |                 display = getattr(self, "_display", None)
170 |                 if display:
171 |                     display.stop()
172 | 
173 |     async def __aexit__(self, type, value, traceback):
174 |         await self.shutdown()
175 | 
176 |     async def get_ms_tokens(self):
177 |         all_cookies = await self._context.cookies()
178 |         cookie_name = 'msToken'
179 |         cookies = []
180 |         for cookie in all_cookies:
181 |             if cookie["name"] == cookie_name and cookie['secure']:
182 |                 cookies.append(cookie['value'])
183 |         if len(cookies) == 0:
184 |             raise Exception(f"Could not find {cookie_name} cookie")
185 |         return cookies
186 | 


--------------------------------------------------------------------------------
/pytok/api/hashtag.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | import json
  4 | import urllib.parse
  5 | 
  6 | from typing import TYPE_CHECKING, ClassVar, Iterator, Optional
  7 | 
  8 | import requests
  9 | 
 10 | if TYPE_CHECKING:
 11 |     from ..tiktok import PyTok
 12 |     from .video import Video
 13 | 
 14 | from .base import Base
 15 | from ..helpers import edit_url, extract_tag_contents
 16 | from ..exceptions import *
 17 | 
 18 | 
 19 | class Hashtag(Base):
 20 |     """
 21 |     A TikTok Hashtag/Challenge.
 22 | 
 23 |     Example Usage
 24 |     ```py
 25 |     hashtag = api.hashtag(name='funny')
 26 |     ```
 27 |     """
 28 | 
 29 |     parent: ClassVar[PyTok]
 30 | 
 31 |     id: Optional[str]
 32 |     """The ID of the hashtag"""
 33 |     name: Optional[str]
 34 |     """The name of the hashtag (omiting the #)"""
 35 |     as_dict: dict
 36 |     """The raw data associated with this hashtag."""
 37 | 
 38 |     def __init__(
 39 |         self,
 40 |         name: Optional[str] = None,
 41 |         id: Optional[str] = None,
 42 |         data: Optional[dict] = None,
 43 |     ):
 44 |         """
 45 |         You must provide the name or id of the hashtag.
 46 |         """
 47 |         self.name = name
 48 |         self.id = id
 49 | 
 50 |         if data is not None:
 51 |             self.as_dict = data
 52 |             self.__extract_from_data()
 53 |         else:
 54 |             self.as_dict = None
 55 | 
 56 |     async def info(self, **kwargs) -> dict:
 57 |         """
 58 |         Returns TikTok's dictionary representation of the hashtag object.
 59 |         """
 60 |         if self.as_dict is None:
 61 |             return await self.info_full(**kwargs)
 62 |         return self.as_dict
 63 | 
 64 |     async def info_full(self, **kwargs) -> dict:
 65 |         """
 66 |         Returns all information sent by TikTok related to this hashtag.
 67 | 
 68 |         Example Usage
 69 |         ```py
 70 |         hashtag_data = api.hashtag(name='funny').info_full()
 71 |         ```
 72 |         """
 73 |         page = self.parent._page
 74 | 
 75 |         url = f"https://www.tiktok.com/tag/{self.name}"
 76 |         await page.goto(url)
 77 | 
 78 |         await self.wait_for_content_or_unavailable_or_captcha('[data-e2e=challenge-item]', 'Not available')
 79 |         await self.check_and_close_signin()
 80 | 
 81 |         challenge_responses = self.get_responses("api/challenge/detail")
 82 |         challenge_responses = [request for request in challenge_responses if f"challengeName={urllib.parse.quote_plus(self.name)}" in request.url]
 83 |         if len(challenge_responses) == 0:
 84 |             raise ApiFailedException("Failed to get challenge request")
 85 |         else:
 86 |             challenge_response = challenge_responses[0]
 87 | 
 88 |         rep_body = await self.get_response_body(challenge_response)
 89 |         rep_d = json.loads(rep_body.decode('utf-8'))
 90 | 
 91 |         if 'challengeInfo' not in rep_d:
 92 |             raise ApiFailedException("Failed to get challengeInfo from response")
 93 | 
 94 |         self.as_dict = rep_d['challengeInfo']
 95 |         return self.as_dict
 96 | 
 97 |     async def videos(self, count=30, offset=0, **kwargs) -> Iterator[Video]:
 98 |         """Returns a dictionary listing TikToks with a specific hashtag.
 99 | 
100 |         - Parameters:
101 |             - count (int): The amount of videos you want returned.
102 |             - offset (int): The the offset of videos from 0 you want to get.
103 | 
104 |         Example Usage
105 |         ```py
106 |         for video in api.hashtag(name='funny').videos():
107 |             # do something
108 |         ```
109 |         """
110 |         await self.info()
111 | 
112 |         try:
113 |             async for video in self._get_videos_api(count, offset, **kwargs):
114 |                 yield video
115 |         except ApiFailedException:
116 |             async for video in self._get_videos_scraping(count, offset, **kwargs):
117 |                 yield video
118 | 
119 | 
120 |     async def _get_videos_scraping(self, count=30, offset=0, **kwargs):
121 |         processed_urls = []
122 |         amount_yielded = 0
123 |         pull_method = 'browser'
124 |         tries = 0
125 |         MAX_TRIES = 5
126 |         data_request_path = "api/challenge/item_list"
127 | 
128 |         while amount_yielded < count:
129 |             await self.parent.request_delay()
130 | 
131 |             search_requests = self.get_requests(data_request_path)
132 |             search_requests = [response for response in search_requests if f"challengeID={self.as_dict['challenge']['id']}" in response.url]
133 |             search_requests = [request for request in search_requests if request.url not in processed_urls]
134 |             for request in search_requests:
135 |                 processed_urls.append(request.url)
136 |                 response = await request.response()
137 |                 try:
138 |                     body = await self.get_response_body(response)
139 |                     res = json.loads(body)
140 |                 except:
141 |                     continue
142 |                 if res.get('type') == 'verify':
143 |                     # this is the captcha denied response
144 |                     continue
145 | 
146 |                 videos = res.get("itemList", [])
147 |                 amount_yielded += len(videos)
148 |                 for video in videos:
149 |                     yield self.parent.video(data=video)
150 | 
151 |                 if not res.get("hasMore", False):
152 |                     self.parent.logger.info(
153 |                         "TikTok isn't sending more TikToks beyond this point."
154 |                     )
155 |                     return
156 | 
157 |             for _ in range(tries):
158 |                 await self.slight_scroll_up()
159 |                 await self.scroll_to_bottom()
160 |                 await self.parent.request_delay()
161 |             
162 |                 search_requests = self.get_requests(data_request_path)
163 |                 search_requests = [request for request in search_requests if request.url not in processed_urls]
164 | 
165 |             if len(search_requests) == 0:
166 |                 tries += 1
167 |                 if tries > MAX_TRIES:
168 |                     raise
169 |                 continue
170 |                 
171 | 
172 |     async def _get_videos_api(self, count=30, offset=0, **kwargs):
173 |         responses = self.get_responses("api/challenge/item_list")
174 |         responses = [response for response in responses if f"challengeID={self.as_dict['challenge']['id']}" in response.url]
175 | 
176 |         amount_yielded = 0
177 |         cursor = 0
178 |         while amount_yielded < count:
179 |             for response in responses:
180 |                 next_url = edit_url(response.url, {"cursor": cursor})
181 |                 cookies = await self.parent._context.cookies()
182 |                 cookies = {cookie['name']: cookie['value'] for cookie in cookies}
183 |                 r = requests.get(next_url, headers=response.headers, cookies=cookies)
184 |                 if len(r.content) == 0:
185 |                     raise ApiFailedException("Response had no body")
186 |                 try:
187 |                     res = r.json()
188 |                 except json.decoder.JSONDecodeError:
189 |                     raise InvalidJSONException
190 | 
191 |                 cursor = res["cursor"]
192 |                 videos = res.get("itemList", [])
193 | 
194 |                 amount_yielded += len(videos)
195 |                 for video in videos:
196 |                     yield self.parent.video(data=video)
197 | 
198 |                 if not res.get("hasMore", False):
199 |                     self.parent.logger.info(
200 |                         "TikTok isn't sending more TikToks beyond this point."
201 |                     )
202 |                     return
203 | 
204 |     def __extract_from_data(self):
205 |         data = self.as_dict
206 |         keys = data.keys()
207 | 
208 |         if "title" in keys:
209 |             self.id = data["id"]
210 |             self.name = data["title"]
211 | 
212 |         if None in (self.name, self.id):
213 |             Hashtag.parent.logger.error(
214 |                 f"Failed to create Hashtag with data: {data}\nwhich has keys {data.keys()}"
215 |             )
216 | 
217 |     def __repr__(self):
218 |         return self.__str__()
219 | 
220 |     def __str__(self):
221 |         return f"PyTok.hashtag(id='{self.id}', name='{self.name}')"
222 | 
223 |     def __getattr__(self, name):
224 |         # TODO: Maybe switch to using @property instead
225 |         if name in ["id", "name", "as_dict"]:
226 |             self.as_dict = self.info()
227 |             self.__extract_from_data()
228 |             return self.__getattribute__(name)
229 | 
230 |         raise AttributeError(f"{name} doesn't exist on PyTok.api.Hashtag")
231 | 


--------------------------------------------------------------------------------
/pytok/captcha_solver.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | import random
  3 | from urllib.parse import urlparse
  4 | 
  5 | import cv2
  6 | import base64
  7 | import numpy as np
  8 | import requests
  9 | 
 10 | 
 11 | class CaptchaSolver:
 12 |     def __init__(self, response, puzzle, piece, page=None, mouse_step_size=1, mouse_step_delay_ms=10):
 13 |         self._request = response.request
 14 |         self._response = response
 15 |         self._client = requests.Session()
 16 |         self._puzzle = base64.b64encode(puzzle)
 17 |         self._piece = base64.b64encode(piece)
 18 |         self._page = page
 19 |         self.mouse_step_size = mouse_step_size
 20 |         self.mouse_step_delay_ms = mouse_step_delay_ms
 21 | 
 22 |     def _host(self):
 23 |         return urlparse(self._request.url).netloc
 24 | 
 25 |     def _params(self):
 26 |         return urlparse(self._request.url).query
 27 | 
 28 |     def _headers(self) -> dict:
 29 |         return self._request.headers
 30 | 
 31 |     async def _get_challenge(self) -> dict:
 32 |         return await self._response.json()
 33 | 
 34 |     async def _solve_captcha(self) -> dict:
 35 |         if self._mode == "slide":
 36 |             solver = PuzzleSolver(self._puzzle, self._piece)
 37 |             maxloc = solver.get_position()
 38 |         elif self._mode == "whirl":
 39 |             maxloc = whirl_solver(self._puzzle, self._piece)
 40 |         randlength = round(
 41 |             random.random() * (100 - 50) + 50
 42 |         )
 43 |         await asyncio.sleep(1)  # don't remove delay or it will fail
 44 |         return {
 45 |             "maxloc": maxloc,
 46 |             "randlenght": randlength
 47 |         }
 48 | 
 49 |     def _post_captcha(self, solve: dict) -> dict:
 50 |         params = self._params()
 51 | 
 52 |         body = {
 53 |             "id": solve["id"],
 54 |             "mode": self._mode
 55 |         }
 56 |         if self._mode == "slide":
 57 |             body.update({
 58 |                 "modified_img_width": 552,
 59 |                 "reply": list(
 60 |                     {
 61 |                         "relative_time": i * solve["randlenght"],
 62 |                         "x": round(
 63 |                             solve["maxloc"] / (solve["randlenght"] / (i + 1))
 64 |                         ),
 65 |                         "y": solve["tip"],
 66 |                     }
 67 |                     for i in range(
 68 |                         solve["randlenght"]
 69 |                     )
 70 |                 ),
 71 |             })
 72 |         elif self._mode == "whirl":
 73 |             body.update({
 74 |                 "modified_img_width": 340,
 75 |                 "drag_width": 271,
 76 |                 "reply": list(
 77 |                     {
 78 |                         "relative_time": i * solve["randlenght"],
 79 |                         "x": round(
 80 |                             271 * solve["maxloc"] / (solve["randlenght"] / (i + 1))
 81 |                         ),
 82 |                         "y": solve["tip"],
 83 |                     }
 84 |                     for i in range(
 85 |                         solve["randlenght"]
 86 |                     )
 87 |                 ),
 88 |             })
 89 | 
 90 |         host = self._host()
 91 |         headers = self._headers()
 92 | 
 93 |         resp = self._client.post(
 94 |             url=(
 95 |                     "https://"
 96 |                     + host
 97 |                     + "/captcha/verify?"
 98 |                     + params
 99 |             ),
100 |             headers=headers.update(
101 |                 {
102 |                     "content-type": "application/json"
103 |                 }
104 |             ),
105 |             json=body
106 |         )
107 | 
108 |         if resp.status_code != 200:
109 |             raise Exception("Captcha was not solved")
110 |         else:
111 |             # status code was 200, but perhaps the response was to say that the CAPTCHA failed.
112 |             if resp.json()['code'] >= 500:
113 |                 raise Exception(f"CAPTCHA server responded 200 but said: {resp.json()['message']}")
114 | 
115 |         return resp.json()
116 | 
117 |     async def solve_captcha(self):
118 |         # this method is called
119 |         captcha_challenge = await self._get_challenge()
120 | 
121 |         if 'mode' in captcha_challenge["data"]:
122 |             captcha_challenge = captcha_challenge["data"]
123 |         elif 'challenges' in captcha_challenge["data"]:
124 |             captcha_challenge = captcha_challenge["data"]["challenges"][0]
125 |         captcha_id = captcha_challenge["id"]
126 |         self._mode = captcha_challenge["mode"]
127 | 
128 |         solve = await self._solve_captcha()
129 | 
130 |         solve['id'] = captcha_id
131 |         if captcha_challenge["mode"] == "slide":
132 |             tip_y = captcha_challenge["question"]["tip_y"]
133 |             solve['tip'] = tip_y
134 |         elif captcha_challenge["mode"] == "whirl":
135 |             solve['tip'] = 0
136 |         return solve
137 | 
138 |     async def solve_and_drag(self):
139 |         """Solve the captcha and perform the drag operation"""
140 |         if not self._page:
141 |             raise ValueError("Page object is required for solve_and_drag method")
142 | 
143 |         # Get the solution
144 |         solve = await self.solve_captcha()
145 | 
146 |         # Perform the drag
147 |         if self._mode == "slide":
148 |             await self._drag_puzzle_slider(solve['maxloc'])
149 |         elif self._mode == "whirl":
150 |             await self._drag_whirl_slider(solve['maxloc'])
151 | 
152 |         return solve
153 | 
154 |     async def _drag_element_horizontal(self, css_selector: str, x_offset: int) -> None:
155 |         """
156 |         Drag an element horizontally with realistic mouse movement.
157 |         Based on tiktok-captcha-solver implementation.
158 |         """
159 |         e = self._page.locator(css_selector)
160 |         box = await e.bounding_box()
161 |         if not box:
162 |             raise AttributeError("Element had no bounding box")
163 | 
164 |         # Start position - slightly offset from center
165 |         start_x = (box["x"] + (box["width"] / 1.337))
166 |         start_y = (box["y"] + (box["height"] / 1.337))
167 | 
168 |         # Move to start position
169 |         await self._page.mouse.move(start_x, start_y)
170 |         await asyncio.sleep(random.randint(1, 10) / 11)
171 | 
172 |         # Press mouse down
173 |         await self._page.mouse.down()
174 | 
175 |         # Drag with small incremental steps (overshoot by 5 pixels)
176 |         for pixel in range(0, x_offset + 5, self.mouse_step_size):
177 |             await self._page.mouse.move(start_x + pixel, start_y)
178 |             await self._page.wait_for_timeout(self.mouse_step_delay_ms)
179 | 
180 |         await asyncio.sleep(0.25)
181 | 
182 |         # Correction movements (simulate human overshoot correction)
183 |         for pixel in range(-5, 2):
184 |             await self._page.mouse.move(start_x + x_offset - pixel, start_y + pixel)
185 |             await self._page.wait_for_timeout(self.mouse_step_delay_ms // 2)
186 | 
187 |         await asyncio.sleep(0.2)
188 | 
189 |         # Final smooth positioning
190 |         await self._page.mouse.move(start_x + x_offset, start_y, steps=75)
191 |         await asyncio.sleep(0.3)
192 | 
193 |         # Release mouse
194 |         await self._page.mouse.up()
195 | 
196 |     async def _drag_puzzle_slider(self, maxloc: float) -> None:
197 |         """Drag the puzzle slider to solve slide captcha"""
198 |         # Get elements
199 |         drag = self._page.locator('css=div.secsdk-captcha-drag-icon').first
200 |         bar = self._page.locator('css=div.captcha_verify_slide--slidebar').first
201 | 
202 |         drag_bounding_box = await drag.bounding_box()
203 |         bar_bounding_box = await bar.bounding_box()
204 | 
205 |         if not drag_bounding_box or not bar_bounding_box:
206 |             raise AttributeError("Could not get bounding boxes for drag elements")
207 | 
208 |         # Calculate drag distance
209 |         bar_effective_width = bar_bounding_box['width'] - drag_bounding_box['width']
210 |         distance_to_drag = int(bar_effective_width * maxloc)
211 | 
212 |         # Perform the drag
213 |         await self._drag_element_horizontal('css=div.secsdk-captcha-drag-icon', distance_to_drag)
214 | 
215 |     async def _drag_whirl_slider(self, maxloc: float) -> None:
216 |         """Drag the whirl/rotate slider"""
217 |         # For whirl captcha, need to calculate based on rotation
218 |         # This is a simplified version - may need adjustment based on actual implementation
219 |         slide_bar_width = 340  # Default from original code
220 |         slide_button_width = 69  # Approximate
221 |         distance = int((slide_bar_width - slide_button_width) * maxloc)
222 | 
223 |         await self._drag_element_horizontal('css=div.secsdk-captcha-drag-icon', distance)
224 | 
225 | 
226 | class PuzzleSolver:
227 |     def __init__(self, base64puzzle, base64piece):
228 |         self.puzzle = base64puzzle
229 |         self.piece = base64piece
230 | 
231 |     def get_position(self):
232 |         puzzle = self._background_preprocessing()
233 |         piece = self._piece_preprocessing()
234 |         matched = cv2.matchTemplate(
235 |             puzzle,
236 |             piece,
237 |             cv2.TM_CCOEFF_NORMED
238 |         )
239 |         min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(matched)
240 |         return max_loc[0]
241 | 
242 |     def _background_preprocessing(self):
243 |         img = self._img_to_grayscale(self.piece)
244 |         background = self._sobel_operator(img)
245 |         return background
246 | 
247 |     def _piece_preprocessing(self):
248 |         img = self._img_to_grayscale(self.puzzle)
249 |         template = self._sobel_operator(img)
250 |         return template
251 | 
252 |     def _sobel_operator(self, img):
253 |         scale = 1
254 |         delta = 0
255 |         ddepth = cv2.CV_16S
256 | 
257 |         img = cv2.GaussianBlur(img, (3, 3), 0)
258 |         gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
259 |         grad_x = cv2.Sobel(
260 |             gray,
261 |             ddepth,
262 |             1,
263 |             0,
264 |             ksize=3,
265 |             scale=scale,
266 |             delta=delta,
267 |             borderType=cv2.BORDER_DEFAULT,
268 |         )
269 |         grad_y = cv2.Sobel(
270 |             gray,
271 |             ddepth,
272 |             0,
273 |             1,
274 |             ksize=3,
275 |             scale=scale,
276 |             delta=delta,
277 |             borderType=cv2.BORDER_DEFAULT,
278 |         )
279 |         abs_grad_x = cv2.convertScaleAbs(grad_x)
280 |         abs_grad_y = cv2.convertScaleAbs(grad_y)
281 |         grad = cv2.addWeighted(abs_grad_x, 0.5, abs_grad_y, 0.5, 0)
282 | 
283 |         return grad
284 | 
285 |     def _img_to_grayscale(self, img):
286 |         return cv2.imdecode(
287 |             self._string_to_image(img),
288 |             cv2.IMREAD_COLOR
289 |         )
290 | 
291 |     def _string_to_image(self, base64_string):
292 |         return np.frombuffer(
293 |             base64.b64decode(base64_string),
294 |             dtype="uint8"
295 |         )
296 | 
297 | 
298 | def _get_images_and_edges(b64_puzzle, b64_piece, resolution=300):
299 |     puzzle = cv2.imdecode(np.frombuffer(base64.b64decode(b64_puzzle), dtype="uint8"), cv2.IMREAD_COLOR)
300 |     piece = cv2.imdecode(np.frombuffer(base64.b64decode(b64_piece), dtype="uint8"), cv2.IMREAD_COLOR)
301 | 
302 |     # get inner edge of puzzle
303 |     r = (piece.shape[0] / 2) + 1
304 |     puzzle_edge = np.zeros((resolution, 3))
305 |     for idx, theta in enumerate(np.linspace(0, 2 * np.pi, resolution)):
306 |         x = int(puzzle.shape[0] / 2 + r * np.cos(theta))
307 |         y = int(puzzle.shape[1] / 2 + r * np.sin(theta))
308 |         puzzle_edge[idx] = puzzle[x, y]
309 | 
310 |     # get outer edge of piece
311 |     r = (piece.shape[0] / 2) - 1
312 |     piece_edge = np.zeros((resolution, 3))
313 |     for idx, theta in enumerate(np.linspace(0, 2 * np.pi, resolution)):
314 |         x = min(int(piece.shape[0] / 2 + r * np.cos(theta)), piece.shape[0] - 1)
315 |         y = min(int(piece.shape[1] / 2 + r * np.sin(theta)), piece.shape[1] - 1)
316 |         piece_edge[idx] = piece[x, y]
317 | 
318 |     return puzzle, piece, puzzle_edge, piece_edge
319 | 
320 | 
321 | def whirl_solver(b64_puzzle, b64_piece):
322 |     resolution = 300
323 |     _, _, puzzle_edge, piece_edge = _get_images_and_edges(b64_puzzle, b64_piece, resolution=resolution)
324 | 
325 |     # find the best match
326 |     best_match = 0
327 |     best_angle = 0
328 |     for angle in range(resolution):
329 |         match = np.sum(puzzle_edge * np.roll(piece_edge, angle, axis=0))
330 |         if match > best_match:
331 |             best_match = match
332 |             best_angle = angle
333 | 
334 |     return (resolution - best_angle) / resolution
335 | 


--------------------------------------------------------------------------------
/pytok/utils.py:
--------------------------------------------------------------------------------
  1 | from datetime import datetime
  2 | import json
  3 | import os
  4 | import re
  5 | 
  6 | import numpy as np
  7 | import pandas as pd
  8 | import polars as pl
  9 | import tqdm
 10 | 
 11 | LOGGER_NAME: str = "PyTok"
 12 | 
 13 | 
 14 | def update_if_not_none(dict1, dict2):
 15 |     dict1.update((k, v) for k, v in dict2.items() if v is not None)
 16 |     return dict1
 17 | 
 18 | 
 19 | def _get_comment_features(comment):
 20 |     comment_user = comment['user']
 21 |     if isinstance(comment_user, str):
 22 |         raise ValueError()
 23 |     elif isinstance(comment_user, dict):
 24 |         if 'unique_id' in comment_user:
 25 |             author_id = comment_user['uid']
 26 |             author_name = comment_user['unique_id']
 27 |         elif 'uniqueId' in comment_user:
 28 |             author_id = comment_user['id']
 29 |             author_name = comment_user['uniqueId']
 30 |         else:
 31 |             author_name = ''
 32 |             author_id = comment_user['uid']
 33 |     else:
 34 |         raise ValueError()
 35 | 
 36 |     mentioned_users = [info['user_id'] for info in comment['text_extra'] if 'user_id' in info and info['user_id'] != '']
 37 | 
 38 |     return author_id, author_name, mentioned_users
 39 | 
 40 | 
 41 | def load_comment_df_from_files(file_paths):
 42 |     comments = []
 43 |     for file_path in tqdm.tqdm(file_paths):
 44 | 
 45 |         if not os.path.exists(file_path):
 46 |             continue
 47 | 
 48 |         with open(file_path, 'r') as f:
 49 |             comments = json.load(f)
 50 |         comments.extend(comments)
 51 | 
 52 |     return get_comment_df(comments)
 53 | 
 54 | 
 55 | def get_comment_df(comments):
 56 |     comments_data = []
 57 |     for comment in comments:
 58 | 
 59 |         try:
 60 |             author_id, author_name, mentioned_users = _get_comment_features(comment)
 61 |         except ValueError:
 62 |             continue
 63 | 
 64 |         comment_replies = comment.get('reply_comment', None)
 65 |         if comment_replies:
 66 |             for reply_comment in comment_replies:
 67 |                 try:
 68 |                     reply_author_id, reply_author_name, reply_mentioned_users = _get_comment_features(reply_comment)
 69 |                 except ValueError:
 70 |                     continue
 71 | 
 72 |                 comments_data.append((
 73 |                     reply_comment['cid'],
 74 |                     datetime.utcfromtimestamp(reply_comment['create_time']),
 75 |                     reply_author_name,
 76 |                     reply_author_id,
 77 |                     reply_comment['text'],
 78 |                     reply_mentioned_users,
 79 |                     reply_comment['aweme_id'],
 80 |                     reply_comment['comment_language'],
 81 |                     reply_comment['digg_count'],
 82 |                     comment['cid']
 83 |                 ))
 84 | 
 85 |         comments_data.append((
 86 |             comment['cid'],
 87 |             datetime.utcfromtimestamp(comment['create_time']),
 88 |             author_name,
 89 |             author_id,
 90 |             comment['text'],
 91 |             mentioned_users,
 92 |             comment['aweme_id'],
 93 |             comment['comment_language'],
 94 |             comment['digg_count'],
 95 |             None
 96 |         ))
 97 | 
 98 |     comment_df = pd.DataFrame(comments_data,
 99 |                               columns=['comment_id', 'createtime', 'author_name', 'author_id', 'text', 'mentions',
100 |                                        'video_id', 'comment_language', 'like_count', 'reply_comment_id'])
101 |     comment_df = comment_df.drop_duplicates('comment_id')
102 |     comment_df = comment_df[comment_df['text'].notna()]
103 |     comment_df = comment_df[comment_df['video_id'].notna()]
104 |     comment_df = comment_df[comment_df['mentions'].notna()]
105 |     comment_df['text'] = comment_df['text'].str.replace(r'\n', ' ', regex=True)
106 |     comment_df['text'] = comment_df['text'].str.replace(r'\r', ' ', regex=True)
107 |     return comment_df
108 | 
109 | 
110 | def try_load_comment_df_from_file(file_path, file_paths=[]):
111 |     assert file_path.endswith('.parquet.gzip') or file_path.endswith('.csv'), "File path must be a parquet or csv file"
112 | 
113 |     if os.path.exists(file_path):
114 |         if file_path.endswith('.csv'):
115 |             comment_df = pd.read_csv(file_path)
116 |         elif file_path.endswith('.parquet.gzip'):
117 |             comment_df = pd.read_parquet(file_path)
118 |         comment_df[
119 |             ['comment_id', 'author_name', 'author_id', 'text', 'video_id', 'comment_language', 'reply_comment_id']] = \
120 |         comment_df[['comment_id', 'author_name', 'author_id', 'text', 'video_id', 'comment_language',
121 |                     'reply_comment_id']].astype(str)
122 |         comment_df['mentions'] = comment_df['mentions'].apply(_str_to_list)
123 |         comment_df['createtime'] = pd.to_datetime(comment_df['createtime'])
124 |         comment_df['createtime'] = comment_df['createtime'].astype('datetime64[ns]')
125 |     else:
126 |         if not file_paths:
127 |             raise ValueError(
128 |                 f"Parquet file: {file_path} does not exist, and no file paths provided to generate dataframe")
129 | 
130 |         comment_df = load_comment_df_from_files(file_paths)
131 | 
132 |         if file_path.endswith('.csv'):
133 |             comment_df.to_csv(file_path, index=False)
134 |         elif file_path.endswith('.parquet.gzip'):
135 |             comment_df.to_parquet(file_path, compression='gzip', index=False)
136 | 
137 |     return comment_df
138 | 
139 | 
140 | def _str_to_list(stri):
141 |     if ',' not in stri:
142 |         return []
143 |     return [word.strip()[1:-1] for word in stri[1:-1].split(',')]
144 | 
145 | 
146 | def try_load_video_df_from_file(file_path, file_paths=[]):
147 |     assert file_path.endswith('.parquet.gzip') or file_path.endswith('.csv'), "File path must be a parquet or csv file"
148 |     if os.path.exists(file_path):
149 |         if file_path.endswith('.csv'):
150 |             video_df = pd.read_csv(file_path)
151 |         elif file_path.endswith('.parquet.gzip'):
152 |             video_df = pd.read_parquet(file_path)
153 | 
154 |         video_df[
155 |             ['video_id', 'author_name', 'author_id', 'desc', 'share_video_id', 'share_video_user_id', 'share_type']] = \
156 |         video_df[['video_id', 'author_name', 'author_id', 'desc', 'share_video_id', 'share_video_user_id',
157 |                   'share_type']].astype(str)
158 |         video_df['createtime'] = pd.to_datetime(video_df['createtime'])
159 |         video_df['mentions'] = video_df['mentions'].apply(_str_to_list)
160 |         video_df['hashtags'] = video_df['hashtags'].apply(_str_to_list)
161 |         return video_df
162 | 
163 |     else:
164 |         if not file_paths:
165 |             raise ValueError(f"File: {file_path} does not exist, and no file paths provided to generate dataframe")
166 | 
167 |         videos = []
168 |         for file_path in file_paths:
169 |             with open(file_path, 'r') as f:
170 |                 file_data = json.load(f)
171 | 
172 |             if type(file_data) == list:
173 |                 videos += file_data
174 |             elif type(file_data) == dict:
175 |                 videos.append(file_data)
176 |             else:
177 |                 raise ValueError()
178 | 
179 |         video_df = get_video_df(videos)
180 |         if file_path.endswith('.csv'):
181 |             video_df.to_csv(file_path, index=False)
182 |         elif file_path.endswith('.parquet.gzip'):
183 |             video_df.to_parquet(file_path, compression='gzip', index=False)
184 |         return video_df
185 | 
186 | 
187 | def extract_video_features(video):
188 |     # get text extra relating to user names
189 |     video_mentions = [extra for extra in video.get('textExtra', []) if
190 |                       extra.get('userId', None) and extra['userId'] != '0']
191 | 
192 |     # get all hashtags used in the description
193 |     hashtags = [extra['hashtagName'] for extra in video.get('textExtra', []) if extra.get('hashtagName', None)]
194 | 
195 |     # get all reply types
196 |     match = re.search(r"^\#([^# ]+) [^@# ]+ @([^ ]+)", video['desc'])
197 |     if match and len(video_mentions) > 0:
198 |         # if there are multiple mentions we get the first
199 |         if video_mentions[0]['awemeId'] != '':
200 |             share_video_id = video_mentions[0]['awemeId']
201 |         elif video.get('duetInfo', None) and video['duetInfo']['duetFromId'] != '0':
202 |             share_video_id = video['duetInfo']['duetFromId']
203 |         else:
204 |             # no way to get shared video id
205 |             share_video_id = None
206 | 
207 |         share_video_user_id = video_mentions[0]['userId']
208 |         share_video_user_name = video_mentions[0]['userUniqueId'] if 'userUniqueId' in video_mentions[
209 |             0].keys() else None
210 |         share_type = match.group(1)
211 | 
212 |         video_mentions = video_mentions[1:]
213 |     else:
214 |         share_video_id = None
215 |         share_video_user_id = None
216 |         share_video_user_name = None
217 |         share_type = None
218 | 
219 |     # get duets that we didn't get with the regex
220 |     if video.get('duetFromId', None) and video['duetFromId'] != '0' and not share_video_id:
221 |         duet_infos = [mention for mention in video_mentions if mention['awemeId'] == video['duetInfo']['duetFromId']]
222 |         # sometimes the awemeId is missing
223 |         if duet_infos:
224 |             duet_info = duet_infos[0]
225 |             share_video_id = duet_info['awemeId']
226 |         else:
227 |             duet_info = video_mentions[0]
228 |             share_video_id = video['duetInfo']['duetFromId']
229 | 
230 |         share_video_user_id = duet_info['userId']
231 |         share_video_user_name = duet_info['userUniqueId']
232 |         share_type = 'duet'
233 | 
234 |         video_mentions = [mention for mention in video_mentions if
235 |                           mention['awemeId'] != video['duetInfo']['duetFromId']]
236 | 
237 |     # get user mentions
238 |     mentions = []
239 |     if len(video_mentions) > 0:
240 |         mentions = [mention['userId'] for mention in video_mentions]
241 | 
242 |     if video.get('duetInfo', None) and video['duetInfo']['duetFromId'] != '0' and share_video_id and video['duetInfo'][
243 |         'duetFromId'] != share_video_id:
244 |         raise ValueError("Comment metadata is mismatched")
245 | 
246 |     ai_gc_label_type = None
247 |     if 'ai_gc_label_type' in video:
248 |         if video['ai_gc_label_type'] is not None and np.isnan(video['ai_gc_label_type']):
249 |             video['ai_gc_label_type'] = None
250 |         ai_gc_label_type = video['ai_gc_label_type']
251 | 
252 |     vid_features = {
253 |         'video_id': video['id'],
254 |         'createtime': datetime.utcfromtimestamp(int(video['createTime'])),
255 |         'author_name': video['author']['uniqueId'],
256 |         'author_id': video['author']['id'],
257 |         'desc': video['desc'],
258 |         'hashtags': hashtags,
259 |         'share_video_id': share_video_id,
260 |         'share_video_user_id': share_video_user_id,
261 |         'share_video_user_name': share_video_user_name,
262 |         'share_type': share_type,
263 |         'mentions': mentions,
264 |         'digg_count': video['stats']['diggCount'],
265 |         'share_count': video['stats']['shareCount'],
266 |         'comment_count': video['stats']['commentCount'],
267 |         'view_count': video['stats']['playCount'],
268 |         'ai_gc_label_type': ai_gc_label_type
269 |     }
270 |     return vid_features
271 | 
272 | 
273 | def get_video_df(videos) -> pl.DataFrame:
274 |     vids_data = []
275 |     for video in videos:
276 |         vid_features = extract_video_features(video)
277 |         vid_data = vid_features | {k: v for k, v in video.items() if k not in ['id', 'desc', 'createTime']}
278 |         if 'video' in vid_data and vid_data['video']['volumeInfo'] is not None and '' in vid_data['video']['volumeInfo']:
279 |             del vid_data['video']['volumeInfo']['']
280 |         vids_data.append(vid_data)
281 | 
282 |     try:
283 |         video_df = pl.from_dicts(vids_data)
284 |     except (Exception, pl.exceptions.PanicException):
285 |         video_df = pl.from_dicts(vids_data, infer_schema_length=len(vids_data))
286 | 
287 |     return video_df
288 | 
289 | 
290 | def try_load_user_df_from_file(file_path, file_paths=[]):
291 |     assert file_path.endswith('.parquet.gzip') or file_path.endswith('.csv'), "File path must be a parquet or csv file"
292 | 
293 |     if os.path.exists(file_path):
294 |         if file_path.endswith('.csv'):
295 |             user_df = pd.read_csv(file_path)
296 |         elif file_path.endswith('.parquet.gzip'):
297 |             user_df = pd.read_parquet(file_path)
298 | 
299 |         user_df['id'] = user_df['id'].astype(str)
300 |         user_df['num_following'] = user_df['num_following'].astype('Int64')
301 |         user_df['num_followers'] = user_df['num_followers'].astype('Int64')
302 |         user_df['num_videos'] = user_df['num_videos'].astype('Int64')
303 |         user_df['num_likes'] = user_df['num_likes'].astype('Int64')
304 |         user_df['createtime'] = pd.to_datetime(user_df['createtime'])
305 |         return user_df
306 | 
307 |     else:
308 |         if not file_paths:
309 |             raise ValueError(f"File: {file_path} does not exist, and no file paths provided to generate dataframe")
310 | 
311 |         entities = []
312 |         for file_path in tqdm.tqdm(file_paths):
313 |             if not os.path.exists(file_path):
314 |                 continue
315 | 
316 |             with open(file_path, 'r') as f:
317 |                 file_data = json.load(f)
318 | 
319 |             if isinstance(file_data, list):
320 |                 entities += file_data
321 |             else:
322 |                 raise ValueError()
323 | 
324 |         user_df = get_user_df(entities)
325 |         # protect against people with \r as nickname, how dare they
326 |         if file_path.endswith('.csv'):
327 |             user_df.to_csv(file_path, index=False, lineterminator="\r\n")
328 |         elif file_path.endswith('.parquet.gzip'):
329 |             user_df.to_parquet(file_path, compression='gzip', index=False)
330 |         return user_df
331 | 
332 | 
333 | def get_user_df(entities) -> pl.DataFrame:
334 |     users = {}
335 |     for entity in entities:
336 |         if 'user' in entity:
337 |             user_info = entity['user']
338 |             if 'stats' in entity:
339 |                 user_info.update(entity['stats'])
340 |             if isinstance(user_info, dict):
341 |                 if 'unique_id' in user_info:
342 |                     user_id = user_info['unique_id']
343 |                 elif 'uniqueId' in user_info:
344 |                     user_id = user_info['uniqueId']
345 |                 else:
346 |                     continue
347 | 
348 |                 if user_id in users:
349 |                     users[user_id] = update_if_not_none(users[user_id], user_info)
350 |                 else:
351 |                     users[user_id] = user_info
352 | 
353 |             elif isinstance(user_info, str):
354 |                 if user_info not in users:
355 |                     users[user_id] = {'unique_id': user_info}
356 | 
357 |         elif 'author' in entity:
358 |             user_info = entity['author'] | entity['authorStats']
359 | 
360 |             user_id = user_info['uniqueId']
361 |             if user_id in users:
362 |                 users[user_id] = update_if_not_none(users[user_id], user_info)
363 |             else:
364 |                 users[user_id] = user_info
365 | 
366 |         elif 'followerCount' in entity:
367 |             user_info = entity
368 |             user_id = user_info['uniqueId']
369 |             if user_id in users:
370 |                 users[user_id] = update_if_not_none(users[user_id], user_info)
371 |             else:
372 |                 users[user_id] = user_info
373 | 
374 |         elif 'userInfo' in entity:
375 |             user_info = entity['userInfo']['user']
376 |             user_info.update(entity['userInfo']['stats'])
377 |             user_id = user_info['uniqueId']
378 |             if user_id in users:
379 |                 users[user_id] = update_if_not_none(users[user_id], user_info)
380 |             else:
381 |                 users[user_id] = user_info
382 | 
383 |         else:
384 |             raise ValueError("Unknown entity type")
385 | 
386 |     if len(users) == 0:
387 |         raise ValueError("No users found in entities")
388 | 
389 |     user_df = pd.DataFrame(list(users.values()))
390 | 
391 |     if 'unique_id' in user_df.columns:
392 |         user_df['uniqueId'] = user_df['unique_id'].combine_first(user_df['uniqueId'])
393 |         user_df = user_df.drop(columns=['unique_id'])
394 | 
395 |     user_df = user_df.drop_duplicates('uniqueId')
396 | 
397 |     if 'uid' in user_df.columns:
398 |         user_df['id'] = user_df['id'].combine_first(user_df['uid'])
399 |         user_df = user_df.drop(columns=['uid'])
400 | 
401 |     # thank you dfir!!! https://dfir.blog/tinkering-with-tiktok-timestamps/
402 |     user_df.loc[user_df['id'].notna(), 'createtime'] = user_df.loc[user_df['id'].notna(), 'id'].apply(
403 |         lambda x: datetime.utcfromtimestamp(int(x) >> 32))
404 |     user_df['createtime'] = pd.to_datetime(user_df['createtime'], utc=True)
405 |     user_df[['followingCount', 'followerCount', 'videoCount', 'diggCount']] = user_df[
406 |         ['followingCount', 'followerCount', 'videoCount', 'diggCount']].astype('Int64')
407 |     # excluding because it messes up the csv and its not accessible anyway
408 |     # user_df['avatarThumb'] = user_df['avatarThumb'].combine_first(user_df['avatar_thumb'])
409 |     if 'avatar_thumb' in user_df.columns:
410 |         user_df = user_df.drop(columns=['avatar_thumb'])
411 |     user_df = user_df[
412 |         ['id', 'uniqueId', 'nickname', 'signature', 'verified', 'followingCount', 'followerCount', 'videoCount',
413 |          'diggCount', 'createtime']]
414 |     user_df = user_df.rename(columns={
415 |         'uniqueId': 'unique_id',
416 |         'followingCount': 'num_following',
417 |         'followerCount': 'num_followers',
418 |         'videoCount': 'num_videos',
419 |         'diggCount': 'num_likes'
420 |     })
421 | 
422 |     return pl.from_pandas(user_df)


--------------------------------------------------------------------------------
/pytok/api/user.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | 
  4 | import asyncio
  5 | import json
  6 | import re
  7 | from urllib.parse import urlparse
  8 | 
  9 | from patchright.async_api import TimeoutError as PlaywrightTimeoutError
 10 | 
 11 | from ..exceptions import *
 12 | from ..helpers import extract_tag_contents
 13 | 
 14 | from typing import TYPE_CHECKING, ClassVar, Iterator, Optional
 15 | 
 16 | if TYPE_CHECKING:
 17 |     from ..tiktok import PyTok
 18 |     from .video import Video
 19 | 
 20 | from .base import Base
 21 | 
 22 | 
 23 | class User(Base):
 24 |     """
 25 |     A TikTok User.
 26 | 
 27 |     Example Usage
 28 |     ```py
 29 |     user = api.user(username='therock')
 30 |     # or
 31 |     user_id = '5831967'
 32 |     sec_uid = 'MS4wLjABAAAA-VASjiXTh7wDDyXvjk10VFhMWUAoxr8bgfO1kAL1-9s'
 33 |     user = api.user(user_id=user_id, sec_uid=sec_uid)
 34 |     ```
 35 | 
 36 |     """
 37 | 
 38 |     parent: ClassVar[PyTok]
 39 | 
 40 |     user_id: str
 41 |     """The user ID of the user."""
 42 |     sec_uid: str
 43 |     """The sec UID of the user."""
 44 |     username: str
 45 |     """The username of the user."""
 46 |     as_dict: dict
 47 |     """The raw data associated with this user."""
 48 | 
 49 |     def __init__(
 50 |             self,
 51 |             username: Optional[str] = None,
 52 |             user_id: Optional[str] = None,
 53 |             sec_uid: Optional[str] = None,
 54 |             data: Optional[dict] = None,
 55 |     ):
 56 |         """
 57 |         You must provide the username or (user_id and sec_uid) otherwise this
 58 |         will not function correctly.
 59 |         """
 60 |         self.__update_id_sec_uid_username(user_id, sec_uid, username)
 61 |         if data is not None:
 62 |             self.as_dict = data
 63 |             self.__extract_from_data()
 64 |         else:
 65 |             self.as_dict = {}
 66 | 
 67 |     def info(self, **kwargs):
 68 |         """
 69 |         Returns a dictionary of TikTok's User object
 70 | 
 71 |         Example Usage
 72 |         ```py
 73 |         user_data = api.user(username='therock').info()
 74 |         ```
 75 |         """
 76 |         return self.info_full(**kwargs)
 77 | 
 78 |     async def info_full(self, **kwargs) -> dict:
 79 |         """
 80 |         Returns a dictionary of information associated with this User.
 81 |         Includes statistics about this user.
 82 | 
 83 |         Example Usage
 84 |         ```py
 85 |         user_data = api.user(username='therock').info_full()
 86 |         ```
 87 |         """
 88 | 
 89 |         # TODO: Find the one using only user_id & sec_uid
 90 |         if not self.username:
 91 |             raise TypeError(
 92 |                 "You must provide the username when creating this class to use this method."
 93 |             )
 94 | 
 95 |         url = f"https://www.tiktok.com/@{self.username}?lang=en"
 96 | 
 97 |         page = self.parent._page
 98 |         
 99 |         self.parent.logger.debug(f"Loading page: {url}")
100 |         if page.url != url:
101 |             try:
102 |                 async with page.expect_request(url) as event:
103 |                     await page.goto(url, timeout=60 * 1000)
104 |                     request = await event.value
105 |                     response = await request.response()
106 |                     if response.status >= 300:
107 |                         raise NotAvailableException("Content is not available")
108 |             except PlaywrightTimeoutError:
109 |                 raise TimeoutException("Page load timed out")
110 | 
111 |         # try:
112 |         await self.wait_for_content_or_unavailable_or_captcha('[data-e2e="user-post-item"]',
113 |                                                             "Couldn't find this account",
114 |                                                             no_content_text=["No content", "This account is private"])
115 |         # resolve any remaining issues
116 |         await asyncio.sleep(0.5)
117 |         await self.wait_for_content_or_unavailable_or_captcha('[data-e2e="user-post-item"]',
118 |                                                             "Couldn't find this account",
119 |                                                             no_content_text=["No content", "This account is private"])
120 | 
121 |         data_responses = self.get_responses('api/user/detail')
122 | 
123 |         if len(data_responses) > 0:
124 |             data_response = data_responses[-1]
125 |             data = await data_response.json()
126 |             user_info = data["userInfo"]
127 |             user = user_info["user"] | user_info["stats"]
128 |             self.as_dict = user
129 |             self.__extract_from_data()
130 |             return user
131 |         else:
132 |             # get initial html data
133 |             html_body = await page.content()
134 |             
135 |             tag_contents = extract_tag_contents(html_body)
136 |             self.initial_json = json.loads(tag_contents)
137 | 
138 |             if 'UserModule' in self.initial_json:
139 |                 user = self.initial_json["UserModule"]["users"][self.username] | self.initial_json["UserModule"]["stats"][self.username]
140 |             elif '__DEFAULT_SCOPE__' in self.initial_json:
141 |                 user_detail = self.initial_json['__DEFAULT_SCOPE__']['webapp.user-detail']
142 |                 if user_detail['statusCode'] != 0:
143 |                     raise InvalidJSONException("Failed to find user data in HTML")
144 |                 user_info = user_detail['userInfo']
145 |                 user = user_info['user'] | user_info['stats']
146 |             else:
147 |                 raise InvalidJSONException("Failed to find user data in HTML")
148 | 
149 |             self.as_dict = user
150 |             self.__extract_from_data()
151 |             return user
152 | 
153 |     async def videos(self, get_bytes=False, count=None, batch_size=100, **kwargs) -> Iterator[Video]:
154 |         """
155 |         Returns an iterator yielding Video objects.
156 | 
157 |         - Parameters:
158 |             - count (int): The amount of videos you want returned.
159 |             - cursor (int): The unix epoch to get uploaded videos since.
160 | 
161 |         Example Usage
162 |         ```py
163 |         user = api.user(username='therock')
164 |         for video in user.videos(count=100):
165 |             # do something
166 |         ```
167 |         """
168 |         if self.as_dict and self.as_dict.get('videoCount', 1) == 0:
169 |             return
170 | 
171 |         try:
172 |             videos, finished, cursor = await self._get_initial_videos(count, get_bytes)
173 |             self.parent.logger.info(f"Got {len(videos)} initial videos, finished={finished}, cursor={cursor}")
174 |             for video in videos:
175 |                 yield video
176 | 
177 |             if finished or count and len(videos) >= count:
178 |                 self.parent.logger.info(f"Finished after initial videos")
179 |                 return
180 | 
181 |             self.parent.logger.info(f"Continuing with _get_videos_api to get more videos")
182 |             async for video in self._get_videos_api(count, cursor, get_bytes, **kwargs):
183 |                 yield video
184 |         except ApiFailedException as ex:
185 |             self.parent.logger.warning(f"API method failed with exception: {ex}. Falling back to scraping method.")
186 |             async for video in self._get_videos_scraping(count, get_bytes):
187 |                 yield video
188 |         except Exception as ex:
189 |             raise
190 | 
191 |     async def _get_videos_api(self, count, cursor, get_bytes, **kwargs) -> Iterator[Video]:
192 |         # Use TikTok-Api's make_request method instead of manual requests
193 |         self.parent.logger.debug(f"Starting _get_videos_api with cursor={cursor}, count={count}")
194 |         amount_yielded = 0
195 | 
196 |         while (count is None or amount_yielded < count):
197 |             params = {
198 |                 'secUid': self.sec_uid,
199 |                 'count': 35,
200 |                 'cursor': cursor,
201 |             }
202 | 
203 |             self.parent.logger.debug(f"Making TikTok-Api request with cursor={cursor}")
204 |             # Use TikTok-Api's make_request which handles signing and headers
205 |             res = await self.parent.tiktok_api.make_request(
206 |                 url="https://www.tiktok.com/api/post/item_list/",
207 |                 params=params,
208 |             )
209 |             self.parent.logger.debug(f"TikTok-Api response received with {len(res.get('itemList', []))} videos")
210 | 
211 |             if res is None:
212 |                 raise ApiFailedException("TikTok-Api returned None response")
213 | 
214 |             if res.get('type') == 'verify':
215 |                 raise ApiFailedException("TikTok API is asking for verification")
216 | 
217 |             videos = res.get('itemList', [])
218 | 
219 |             if videos:
220 |                 amount_yielded += len(videos)
221 |                 for video in videos:
222 |                     yield self.parent.video(data=video)
223 | 
224 |             has_more = res.get("hasMore")
225 |             if not has_more:
226 |                 self.parent.logger.info(
227 |                     "TikTok isn't sending more TikToks beyond this point."
228 |                 )
229 |                 return
230 | 
231 |             cursor = res.get('cursor', cursor)
232 |             await self.parent.request_delay()
233 |         
234 | 
235 |     async def _get_videos_scraping(self, count, get_bytes):
236 |         page = self.parent._page
237 | 
238 |         url = f"https://www.tiktok.com/@{self.username}"
239 |         if url not in page.url:
240 |             await page.goto(url)
241 |             await self.check_initial_call(url)
242 |         await self.wait_for_content_or_unavailable_or_captcha('[data-e2e=user-post-item]', "This account is private")
243 | 
244 |         video_pull_method = 'scroll'
245 |         if video_pull_method == 'scroll':
246 |             async for video in self._get_videos_scroll(count, get_bytes):
247 |                 yield video
248 |         elif video_pull_method == 'individual':
249 |             async for video in self._get_videos_individual(count, get_bytes):
250 |                 yield video
251 | 
252 |     async def _get_videos_individual(self, count, get_bytes):
253 |         page = self.parent._page
254 | 
255 |         await page.locator("[data-e2e=user-post-item]").click()
256 | 
257 |         self.wait_for_content_or_captcha('browse-video')
258 | 
259 |         still_more = True
260 |         all_videos = []
261 | 
262 |         while still_more:
263 |             html_req_path = page.url
264 |             initial_html_request = self.get_requests(html_req_path)[0]
265 |             html_body = self.get_response_body(initial_html_request)
266 |             tag_contents = extract_tag_contents(html_body)
267 |             res = json.loads(tag_contents)
268 | 
269 |             all_videos += res['itemList']
270 | 
271 |             if still_more:
272 |                 await page.locator("[data-e2e=browse-video]").press('ArrowDown')
273 | 
274 |     async def _load_each_video(self, videos):
275 |         page = self.parent._page
276 | 
277 |         # get description elements with identifiable links
278 |         desc_elements_locator = page.locator("[data-e2e=user-post-item-desc]")
279 |         desc_elements_count = await desc_elements_locator.count()
280 | 
281 |         video_elements = []
282 |         for video in videos:
283 |             found = False
284 |             for i in range(desc_elements_count):
285 |                 desc_element = desc_elements_locator.nth(i)
286 |                 inner_html = await desc_element.inner_html()
287 |                 match = re.search(r'href="https:\/\/www\.tiktok\.com\/@[^\/]+\/video\/([0-9]+)"', inner_html)
288 |                 if not match:
289 |                     continue
290 |                 video_id = match.group(1)
291 |                 if video['id'] == video_id:
292 |                     # get sibling element of video element
293 |                     video_element = page.locator(f"xpath=//a[contains(@href, '{video['id']}')]/../..").first
294 |                     video_elements.append((video, video_element))
295 |                     found = True
296 |                     break
297 | 
298 |             if not found:
299 |                 pass
300 |                 # TODO: log this
301 |                 # raise Exception(f"Could not find video element for video {video['id']}")
302 | 
303 |         for video, element in video_elements:
304 |             await element.scroll_into_view_if_needed()
305 |             await element.hover()
306 |             try:
307 |                 play_path = urlparse(video['video']['playAddr']).path
308 |             except KeyError:
309 |                 print(f"Missing JSON attributes for video: {video['id']}")
310 |                 continue
311 | 
312 |             try:
313 |                 requests = self.get_requests(play_path)
314 |                 resp = await requests[0].response()
315 |             except Exception as ex:
316 |                 print(f"Failed to load video file for video: {video['id']}")
317 | 
318 |             await self.parent.request_delay()
319 | 
320 |     async def _get_initial_videos(self, count, get_bytes):
321 |         self.parent.logger.debug("Getting initial videos from page responses")
322 |         all_videos = []
323 |         finished = False
324 | 
325 |         cursor = 0
326 |         video_responses = self.get_responses('api/post/item_list')
327 |         video_responses = [res for res in video_responses if f"secUid={self.sec_uid}" in res.url]
328 |         self.parent.logger.debug(f"Found {len(video_responses)} video responses in page")
329 |         for video_response in video_responses:
330 |             try:
331 |                 if len(video_response._body) == 0:
332 |                     continue
333 |                 video_data = await video_response.json()
334 |                 if video_data.get('itemList'):
335 |                     videos = video_data['itemList']
336 |                     video_objs = [self.parent.video(data=video) for video in videos]
337 |                     all_videos += video_objs
338 |                 finished = not video_data.get('hasMore', False)
339 |                 cursor = video_data.get('cursor', 0)
340 |             except Exception as ex:
341 |                 pass
342 | 
343 |         if len(video_responses) == 0:
344 |             raise ApiFailedException("Failed to get videos from API")
345 | 
346 |         self.parent.request_cache['videos'] = video_responses[-1]
347 |         return all_videos, finished, cursor
348 | 
349 |     async def _get_videos_scroll(self, count, get_bytes):
350 | 
351 |         data_request_path = "api/post/item_list"
352 |         data_urls = []
353 |         tries = 1
354 |         amount_yielded = 0
355 |         MAX_TRIES = 10
356 | 
357 |         cursors = []
358 |         while tries <= MAX_TRIES:
359 |             await self.check_and_wait_for_captcha()
360 |             await self.parent.request_delay()
361 |             await self.slight_scroll_up()
362 |             await self.parent.request_delay()
363 |             await self.scroll_down(30000, speed=12)
364 | 
365 |             data_requests = [req for req in self.get_requests(data_request_path) if req.url not in data_urls]
366 |             data_requests = [res for res in data_requests if f"secUid={self.sec_uid}" in res.url]
367 | 
368 |             if not data_requests:
369 |                 tries += 1
370 |                 if tries > MAX_TRIES:
371 |                     raise ApiFailedException('TikTok backend broke')
372 |                 continue
373 | 
374 |             for data_request in data_requests:
375 |                 data_urls.append(data_request.url)
376 |                 try:
377 |                     data_response = await data_request.response()
378 |                     res_body = await self.get_response_body(data_response)
379 |                 except Exception as ex:
380 |                     continue
381 | 
382 |                 if not res_body:
383 |                     continue
384 | 
385 |                 self.parent.request_cache['videos'] = data_request
386 | 
387 |                 res = json.loads(res_body)
388 |                 videos = res.get("itemList", [])
389 |                 cursors.append(int(res['cursor']))
390 | 
391 |                 if get_bytes:
392 |                     await self._load_each_video(videos)
393 | 
394 |                 amount_yielded += len(videos)
395 |                 video_objs = [self.parent.video(data=video) for video in videos]
396 | 
397 |                 for video in video_objs:
398 |                     yield video
399 | 
400 |                 if count and amount_yielded >= count:
401 |                     return
402 | 
403 |                 has_more = res.get("hasMore", False)
404 |                 if not has_more:
405 |                     User.parent.logger.info(
406 |                         "TikTok isn't sending more TikToks beyond this point."
407 |                     )
408 |                     return
409 | 
410 |         return
411 | 
412 |     def liked(self, count: int = 30, cursor: int = 0, **kwargs) -> Iterator[Video]:
413 |         """
414 |         Returns a dictionary listing TikToks that a given a user has liked.
415 | 
416 |         **Note**: The user's likes must be **public** (which is not the default option)
417 | 
418 |         - Parameters:
419 |             - count (int): The amount of videos you want returned.
420 |             - cursor (int): The unix epoch to get uploaded videos since.
421 | 
422 |         Example Usage
423 |         ```py
424 |         for liked_video in api.user(username='public_likes'):
425 |             # do something
426 |         ```
427 |         """
428 |         processed = User.parent._process_kwargs(kwargs)
429 |         kwargs["custom_device_id"] = processed.device_id
430 | 
431 |         amount_yielded = 0
432 |         first = True
433 | 
434 |         if self.user_id is None and self.sec_uid is None:
435 |             self.__find_attributes()
436 | 
437 |         while amount_yielded < count:
438 |             query = {
439 |                 "count": 30,
440 |                 "id": self.user_id,
441 |                 "type": 2,
442 |                 "secUid": self.sec_uid,
443 |                 "cursor": cursor,
444 |                 "sourceType": 9,
445 |                 "appId": 1233,
446 |                 "region": processed.region,
447 |                 "priority_region": processed.region,
448 |                 "language": processed.language,
449 |             }
450 |             path = "api/favorite/item_list/?{}&{}".format(
451 |                 User.parent._add_url_params(), urlencode(query)
452 |             )
453 | 
454 |             res = self.parent.get_data(path, **kwargs)
455 | 
456 |             if "itemList" not in res.keys():
457 |                 if first:
458 |                     User.parent.logger.error("User's likes are most likely private")
459 |                 return
460 | 
461 |             videos = res.get("itemList", [])
462 |             amount_yielded += len(videos)
463 |             for video in videos:
464 |                 amount_yielded += 1
465 |                 yield self.parent.video(data=video)
466 | 
467 |             if not res.get("hasMore", False) and not first:
468 |                 User.parent.logger.info(
469 |                     "TikTok isn't sending more TikToks beyond this point."
470 |                 )
471 |                 return
472 | 
473 |             cursor = res["cursor"]
474 |             first = False
475 | 
476 |     def __extract_from_data(self):
477 |         data = self.as_dict
478 |         keys = data.keys()
479 | 
480 |         if "user_info" in keys:
481 |             self.__update_id_sec_uid_username(
482 |                 data["user_info"]["uid"],
483 |                 data["user_info"]["sec_uid"],
484 |                 data["user_info"]["unique_id"],
485 |             )
486 |         elif "uniqueId" in keys:
487 |             self.__update_id_sec_uid_username(
488 |                 data["id"], data["secUid"], data["uniqueId"]
489 |             )
490 | 
491 |         if None in (self.username, self.user_id, self.sec_uid):
492 |             User.parent.logger.error(
493 |                 f"Failed to create User with data: {data}\nwhich has keys {data.keys()}"
494 |             )
495 | 
496 |     def __update_id_sec_uid_username(self, id, sec_uid, username):
497 |         self.user_id = id
498 |         self.sec_uid = sec_uid
499 |         self.username = username
500 | 
501 |     def __find_attributes(self) -> None:
502 |         # It is more efficient to check search first, since self.user_object() makes HTML request.
503 |         found = False
504 |         for u in self.parent.search.users(self.username):
505 |             if u.username == self.username:
506 |                 found = True
507 |                 self.__update_id_sec_uid_username(u.user_id, u.sec_uid, u.username)
508 |                 break
509 | 
510 |         if not found:
511 |             user_object = self.info()
512 |             self.__update_id_sec_uid_username(
513 |                 user_object["id"], user_object["secUid"], user_object["uniqueId"]
514 |             )
515 | 
516 |     def __repr__(self):
517 |         return self.__str__()
518 | 
519 |     def __str__(self):
520 |         return f"PyTok.user(username='{self.username}', user_id='{self.user_id}', sec_uid='{self.sec_uid}')"
521 | 
522 | 


--------------------------------------------------------------------------------
/pytok/api/base.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | from datetime import datetime
  3 | import random
  4 | 
  5 | from patchright.async_api import expect, Page
  6 | 
  7 | from .. import exceptions, captcha_solver
  8 | 
  9 | TOK_DELAY = 20
 10 | CAPTCHA_DELAY = 999999
 11 | 
 12 | 
 13 | def get_login_close_element(page):
 14 |     return page.get_by_text("Continue as guest", exact=True) \
 15 |         .or_(page.get_by_text("Continue without login", exact=True))
 16 | 
 17 | 
 18 | def get_captcha_element(page):
 19 |     return page.locator('Rotate the shapes') \
 20 |         .or_(page.get_by_text('Verify to continue:', exact=True)) \
 21 |         .or_(page.get_by_text('Click on the shapes with the same size', exact=True)) \
 22 |         .or_(page.get_by_text('Drag the slider to fit the puzzle', exact=True).first)
 23 | 
 24 | 
 25 | class Base:
 26 | 
 27 |     async def check_initial_call(self, url):
 28 |         event = await self.wait_for_requests(url)
 29 |         response = await event.value.response()
 30 |         if response.status >= 300:
 31 |             raise exceptions.NotAvailableException("Content is not available")
 32 | 
 33 |     async def wait_for_content_or_captcha(self, content_tag):
 34 |         page = self.parent._page
 35 | 
 36 |         content_element = page.locator(content_tag).first
 37 |         # content_element = page.get_by_text('Videos', exact=True)
 38 |         captcha_element = get_captcha_element(page)
 39 |         
 40 |         max_tries = 10
 41 |         tries = 0
 42 |         self.parent.logger.debug("Waiting for main content to become visible")
 43 |         while tries < max_tries:
 44 |             is_content_visible = await content_element.is_visible()
 45 |             is_captcha_visible = await captcha_element.is_visible()
 46 |             if is_content_visible or is_captcha_visible:
 47 |                 break
 48 |             await asyncio.sleep(0.5)
 49 |             await self.check_and_resolve_refresh_button()
 50 |             tries += 1
 51 | 
 52 |         captcha_visible = await captcha_element.is_visible()
 53 |         if captcha_visible:
 54 |             await self.solve_captcha()
 55 |             asyncio.sleep(1)
 56 |             await expect(content_element).to_be_visible(timeout=TOK_DELAY * 1000)
 57 | 
 58 |         return content_element
 59 |     
 60 |     async def wait_for_content_or_unavailable(self, content_tag, unavailable_text, no_content_text=None):
 61 |         page: Page = self.parent._page
 62 |         content_element = page.locator(content_tag).first
 63 |         captcha_element = get_captcha_element(page)
 64 |         unavailable_element = page.get_by_text(unavailable_text, exact=True)
 65 | 
 66 |         # try:
 67 |         expected_elements = content_element.or_(captcha_element).or_(unavailable_element)
 68 | 
 69 |         def add_no_content_text(expected_es, text):
 70 |             if no_content_text:
 71 |                 if isinstance(no_content_text, list):
 72 |                     for text in no_content_text:
 73 |                         expected_es = expected_es.or_(page.get_by_text(text, exact=True))
 74 |                 elif isinstance(no_content_text, str):
 75 |                     expected_es = expected_es.or_(page.get_by_text(no_content_text, exact=True))
 76 |             return expected_es
 77 |         expected_elements = add_no_content_text(expected_elements, no_content_text)
 78 | 
 79 |         await self.check_and_resolve_refresh_button()
 80 |         await self.check_and_resolve_login_popup()
 81 | 
 82 |         self.parent.logger.debug(f"Checking for '{unavailable_text}'")
 83 |         if await unavailable_element.is_visible():
 84 |             raise exceptions.NotAvailableException(f"Content is not available with message: '{unavailable_text}'")
 85 |         
 86 |         if no_content_text:
 87 |             if isinstance(no_content_text, list):
 88 |                 for text in no_content_text:
 89 |                     no_content_element = page.get_by_text(text, exact=True)
 90 |                     if await no_content_element.is_visible():
 91 |                         raise exceptions.NoContentException(f"Content is not available with message: '{text}'")
 92 |                     else:
 93 |                         self.parent.logger.debug(f"Could not find text '{text}'")
 94 |             elif isinstance(no_content_text, str):
 95 |                 no_content_element = page.get_by_text(no_content_text, exact=True)
 96 |                 if await no_content_element.is_visible():
 97 |                     raise exceptions.NoContentException(f"Content is not available with message: '{no_content_text}'")
 98 |                 else:
 99 |                     self.parent.logger.debug(f"Could not find text '{text}'")
100 | 
101 |         max_tries = 10
102 |         tries = 0
103 |         self.parent.logger.debug("Waiting for main content to become visible")
104 |         while not (await content_element.is_visible()) and tries < max_tries:
105 |             await asyncio.sleep(0.5)
106 |             await self.check_and_resolve_refresh_button()
107 |             tries += 1
108 |         
109 |         if tries >= max_tries:
110 |             # try some other behaviour
111 |             url = page.url
112 |             await page.goto("https://www.tiktok.com")
113 |             await asyncio.sleep(5)
114 |             await page.goto(url)
115 | 
116 |         return content_element
117 | 
118 |     async def check_and_resolve_refresh_button(self):
119 |         page: Page = self.parent._page
120 |         refresh_button = page.get_by_text('Refresh')
121 |         self.parent.logger.debug("Checking for refresh button")
122 |         if await refresh_button.is_visible():
123 |             self.parent.logger.debug("Refresh button found, clicking")
124 |             await refresh_button.click()
125 |             await asyncio.sleep(1)
126 | 
127 |     async def check_and_resolve_login_popup(self):
128 |         page: Page = self.parent._page
129 |         login_popup = page.get_by_text('Log in to TikTok')
130 |         self.parent.logger.debug("Checking for login to TikTok pop up")
131 |         if await login_popup.is_visible():
132 |             self.parent.logger.debug("Login prompt found, checking for close button")
133 |             login_close = page.locator('[data-e2e="modal-close-inner-button"]')
134 |             if await login_close.is_visible():
135 |                 await login_close.click()
136 |                 await asyncio.sleep(1)
137 |             else:
138 |                 raise exceptions.NotAvailableException(f"Content is not available with message: 'Log in to TikTok'")
139 | 
140 | 
141 |     async def wait_for_content_or_unavailable_or_captcha(self, content_tag, unavailable_text, no_content_text=None):
142 |         page: Page = self.parent._page
143 |         content_element = page.locator(content_tag).first
144 |         captcha_element = get_captcha_element(page)
145 |         unavailable_element = page.get_by_text(unavailable_text, exact=True)
146 | 
147 |         # try:
148 |         expected_elements = content_element.or_(captcha_element).or_(unavailable_element)
149 | 
150 |         def add_no_content_text(expected_es, text):
151 |             if no_content_text:
152 |                 if isinstance(no_content_text, list):
153 |                     for text in no_content_text:
154 |                         expected_es = expected_es.or_(page.get_by_text(text, exact=True))
155 |                 elif isinstance(no_content_text, str):
156 |                     expected_es = expected_es.or_(page.get_by_text(no_content_text, exact=True))
157 |             return expected_es
158 |         expected_elements = add_no_content_text(expected_elements, no_content_text)
159 | 
160 |         await self.check_and_resolve_refresh_button()
161 |         await self.check_and_resolve_login_popup()
162 | 
163 |         # await expect(expected_elements).to_be_visible(
164 |         #     timeout=TOK_DELAY * 1000)
165 | 
166 |         self.parent.logger.debug("Checking for captcha")
167 |         if await captcha_element.is_visible():
168 |             self.parent.logger.debug("Captcha found")
169 |             await self.solve_captcha()
170 |             await asyncio.sleep(1)
171 |             if await captcha_element.is_visible():
172 |                 raise exceptions.CaptchaException("Captcha is still visible after solving")
173 |             expected_elements = content_element.or_(unavailable_element)
174 |             expected_elements = add_no_content_text(expected_elements, no_content_text)
175 |             await expect(expected_elements).to_be_visible(
176 |                 timeout=TOK_DELAY * 1000)  # waits TOK_DELAY seconds and launches new browser instance
177 | 
178 |         # check after resolving captcha
179 |         await self.check_and_resolve_refresh_button()
180 |         await self.check_and_resolve_login_popup()
181 | 
182 |         self.parent.logger.debug(f"Checking for '{unavailable_text}'")
183 |         if await unavailable_element.is_visible():
184 |             raise exceptions.NotAvailableException(f"Content is not available with message: '{unavailable_text}'")
185 |         
186 |         if no_content_text:
187 |             if isinstance(no_content_text, list):
188 |                 for text in no_content_text:
189 |                     no_content_element = page.get_by_text(text, exact=True)
190 |                     if await no_content_element.is_visible():
191 |                         raise exceptions.NoContentException(f"Content is not available with message: '{text}'")
192 |                     else:
193 |                         self.parent.logger.debug(f"Could not find text '{text}'")
194 |             elif isinstance(no_content_text, str):
195 |                 no_content_element = page.get_by_text(no_content_text, exact=True)
196 |                 if await no_content_element.is_visible():
197 |                     raise exceptions.NoContentException(f"Content is not available with message: '{no_content_text}'")
198 |                 else:
199 |                     self.parent.logger.debug(f"Could not find text '{text}'")
200 | 
201 |         max_tries = 10
202 |         tries = 0
203 |         self.parent.logger.debug("Waiting for main content to become visible")
204 |         content_is_visible = await content_element.is_visible()
205 |         while not content_is_visible and tries < max_tries:
206 |             await asyncio.sleep(1)
207 |             await self.check_and_resolve_refresh_button()
208 |             tries += 1
209 |             content_is_visible = await content_element.is_visible()
210 |         
211 |         if tries >= max_tries:
212 |             pass
213 |             # raise exceptions.TimeoutException(f"Content is not available for unknown reason")
214 | 
215 |         return content_element
216 | 
217 |     async def check_for_unavailable_or_captcha(self, unavailable_text):
218 |         page = self.parent._page
219 |         captcha_element = get_captcha_element(page)
220 |         unavailable_element = page.get_by_text(unavailable_text, exact=True)
221 | 
222 |         captcha_visible = await captcha_element.is_visible()
223 |         if captcha_visible:
224 |             num_tries = 0
225 |             max_tries = 3
226 |             captcha_exceptions = []
227 |             while num_tries < max_tries:
228 |                 num_tries += 1
229 |                 try:
230 |                     await self.solve_captcha()
231 |                     await asyncio.sleep(1)
232 |                     captcha_is_visible = await captcha_element.is_visible()
233 |                     if captcha_is_visible:
234 |                         captcha_exceptions.append(exceptions.CaptchaException("Captcha is still visible after solving"))
235 |                         continue
236 |                     else:
237 |                         break
238 |                 except Exception as e:
239 |                     captcha_exceptions.append(e)
240 |             else:
241 |                 print(
242 |                     f"Failed to solve captcha after {max_tries} tries with errors: {captcha_exceptions}, continuing anyway...")
243 | 
244 |         login_element = get_login_close_element(page)
245 |         login_visible = await login_element.is_visible()
246 |         if login_visible:
247 |             try:
248 |                 login_close = get_login_close_element(page)
249 |                 login_close_visible = await login_close.is_visible()
250 |                 if login_close_visible:
251 |                     await login_close.click()
252 |             except Exception as e:
253 |                 print(f"Failed to close login with error: {e}, continuing anyway...")
254 | 
255 |         if await unavailable_element.is_visible():
256 |             raise exceptions.NotAvailableException(f"Content is not available with message: '{unavailable_text}'")
257 | 
258 |     async def check_for_unavailable(self, unavailable_text):
259 |         page = self.parent._page
260 |         unavailable_element = page.get_by_text(unavailable_text, exact=True)
261 |         if await unavailable_element.is_visible():
262 |             raise exceptions.NotAvailableException(f"Content is not available with message: '{unavailable_text}'")
263 | 
264 |     async def check_for_reload_button(self):
265 |         page = self.parent._page
266 |         reload_button = page.get_by_text('Refresh', exact=True)
267 |         if await reload_button.is_visible():
268 |             await reload_button.click()
269 | 
270 |     async def wait_for_requests(self, api_path, timeout=TOK_DELAY):
271 |         page = self.parent._page
272 |         try:
273 |             async with page.expect_request(api_path, timeout=timeout * 1000) as first:
274 |                 return await first.value
275 |         except Exception as e:
276 |             raise exceptions.TimeoutException(str(e))
277 | 
278 |     def get_requests(self, api_path):
279 |         """searches a list of all requests thus far issued by the Playwright browser instance"""
280 |         return [request for request in self.parent._requests if api_path in request.url]
281 | 
282 |     def get_responses(self, api_path):
283 |         return [response for response in self.parent._responses if api_path in response.url]
284 | 
285 |     async def get_response_body(self, response):
286 |         return await response.body()
287 | 
288 |     async def scroll_to_bottom(self, speed=4):
289 |         page = self.parent._page
290 |         current_scroll_position = await page.evaluate(
291 |             "() => document.documentElement.scrollTop || document.body.scrollTop;")
292 |         new_height = current_scroll_position + 1
293 |         while current_scroll_position <= new_height:
294 |             current_scroll_position += speed + random.randint(-speed, speed)
295 |             await page.evaluate(f"() => window.scrollTo(0, {current_scroll_position});")
296 |             new_height = await page.evaluate("() => document.body.scrollHeight;")
297 | 
298 |     async def scroll_to(self, position, speed=5):
299 |         page = self.parent._page
300 |         current_scroll_position = await page.evaluate(
301 |             "() => document.documentElement.scrollTop || document.body.scrollTop;")
302 |         new_height = current_scroll_position + 1
303 |         while current_scroll_position <= new_height:
304 |             current_scroll_position += speed + random.randint(-speed, speed)
305 |             await page.evaluate(f"() => window.scrollTo(0, {current_scroll_position});")
306 |             new_height = await page.evaluate("() => document.body.scrollHeight;")
307 |             if current_scroll_position > position:
308 |                 break
309 | 
310 |     async def slight_scroll_up(self, speed=4):
311 |         page = self.parent._page
312 |         desired_scroll = -500
313 |         current_scroll = 0
314 |         while current_scroll > desired_scroll:
315 |             current_scroll -= speed + random.randint(-speed, speed)
316 |             await page.evaluate(f"() => window.scrollBy(0, {-speed});")
317 | 
318 |     async def scroll_down(self, amount, speed=4):
319 |         page = self.parent._page
320 |         
321 |         current_scroll_position = await page.evaluate(
322 |             "() => document.documentElement.scrollTop || document.body.scrollTop;")
323 |         desired_position = current_scroll_position + amount
324 |         while current_scroll_position < desired_position:
325 |             scroll_amount = speed + random.randint(-speed, speed) * 0.5
326 |             await page.evaluate(f"() => window.scrollBy(0, {scroll_amount});")
327 |             new_scroll_position = await page.evaluate(
328 |             "() => document.documentElement.scrollTop || document.body.scrollTop;")
329 |             if new_scroll_position > current_scroll_position:
330 |                 current_scroll_position = new_scroll_position
331 |             else:
332 |                 # we hit the bottom
333 |                 break
334 | 
335 |     async def wait_until_not_skeleton_or_captcha(self, skeleton_tag):
336 |         page = self.parent._page
337 |         content = page.locator(f'[data-e2e={skeleton_tag}]')
338 |         try:
339 |             await expect(content).not_to_be_visible()
340 |         except TimeoutError as e:
341 |             captcha_element = get_captcha_element(page)
342 |             if await captcha_element.is_visible():
343 |                 await self.solve_captcha()
344 |                 asyncio.sleep(1)
345 |             else:
346 |                 raise exceptions.TimeoutException(str(e))
347 | 
348 |     async def check_and_wait_for_captcha(self):
349 |         page = self.parent._page
350 |         captcha_element = get_captcha_element(page)
351 |         captcha_visible = await captcha_element.is_visible()
352 |         if captcha_visible:
353 |             await self.solve_captcha()
354 |             await asyncio.sleep(1)
355 | 
356 |     async def check_and_close_signin(self):
357 |         page = self.parent._page
358 |         signin_element = get_login_close_element(page)
359 |         signin_visible = await signin_element.is_visible()
360 |         if signin_visible:
361 |             await signin_element.click()
362 | 
363 |     async def solve_captcha(self):
364 |         if self.parent._manual_captcha_solves:
365 |             input("Press Enter to continue after solving CAPTCHA:")
366 |             await asyncio.sleep(1)
367 |             if self.parent._log_captcha_solves:
368 |                 request = self.get_requests('/captcha/verify')[0]
369 |                 body = request.post_data
370 |                 with open(f"manual_captcha_{datetime.now().isoformat()}.json", "w") as f:
371 |                     f.write(body)
372 |             return
373 |         """
374 |         this method not only calculates the CAPTCHA solution but also POSTs it to TikTok's server.
375 |         """
376 |         # get captcha data
377 |         request = self.get_requests('/captcha/get')[0]
378 |         captcha_response = await request.response()
379 |         if captcha_response is not None:
380 |             captcha_json = await captcha_response.json()
381 |         else:
382 |             raise exceptions.EmptyResponseException
383 | 
384 |         if 'mode' in captcha_json['data']:
385 |             captcha_data = captcha_json['data']
386 |         elif 'challenges' in captcha_json['data']:
387 |             captcha_data = captcha_json['data']['challenges'][0]
388 |         captcha_type = captcha_data['mode']
389 |         if captcha_type not in ['slide', 'whirl']:
390 |             raise exceptions.CaptchaException(f"Unsupported captcha type: {captcha_type}")
391 | 
392 |         """
393 |         captcha_data['question']['url1'] is a URL from TikTok's content delivery network. If you copy-paste it into your
394 |         web browser, you should GET the puzzle image. puzzle_response is the full response from the server, and
395 |         puzzle is the image itself, returned as a sequence of bytes.
396 |         """
397 |         puzzle_req = self.get_requests(captcha_data['question']['url1'])[0]
398 |         puzzle_response = await puzzle_req.response()
399 |         puzzle = await puzzle_response.body()
400 | 
401 |         if not puzzle:
402 |             raise exceptions.CaptchaException("Puzzle was not found in response")
403 | 
404 |         """
405 |         captcha_data['question']['url2'] is a URL from TikTok's content delivery network. If you copy-paste it into your
406 |         web browser, you should GET the puzzle piece that has to be moved to the correct position in the puzzle. 
407 |         piece_response: the full Playwright/HTTP response object
408 |         piece: the image of the puzzle piece, returned as a sequence of bytes
409 |         """
410 |         piece_req = self.get_requests(captcha_data['question']['url2'])[0]
411 |         piece_response = await piece_req.response()
412 |         piece = await piece_response.body()
413 | 
414 |         if not piece:
415 |             raise exceptions.CaptchaException("Piece was not found in response")
416 | 
417 |         """
418 |         -at this point in the code you have the puzzle image (puzzle) and the piece image (piece)
419 |         -now a local CAPTCHA solver will decide how to place the piece in the puzzle
420 |         -finally, the solution will be POSTed to TikTok, and the server's response will be obtained
421 |         """
422 |         page = self.parent._page
423 |         solver = captcha_solver.CaptchaSolver(captcha_response, puzzle, piece, page=page)
424 |         solve = await solver.solve_and_drag()
425 | 
426 |         if self.parent._log_captcha_solves:
427 |             await asyncio.sleep(1)
428 |             request = self.get_requests('/captcha/verify')[0]
429 |             body = request.post_data
430 |             with open(f"automated_captcha_{datetime.now().isoformat()}.json", "w") as f:
431 |                 f.write(body)
432 | 
433 | 


--------------------------------------------------------------------------------
/tests/captcha_examples.json:
--------------------------------------------------------------------------------
1 | {
2 |     "whirl": [
3 |         {
4 |             "puzzle": "b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAFbAVsDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+iiigAooooAKKKKACiiigAooooAKKKKACigkAZJwKrvdxr935j7UAWKRmVRliAPc1nvdyvwPlHt1qLOTk8mnYVy+13EvQlvoKhe+P8CDr1NU5GKRswRnIGdq4yaUinYNSwbuVjwQPoKY08rHJdvwOKjFBNAhTPL/AM9G/M0CaQnl2/Om8Gjco70AS+bJ/wA9G/OmtcsnWVv++qheVcHkVl3T5Y4fApNjNM6ywiV2JjJGdrAZHsccUz/hI8EN5aOpz93g1zVwQQfmJNJBIiRgCs3LU05ep06eK7HKrMksbE4JABA9/X9K0rTVbC+O23uo3bOAucMe/Q8159NGZXLBTj6U1ImQ5FTzu5XIrHp9FcDZ6nqNqV+zs2wfwNyvXPTt+Fbtl4kdiq3ttsz1kiOR1/untj3NWpohwZ0NFQ291BdpvgkVwOuOo+o6ipqokKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiio5Z0iHPLdhQBISAMk4AqtLeIuQnzH17VTkuXmkdCGVRg9ODn+fSmBec96dmJskkleXljx6dqjFOpKoQHrS9qSigBcHrSGlpD0oAAaDyKaAc0+kBCVk3t02YGMZzn/DpWdPFdtL8jYWtekOACcc0NIZgz292i53kmqC295M3L4Fbd000jbEXj1psNnMByeD1rNxuykyjHpfH72Qn2FWI7G3j5C5+tX1syepqdLNVp8iFzGPKnGFTP4VAID1ZK6B4VUdqyZ2KyvlgVJ+UAYx9eeaTRUWLEqhckVE78424qRWBXJNV5mUE5NIa3GrM8UgeNyrDuDg1s2viEqAtyu8f3l4P5dP5Vz7SLmkDA9Km9irX3O8t7qC6TdBKrgdcdR9R2qauDhuZreQSQyMjDuDW/p3iGOXEV5iN8ACTsx9/T+XXpVqSZDibtFFFUSFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRQSAMk4AqhcXRcFU4T17mgCSe8CjEWCe7VTyScnrTKcKokU0CjtRTAWkpaSgQUo600Agk7iQeg9KWgNxaSloxQMKKKKAG85p3am4IYYAwevrTqSARQuelPptLmgY4GlJ4po5pwpiKs7YBzmsW5OX4rflj3Csm7gC9azkiosqwq5ODgipntcrk0kHy9elTvKChwOPWpRRQe1ANC269BzUU11hiAeahhuJVkJxkVF1cuzsWJAqHB4qpPcBPr7UXRlnkGOBSraqwBc5qW29i0lbUt6T4juLF1iZPNti2Sv8S+u3+eP5Zrtra6hvIRLbyB0zjOMYP0NcLHHEmcAVbtr6Wwm327ZBHzKw4NXC63Imk9jtaKq2F/FqEHmR8MOHQnlT/nvVqtTIKKKKACiiigAooooAKKKKACiiigAoJAGScAUVnXdx5jbEb5B19zQAl1dNIdqD5B+tQDJHNFKvTmqJEpRSGgdaYDhQKKWgBskixIXc4UYGcfhRnJpSOD601aAH0YpKWgAopqb8tvCgZ+XBzke/vTZJlj+8aAJO1FVWvoweoqNtSjXvSugLrAlSA20+opeKzDqauCV7HFQSamccUnIEr6o2sj1pGkQdxWGNRkK9Dn6U0vcyjhWpcw7G39ojUckVG978y7Cmzndzz7YrIW1u5BgjAo+yXUfGMijmY0i8+pqGwapT3qtIO4qBLSbH74BmyeQMDGeKSWAIvFRJspJXJHuQ/C1CZJEHXNQpkPxVxod8fHBFTqy7WM11BcsxyamV1VcBeafHa5c7jn2q6IYlXgAVKiynJGS0shb5VP5VYhjmfqMVOUABYDI9hUm4iIMoqkhNortaybgQwFRSgwrktk+lXYN05IyBiorZN+pNHKQQOgocewubuQ2VzqK3n2m2VvMByS/RvUH1ruLG9S+g3hdki8SRk8qf6j3rPhiRRgAVC8UlveLcwvtYDBHYj0NXGPKiJTUuh0FFMilSaMOhyP5U+qJCiiigAooooAKKKKACiioriYQRluNx+6D3oAgvbjaDEvUjk56VQFISSck5J70oqkIdThTeopwpiEIpFGKcelN70APooooAQ8imr1p9Mxg+1ADhS0gpQCTgdaACs3UYZpM7AOvGK2o7R2GWO3+dWUtol/h3H/a5pMLHHR6dcyTBAHLbd3CkjH1q8nh6d8FyQD79K6gAAYAwKKiyLuYC+HMdXU/ianj0ONOyVsUUxGYukqpzhKk+wlcYUH6Gr9FArGZKPIBLRkAdTjj86pPfQBsFxmugqtcafZ3W4zW8bM2MtjDce45obY7I56e7izgMKzLi6TzQCeK1L3wjIw3Wd+4IAwswzk5/vDpx7VgT6BqET7ZwQfXqD+NZScuxtGMO5feW1EQG5Rjn3rPm1Da21WyKP7GlbCs5qeHw+oOWYmpfO9kWvZrdkSagqIcgkkcYPeltb1MP5mc+9XTo0CAE1BNbQR9AM0/eW5DcXsRHVsRMvlkcnFVTeTshVelSGLceBxViCAKORSu2xpJIz0mu42LKTk0tlPPDf/aJskGtZUUKAeT6kdaR4lbGGK4OTjHPtVJW6k6Pc04datiAN4B96sSXcN1EVWQc9wawWtYm7DNQtYFWzG5X6Gq5mTyI3tNY6fdGXzmkRhtZSf1+v/166cEMoZSCDyCO9eepZXB5EzVveHr2W3kNjcuCrHMbMeQf7v49fr9aE/IGvM6WiiirICiiigAooooACQASTgDvWRcTmeYkH5RwtW7+fZGIgfmfr9P8/wBazSdvOCeccCmhN2HUooFApiHZAxkjnpThUbFRjdjrx9aetMQ6mkU7IGOetIaVwFopBS0xhSHpSgEnA5NXYLUL80gBPYUgRWhtZJcE/Kvc9z9KvRQpCPlHPcnqakopXKCiiikAUUUUAFFFFABRRRQAUUUUAFIyq6lWAIPYjNLRQBmXel7hutuG/uE9fpWFLcGByjqyuOoYYNdhVS+06C/iKyKA/wDDIByP/re1JgclLdPKMIKiW3Z2y5NaU9ibFgsoHI4YdDUOeeKhruWn2KzQCPoKaM9qtM2AcgMeccdBj+fWoQdvXAoGncYEY8npS7T2prXKDIHJqF7ps/dpXRSTJgCOpqaMBuM81Q3tKRgkc5+tSIZInyBkUJiZdaRouNlUHuZDcAplSDkEcYq4tyGGGGD71E7LvLBRih7aMSOv0u9F9Zo5YGVRiQd8+v49au1yOlaklpeqHIWKT5XJPA9D+H8s111aJ3IasFFFFMQUhIAJJwB1Jpaq38m232AnL8fh3oAzp5PPnaQ9Og9hURdY2Bd1VWIVc9Sx/wAin7SFz2oBqifUcDg0opm47qcKQC08VE6syEI+xj0bGcVKKoBaKKKBCZGaUDJwBk0wKwYfOSOeoHc/0rQtYdoEjdSOB6UtgWo63txGNzcv/Kp6KKksKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCK4t47qExSrlT+YPqK5u6s5rOQoVDA/dfsRXU1FcwLcwNG3Gehx0NJq407HHMkpzniqroejHNaFyz29w8Eijep5wc1nXEpJIzWTsaq4+KKPIyKshYO4FT6b5Bt/mxn3ptw0HmEIAfpVJaEttsrO0UfQKKa13EE6j2xSzNGF5UVTkmiUjAH4Ck3YpRuPa7DHhf0qcRGSEsD16DFVxcxMAAOfpVoXbJFjA/KhPuDRHHpvmRknOa67R7priwVZGzNF8jknk+h69x39c1ySapsBBz+VX9F1dF1FY3OFm+Tv17f4fjTTitiZKT3OuooorQzCsq9l33BA6Jx+PetR2CIzHooyawSSzZJyT1zTQmO3BuKDjsKZkKxGKdnNAhRThSCnCgBQKcKFGelL2xTAKKKdjJwBzTESW0PmPuI+Ufr7VoUyKMRxhe/f60+pZaCiiikAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBj6/YNc2v2mFGaeEcKoyXXuPw6j8fWuMs9TtLlsblOehz1r0uvm/4g2N74H8bym1Lrpt7m5tgBhVz95BgADa3YZwpTPWuLExqRftKfzR6GBpwrt0pb9D2C2igeMsjZI6jNDmJcjjNeceE/FhnuFWWTCTL0z0Nd5HcRSOCGBLcdaijjIzVpaMWIwk6MrMWUhl+4cepqoIVL5x+FajpmMj1qBIAuSe1dTRzJsriMLj5asqquuMc08hAM5piXCszKFYFTg5GM8A8evWqSJ1ZGtsm45FPFuEkV0yrA5BHBBpXfB4NPicN1OTRoGp19tOtzbRzLwGHT0PcVLWXoTt9leJ33Mjk5AwAD0HX2rUrQh7la/bFqRj7xA/r/Ssdc7zmtLUX+ZEGeBk+lZ46mmSI+C4HOcZ6cfnSjijNNZgB70CsyUUb1HUiqctwNvBINUJrg881lOqomkabZt/aI1HLCoTfR5IDD8a5ua6k3bVJJoTzmyXO0VzPFvojdYfq2dD/aQU9j9Kv6XJ9qkZ+Nqds9+1cn5iR9SWNdlosPk6ZExPMo8w46DI4/TFaUqspysyKlNRVzQooorpMQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK4X4seGE8R+CbmVAgvdOBuoHbA4UfOmcE4K54GMsq5OBXdUUpK6sXTqOnNTXQ+TNH0bUgUkiLAjnoRj867vRV1S2lWSUtJjgg+la+qzadoGvy6RM3zqR5eT95SMjsOcEZ96sxR4HmQnKZ4r5qrCcp+9uj6OeJ547aM3Le582AMQc9CKkLEjgVShmUY7H0q1HE2z75fknJxn6cV7GHq8y5XueLVhyu6A5NIE75oPBpY23PtrpMAK+tBBA+UY96nlgAj3DrVNZMHDGm9BLU1tBmKamFIz5ilevTv/Suqrh7W5FvdxTDIVWBO3rjv+ldxVRJkZd8xa5IP8IAqpICF3Dt1qxdMWuHJ9cflURG5SD3qyDJGpIUBcqHx8wU5GfrUEl3nJDVR1BIrS6KR5Iz09KqtdAKTnpXDUrSTaZ1wpJ6l6S969eKrSXsYZQzYLHAyeprMur9FAK5HHJJ61TW+38hwCelckqje51RpI3hIq/do831NZqXYPVqcblWOQfwpJj5TStwbm5jgCjdIwRSTgZPAr0pVVEVEUKqjAUDAArzvwwftHiC2BjLqu5zkZC4U4P54/HFei13YRe65HHiX7yQUUUV1nMFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB4H8edNEHiLTNTKqI7i1eJTHw3mofvNx6OoH09hWV4M8S389nb203zlW2s7cEivUvi5p0l74Xtpre2Ek9vdBhIEyUUqwPPYE7fxxXlGmxyWrRiWM+cfnPavIzGqorlS1PZwFHmjzt6bWPS4G3xKzjD9CPSrsE2H21zunXks+WcMM9z3rTjcBySa5qNV6NCqUt0zSJO7PSmKm055oYnapx1FPjbIwRXsxd9TzZKwkk7sNuaZFbbz3q2bddpYGokmCMB0q7dyb9h/2TC4rsoHaW3ikbG5kDHHTJFcm0wK5rpdMlM2nQuQOmOPY4/pVIhlG4/4+Hx/eNMHSnzczScY+Y8fjUeatIg5rXwiXQKjlhk1z00nJIGeCDmuk8RQyu8fkIDI3HJqvYeHC+HuTj1FcValKc9DtpVIxhqcrLbyzIAvJxioE0u7UgpEzkHsK9Pi0y1hQKIl474qZbeJD8qKPwpLB92V9ctsjgLfSprgkylYdvVe9b9l4etGQfvjn1NVdaSSC/LJ909cU23mlSMspMgPoelQoxhJpq5TlKcbp2Ot0DSIrC9eaMkkxlcn6g/0roq5PwtfSXN/LE7/AHYidpzk8jn/AD611ldlJxcfdOOopKXvBRRRWhAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAGT4mUN4eugf9j/0MV5lN5QbiNSw74r0/wARgHQLvLBeFOTn+8OOK8a1XW4LDK8Gc/cTH3ucV5GYfxF6HqYCLlFpdzdgHTkfSrKx5cFmrndHubu8bzXOAT90V1BhbapripS5tjpqx5XZmhZgOm0846Grfkrjjr61SslKBiau+cF6mvaoO8Fc8qsvfdhXLbduB9aqmPnPFWDKr96gHzyYHSt2ZIeDhevQV1Gi/wDIJg/4F/6Ea5d0A4rpdKkePTYVEEjAZ5BXB5PqaaEyvO/+mOm1uSx3Y4GD0/Woj1qef/Xyf7x/nUJHpVkGfdgPcIf7tXYx8oqhOf3+Per6MNo5pDJe1NNAkU9GFRtKv978qLiMzVrZnKuoHXkEcGqsWmFSJUBX1APFac0iFeWz9ahh1GG2Ty8gAdOawnCLd2bRlK1kXfD8Hl6i7lMN5RG7HuK6Wua0fVVl1JIeCZAyjkduc8fSulrSFraESbbuwoooqyQooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAMXxaSvhi9I64T/0Na+ZPEOpfZ/EQaSE7YgAxVj83vzX0v4vZv8AhH5IVXJmdUz/AHcHdn/x39a8O1/wc+rapBJC2EOBN7gV52I5Pb+/tY9LCynGi+Te5t+E3MtopMTgNhgT0wfeutDbpMDkdqoW1qlpaRQRhVjUbcA88VqRIFjDEYPauGjTV+WJtXnd8zJ1YKmKjLFjTghK5xTBXsRjypJHmt3dxjylfrU9q4Az1zVVwWkq1EqhevFNXuD2HsJJWJHQdK6vR8/2VDnr83/oRrlPPZHVVQFcHJzyPTj866zSG36XC3ru/wDQjWiIkiC8/wCPp/w/lVSaVYYy7EAD1q/fg+ardiuKxNZB/s5zjOOaqTsrkJXdjEl1MvKzlCpDdD3qR9W3YB4NZLSmRAR1qBmYNXmuvO+53qhHsaU2pskgKNkd6rvrN0GyJM9gD6VSlV5FBUkDv70oiYAEiodWT6mipwXQuSX91Kq/vAp+lUZ4JpW3CXBqxGmRjGPrUiw4fed3IxjPH5VLk3uUko7EWjebp2s2lz5m1UlG9sZwh4bj6E165XlaxHO7bXpWmXP2vTLactuZoxuOMfMOD+ua6sG7XicuK1sy1RRRXacgUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAHlXxi8ayeHzp+mWbRmeVWnlV1PyrnahB98P+X0rzK08c6hcTRw28Q+c/MxHWrHxYNxr3xGuxDIZ0g2WsI2bfLCj5h0BPzmTk568cYrR8M+DpRJFui5UA5xXjYuUJS01Z9JhoU6NBc6W1zr9GtbyVEnuP4ulb8kL+YMA9KuxWflwqmBgDircax4+bGa6sLhPZLU8eviPaMygNq9DmoxE55C1fuHjVjjGKWKWIxmuyxzXIrS0VuXHNQ3aiKTC9O4FJLcujnZxVYztnc/f3pNoaTEYsTwK7qw/wCQda/9ck/kK4iNjPMkcYy7kKo967+nDUUirfLmJWxyD+lYeqDOnT84wtdFMnmQuo6kcfWsC/DNYz7FDNsOATgH8avdWIW6OOjhchdynnnNWRaqTkgZq/BD5yDrhRU62wzyOPSvP9g3qjsdYy/sZb+GpF0/P3sVvLajA6fSmGxZs8jHpWn1axHtzF+zxqcd6esa/d4/KtNrIKfWpI7BSc8VHsJXH7VGSYCwA4/Cui8NMYo5rY4HPmLx17H+lNSxjB+7mrVvAlu6uo+YVtToOMuYznVUlY1qKQEEAjoaWukxCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqrqNy1np1xcJt8xEOzcCQW6KDjtnFWq5/X5/NkS2XkR/M317fp/OplezsVG19TiNK8L21tIs0iKzju1dRE1vaL8uCT1IFUucnrTGfBrmpUY0laKOirVnVd5M0Zb0FflGKpGZ2P3iPpTQNw60hTjitm2zK1hzjIySTTEfb0pcE/WhYskDqTQAgbc/IqVreNk3d6idPSl3vt2k0X7gWdEhWTWYV2FlUljjtgcH88V2tc74ZtcPNdHPTy154Pc/0roquCsiJPUKx7qHa0kRGFPT6VsVSv48hZAOnBqiWYsNusKlVHA4zUyxrjpUjDjrTQaLIVxw4GKMkGjrQRkUxDXIwaZDOhyCwBHvWbqOoNBKYQO3UVgPeE3ShC5U/eNZTqcprCm5HbC7hhBLyjrnk1n3niOCL5IfnfpXLT3m8FVz16moYomc7q56mJltE3hhklqdz4c11r2WS0uABIMtGQRgjuPr3/AD9K6OvMLdZIJkkhB8xGDKQM4Ir0awujd2ccjrtk2jeuMYPfHtWmHqOSsyK0FF3RZoooroMAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiikZlRSzEKoGSScACgCG8uks7ZpnGccBc4yfSuUaYMzO/LsSSfc03W9W+13OIzmFOE4xn1NZRuHbpxWcpmkYNmhI2RxiqTtlqsRpIybiKgeCTdxUNFqxOsg2DC4IGCc9aA9MjhZRzUmz0oVwY1sgg8c88GlGTSFR1pRIsfWn6iHKSp3AkMOQR61GwYckY5xzUck5YnbUumWb3uowxEbl3BpOoG0denT0/GlfULHY6Ram006NGXa7fOw56n/62KvUUVujEKZLGJY2Q9xT6KAMJ1KsVIwQcGmVf1CLDCUdG4P1qjTEKDTqaBThQIydUhzOkgRScYDE96xJ4vIJMq4LeldTewGaAgcMORXPvb3F1P5ZVsDuawrJ9Dek7bleG1RogPLyxrWsdHJw0mFWtKwsUt4hkZb1q6FUkZAOORRCgt5BOtLaJDFZQRYKxjI7mrcchhfKj2Ipnc0ldCSWxg22aiOsihlPFOrMimaKTIHB6+9aEcqyjKn6j0qSh9FFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRQSAMngUAFc9rt686G1h/1Wfnb+9jsPb/AD9dO6ukK7Vche5Hf2rn9QuU+6v6UpbAtyh9mG0bqjESqe2BT3kd+BR5LEeprJmyZZSeMLgcn0qKSeNevX0pscRUHJwarSwlpCc8UOTsCSJDcqx4pDIzNhRzTEhUHJOavWoTccLk0rXBuxTkjnAyRgVF5DHkknNadxcRk7V59arCQMcCnyoFJkaRFBwK6Xw5Y+VE924+aT5E/wB0Hn9f5e9ZFrH9puY7dfvOcZ9B3P5V2SIscaogwqgAD2qoxREmOooorQgKKKKAGSxiWJkPQisWRDHIyN1BxW7VO+t9y+ao5H3uOooEzLUv5jKVGwAENnknnIx+X51IKaOtLVAO7UKgHOBzQOlHvSEOHBp4puKdTAKMd6QnFANMBacjtG25Tg02igC/FcrJhW+Vu3vU9ZNSx3EkfGdy+hqbDuaNFQx3Mbjk7T6GpqQwooooAKKKKACiiigAooooAKKhnu7e2H76VVOM46n8utY11rsrAi2hKL/ffGfy6fzoA2Lq8gs4w0z4z91RyW+lYF3q89wMKPLj/u5z+ZrIlknmlLne7HqScmpUtrqfA+6Km47F2CF5+XkJ9qnlsoxHnAzUlrayRIATRcbkXnpVW0JMxYCM8AU8Ic81BLe7cgLUS3znkLzWd0XystPCznAPFQvalRyab9sl3dqR5JJerflS0KSZHsVTycDB96rs7/NgEAdDnrVsx8E9aFiDccUmiivEPkOelLCjb8Y4Jq4LZRV3TLBbm8HXYnLeh9v8+9NRE5GjoWn+QjXTj55Fwo54XP8AXj/JrZpAAoAAAA4AFLWqVjNu4UUUUCCiiigAooooAyL63eBgYVDKxH3mxgZ57Ht09f1qGtuSNZYyjdDWRNC0Emw89wfUVSYmNoFFAoQh4NOqFiVUsATgdB1NSKT3piFPNHSikJoGKKWkHSloAB156UUUUAFKJJYx+7fafcZH5UxmYOgCFgTycj5eKdmgGiyt44+8oPH0qUXkZxkMPWqNFKwXNH7TD/f/AENH2mH+/wDoazqKLDuaH2qH+/8AoaRruIdCW+gqhRRYLlmW+bBEUYz2LH+lVXmupc7piqnsgxj8etLTNn70vubkYxnj8vWiyAYlpEo6VIbeMj7opskuwcVB58rdFNAiYQRJngU1p44vSqjtcM3PApfID4BJNQUTPqSgYWqNzPJMpGcCr0dkmPu02e3whwBTaGmjn/s+Xwzk1ZW2QDvUF0zRS4BApFnfHL1ldI01HSJt5FLG7Y6U0uWyOSKnhwVx0NC1C5Xe5IbABpFncdgKsG13MWANRPbMTwdtFmFyWFp7mVYowWdjgAd67Kxs0sbYRISxzlmPc1U0fSRYIZZDmdxg4PCj0rUq4ruRJ3CiiiqJCiiigAooooAKKKKACo54FnTa3XsfSpKKAMaSNonKOMH+dNFa80KzJtbr2PpWXJG0TlWHI/WmhDSccUZpuOacKoQ4U0uPNCYbJBbO04/Pp36U+jtQACigUd6ADDZJJGOwxS9aKKBLQKQ57UtFJjCikpaYgooooAQ0Cg0CgYtFFJQAFQetKAqjoKbSspYYFICKV0FRGZRg5AFMktGZslzUbWigckmpZSLa3CEffFQzTKynBJ+lJDAOgAxVpbbA6CmI5u6iMkmcGljt1HUVtT26Lz3rOlgEjHDVnyl82gmyML2qJiUOVFPG2Hg80ySYtwBigEKsxOATXR6Xpnlhbi4X5+qof4fc+/8AL+UOjaOYtt1dL+86oh/h9z7/AMvr03KaQNhRRRVEhRRRQAUUUUAFFFFABRRRQAUUUUAFNkjWVCrDj+VOooAypbd4GGeVPQimdK2CARgjIqjPZEZaLkd1PX8KpMVitQKQjBwRg+lGaYhelJ3ozSBhv255xnFAXHUvem859qdSAKSl7U2mAtKKaSccUopALSA5ozSKAvAAAznimIU0UpoFAwpKWigBMU4VFJKqdTUAvY1B3ODzSuBZaNm6UhgG3k1WOpxjo2ahl1EMO/ByMHFTdD1NOHYnpUd1exxKfmArEl1CQAkNimQx/aV3yEt9aV+iHYS5u5LglYifQVHBLLGCr8t71JJthZcYAB54q5Y2EmoHeq7Y/wDnow4PPb171NtSiksUlzcpEgJdugFdRp2lR2X7xgGm6ZHQfSrNpZw2UZSJepyWPU1YqkhNhRRRTEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQBFLbpKDkYb+8KpS20keTjcvqK0qKdxWMal7VpS20cmTja3qKqSWkicgbh7U7hYhFLSfdJBGDSZpiHU00tJQAUopKBQAUUUUAIJFZ2QZ3KATx6/8A6qUnAzS9hSHkEUAZs+qpFN5feoZNSfblRUsmkrLcGRqsLp8Q5IyamzHoY4uLi4zuUgUhsppfu5FdAttEg4UVIqKOgpcvcdzn49OljHOTTnspmHXFb7ICKWO2MmNqnHqelHKguc/aacXlIkJNb0GnqEwq1ah05EffIcn+6OlXQABgDAoSsDM1dEtXlSSdd5Q7lUEgZ9/X6HitJVVFCqAqgYAAwAKWigLhRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFADXjSQYdQartYoR8rsOc881aooCxnmzlUcAN9DULI6feRhzjJFa1FO4rGPRWsyIxyyKT7imNbQsclB+HFO4WMw0VpfZYf7n6mmtbQhkATq2DyfQ0XCxn9qB1rS+yw/3P1NH2SD+5+pouFjOpQCxwASfQVp+VH/AM80/wC+RT6VwsZi20zYITAPqcVYSzP8bfgtW6KLhYiSCNMELkjualoopDCiiigAooooAKKKKACiiigAooooAKKKKACiiigD/9k='",
5 |             "piece": "b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCADTANMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+iiigAooooAKKKKACiqt5qFtYrmeUBsZCDlj+H4delZkmtvLnykKJ2zyT/hRcLG4zKoyxAqpLfBB+7iZznucCsYauobDE596tJexyDORRoGo2fUNQd9kYSIZJBVc8ehJ471Xea/kYF7hx2+U7f5VeWRH6YpSq+lOwrlQR3LLzdT/AISH/Gq7QXqf6u8nUZJxvPU1p9KKOVMLmbbHUUlPm3MjL7sa11uphglgfYiosD0op2Atpe9N6fUip4545BwcH0NZuDjPaiiwXNais2OeSLhTx6HpVuO6jfhvlPv0pWC5PRRRSGFFFFABRRRQAUUUUAFFFFABRRSMyopZiFUDJJOABQAtY19rQUmK0wzZIMhGR+Hr/nrVDVNZ+0MYoJCkI4JBwX/+t7ViSX8MLgbuaxnWjHdmkaUpbI0BbtPNukdnc9WY5Na0NoAlY1reB/mUj61oR6rHny1O9/RauEotXRMoyTsyaSxRuqA1EunKDxkCpRfL3yDUyXSnoQavQgIoPLGKmNMWXcelPPSqQhtFFFMYUUUUCCiiigAooooAmhuWjwD8y/yq8jrIgZTxWXTkkaNsqce3rSaHc1KKZFKsq5HXuPSn1IwooooAKKKKACiiigAryLxl8SovtUlpayKtpG5XehDGYg9Rjjb6fn9GfGjxtc2cK+GNIZxcyqJLyVHXAjOcREdQW4Y9PlwOQxx4TDLqLzujRlzNhH3IDxuB4P8ADyByMenQmuLEt1PcjKy6nq4PDxilUqxbvtp+J6O/jZy6mWRlUjIUcmlPjC1xl5tufXJriXvI0vpLoWRit4gECE7gWAx3rZ8H2NtqV99ovNOaWMtwWB2/l3Fea8NbVnpc1Fxukav/AAlesahdJa6NbyTbzjcqnA/GvWPDGnvptjm8m827k5c9h7Co9Pght0VIIY4lHQIuBWqgDMAFz616eHoxhr1PGxWIdRcsVZFqSMOMgA1Gg2NylSrgYUZz/KiRSiM+GbaCcKMk/SuxanCSrIgFPFwmOtILdSvvUT2wJ4piLAkVuhp1V4oChzmrOKYCUUUUxBnnFFFFJAFFFFMAooooAcjtGwZTzWjFIJUDD8R6VmVJDKYpM87T1FJoaNKigEEZHIoqRhRRRQAVU1O+TTrCS4bG4DCA92PT/H6A1brhPFmoveaqllAxMNufn2n7znr3xwOPUHdWOIq+zg2tzWjT5526HM3vhzR9TuZ7i5tUaad2kkcDBZmOSePc1mr4FsopEFq8iHcWAzntxXVi1jWTgvsI4J5OanwkWCyEEDjPFeV7P+Y9NYiaVos801Pw9Nc6jb2C2iCxjGZZifvN3Na+oakdMtoLQLCptU8uIx9NvrnvXW+RArYjQnJ5GR0PU81yfi3w20kBaxJzj52/p9Kd3GGhpGcak4xnsZNr4/ltJPKnVWA6Nu5rrdH8bWtyA8kyoTwAa8efSFNw0ccjyOOpxjFNAW1nSF7rL/3Y2AAPux4ohUkvgZ21cHQmvM+jbLXLO4Y7ZUOPetaO7hdA4ZQmcbietfO2k+IYbSYo+7eDgZcED/Gu5s9R+0WEt59oSOCOPezs2AD6VtDHVL8rgeZXy1Q1T0PVFmSQna4KjuKfxjgV5Rpni1iDsk3gj5SG79q6nT9eupABcOFz3HNdNLHU56PRnJVwNSm2dfnHWjNUILkbcvKGz6GrUdxE/CuCfSuxSTONpokooBBzil6CqAYEIkZ9zHIA2noMZ5/X9BTqXNFACUUdaXFAhKKM8470Ur3GFFGKCKYi3aS/8sz9RVustSUYMvUVpqwdQw6GpY0LRRRSGQ3lytpZzXDYIjQtgnGT2Gfc8V53AwkvIo2bLyv87McknvXWeKLrybWCIFgzybuOhCjofxI/KuX01BNrcLYX5AzcjviuKs+erGFzrorlpuRd8S63/wAI5pMt7BZefFCuWRDggetec3PxMg1do/KidFUfM/bJ7Cum+JHiaLQNM8s2T3DXKmNQnQHua8Oe/aW0/dNtjdtxjC4xXROKknF7HPGTi7no1t4ykfUzDIuVyBlDmuh/4SGN0aMDzZX++qjoK8z8P3cVpYXNz9nE0wXYgJ4XP8Rrt/CUqSSWqytGHlU59TiuGcfZr3ep6ELT1l0ItS0E6xaTS2IKMM4hX5dx968x1HTr22naOa1licHGGQ17rcX1jZ3jMo2sBh2xgD6mq02q2Uqlw8cn0IIrkjNQuz0aOInHTlujwmOxvC2BbzsxwBhD1rVex1GVvskU1z9jAXzN+V3N3AB7A5Festc29wuCsezHTHesm7nhgyAAq/Ss54x/ZSNYzc3ZoydFit43ES7lij4y/Wuwju44YlDEevNefz6nbRXXmCTnPStU3AuGWUznyCAQO/0rmTlFc1jWvRcmmzsU1HzV+ViQ3ep47+SFwftDYI+4SPb8e361zlrqEEceNwVQCevNW47uC6OYZFDjoH4zW8K04q6epwyoK+qOottdlU7S345rVg1kvnkNXGorgF2g2YHG05yas21xsfkshNdcMZUj8RyTw0Hqjtk1BGHIxUqXSM3JwBXNRXxCKGG/8OlTM0hi3wncO4zXYsU2rrU5Hh0dAbuMHg5p32uI/wAQrlvtkigEqcemakjlEi5U49aFjddhPDWR1CTxyZCuDg4NP4PeuTkuHjwFYmnw6lOp+9WqxceqI+rysdXSdTWKuqS7ORzUD6jdtnBArR4iCVyFRkdCTVuzfKMmenI5riJby7BBSQkY78c1a0fULpdXt/OkYxs2wgd88D9SKj6zFu1ivYSSudvRRRW5kef+LNVjfxC1qrOfs0Sq4IOFZvm4/Ar/AJFU1RS4uYD0ABweaxtWuZLvxLqlzOiq32p4RsBwVQ7FP1woz71asZWSPejA4P3a8OdW9WV+57HsrU427GrNeaZeW22/gZ2ToSmRXkd14cSK4uf3LbJJC0YHAUEk/wD1q9VR4pwVIAY881E2mq8gbb/DxWksTVaVjGFGnFu55Zpvhq4gs5op5ViM0e4Hdz7Cu38KW8bSxZjjzAhDbRgg+lat1pskm6V2DNg5Y9TTLi8axjiks7UM87eUx6YIHU05VHKLk+hpFL4Y9St4g1Syt9IvS0XmyYKiJV3MT9K8at9P12aYvbwTxAnoTtFe6SQ28UYuLswxzP1OcZqo4sXUFJEY4J+XqK5vayh0Wp00pKK0ueZQWPiaBFLuxH90EGq12NSupPJP2neBypI5P4V6DeX1nGhAKsw4AzXKalqQgLGGEtIehHasFUblolc9OjOTWxzEFmVnb7X5g287B1OKW61W4kuMxI8cCjCLg4AFXG1drkxpNa5kXjcowW+tW7fKljLHcwRgFW2AE8jpyK6nKzvNGsnIm0G2v9ZmUYaCBQGeVugHtWxquo2GjTqtr+9k2jJcHjIBGB7jnNY32h57RrTTJ9loABKS+HfPYe1VxoVy8ymKynNuoLs2OG+lZtU3pazM+Vud5vTsdZp2rXl2iSiXGfU1tpqIXCTOrN7D+tcdbW16jk7QiEccgYFXbdgAWLHb2J4BI9K5G2noZ1KUG9DsYr+DaGjLgj7wrTtdSOcsoZMdRXIRzmRgd7CTAxWjaTybwdxPr71pCu4vQ4alBNanUPtkYOhO3rgdqqTs8DlgvGB7UlpeRt8qttY8bTV9XTbtdgG9G6V6EWqiucLTg7FOO5jnJQH25qVbQ5yrg/WiXTYZm82GUJJ3VOhojKQgoZWZ89NuKOWSfvCbT+EniimdgDjjvUcsqq2xW3N7Un2p0bAyFpYzCXDYw/qK0TjsRZrUgYymQ5D4Ixt29/WkEN4rB445AwOQRwQa3Lacf3lY+4rVgkRxyoFdEaEZa8xlKtKPQ14ZRNBHKFZQ6hgG6jI70UkB3QIfbFFdhynlGpop13UNoB/0qXOPXeabEmx+hrjNT8VTWPjPWLWCMSpDfTRlHc7jtcj7xzk8dTWxF4ts3h3zRyxN6FCf1FfOzi1N33Po/YzUVbVWOnWS3Qh3OGHWh9ZgiWPhirdx6Vy7+LNL2FlkLEcHg1UvfFthFBCxDYYHHHvQqkloiFhZSeqZ3Fzfwmzk2H5m6Z7Vzeq6zDp9jGqyr555ALcZ9K53/hM4boNbqcF+Fz6+n41w+qau9zM2/wCYKSBWsXUm7JWNqeDUf4mli7r3iLV7y8dblmiGMCN16e4z/OsdNa1K3X93dYHT3qKe9aZUVzuCjAY9QPSoVs3nBZBwP1rrjCKXvI0qK+lP8NC+NY1Rhuk2tHnBcrkD6kU1dWluyIjDuc9CnWlthcrbi1RWkiJBePGc11mmeGhM73mmxALHCS0MjEOj47eozUT9itlqVBV6dnKVl5/5lPToM/Z5SIlWNT5js3Tnv711gstO1HyX/e+Wi7cREBZGxnnHtXHrpupyukU9uVKKE5UkkV1kUAttFWCQeWuTk8A59gK4JySd73N6ybtrqU5NG09JWktLwQlgSIzjHX/CtabR7mJhFBdXEkJVSu0jBBGSKoJbWUmFZQvPT2rWgurayi8uHDZXG3ArD2ilvoZVHNWtr6mff2VxbxrAsfk7ursC7H+lFnpNx5QMjMYR1Zh/IVrQymcYdyoHOWpk1xGhVDM0megXpUNp6rYlTmvd6gJLaCN7YRsWwOWyK0tMmtRGVfdk8YHaqkEsjNmSCMp23jccVdAjVfltk59DitqVr3Oaq9LFlbWGSYmMMhHIIPWiW4ntYQZZoXjPeTgj6VEZXt7NpIY1eQH5Yi+AfxrnLr+1r9iLiwUKWyBGc7a6rqMdNznjFzlq9Dqk1qwReLoKxHVafDqkEyjyrkzuDn5gBXFzaacrIu5QB0fjFaOnbFz5SHcBU+2nexUqEOW6Z1UmotH8pijGTkhutQRzjzvMEu7I5ULgA1mJco7kS79w6ZNWFkjUggjB7itfaSepjyKOhqw3BySTgVbhvH9ePWsQSgHAOasrOoUA5qoVCJQTPQ9IkMulwuTknd/6EaKpaBbRz6LbyM0wJ3fdmdR949gcUV6sHJxTPOmkpNHy34vEi+OvELxFlI1O5yR/11apLLxFe2QjW8tY3jf7rMuCfxr0X4j+GrpvG+oaiUBjuRE6d8qI1Q/qprzrX9EnW2WRCzRr0XP3a8qdWlKq6c0fSxvOjF03rZaHRi9s7u18waaYyf7q5zVG5FnPFEio7NkgL0Ark9P1e8sCE859o7ZrpItTtr1UmljZrjfjgc/XNYVKDhqa0al/IhudFDHdFCFIGepzWNqNsJZ1fyWVzxIPU+tekWd3bRsIbkBHYfKH4zWfrH2BSJoIk8xD8yMetKnVnHVle0UpcskcHp+ktcT+WRz0INdTp2gSWc56nHA9x9Kpz61YRzFUiaJ88NnpUD+INRWb9xMHjA6sM1VT21VdjVNR92CO1GmQTyfaNscEuACQMA+/FNkju7OXzklCDjlRnNc4niG/ukRCUyox8iYzV211a4W7UvGfLC7SSTge9c0qTWrepHLU2ZqyeJL5YsFLd2UYBkORisa81pXuQ8gVnK8hFIWM+1Wn8me4LrESDzk1oW2jRTLnyAc9SRUe0TdpaglTp+9axStmt7pd0UytkDdvbBzjnp75q7FMkO5Y40LDoR3pL7wvaQxGdJFtW/vZwDTrLTJZIipmBbs6nINTKnbYn2sJLcss5uE2ssUZx93dT9PtbVsqVJJOdwPeqo0+aJyrSJLsOSO4+tX4b3y/3aW6tJjhQwFSoOTsZzkox91m0kEhjVAUwOh71ZW2QjYxAb2rMtr45/eIqY6nPSrDXwkQtHNlDxuUV6FPD31Z5k6tnZFk6cEkLNK2Bz97ipYEgMpeN1Z9vQc4rhtc1l4mxHfKZw3ELgMCK3/CmovPN50ypgpg7RjNdVPDq1zCpWa0Nt7ZLsbJYwR6nqKyX0m4s5jtAkXs3eunhCON3cmrvkK8eWAwKcaCnqL28oaHFyWTzKBt3OOhJx+FZ01pPbvsxIB3ycgfjXdzWsaDfg49MVV+x28xOz8Qehpyw6asOGJaOQjNwhDCTI7g1eS8cAb0BHqK2ZfD0bBpEfY45C9jWWdMuS21SoYnkHpWDw84s29vCSuek+FpBL4btHXod/8A6G1FX9Ohe20u0gkGJI4URvqFANFexBWikzypu8m0c94ztA8MN15W7AKM2fxAx/31XnJSxuy0B+Rj/CRXsmqWi3lhJGQSwG5ceo/zj8a8U8TWTLcnyxsnXJQjjcK8fMaNpc/c9TL6l1yXOS1rwrbtO/ksInH8B6fhVXSbe201xDeRsrE/LKvI/KteHUFv4xBcErcr91j1+hqlcQyTNs2EEHg9s1w+0mlySeh7kVf4txb3Tvt7GQMZYyMAqc/iKyGsrhcoyee6fIC5YEDseD1rp9AuI9PmMNygHmnABHeuok0yFh5hRQM7sEZBrSDkkuV3MpVlB2kjzKTRjeTxyvERuGHAHetW28NRfeVGA6YFdxa2FpcszIYSc87G6f4VqRWUdsACgHuBkGiSqvS9kZyxqXwo4qz0AQcrG2O4x1q7/Y0s/Hl7VrrJpbaFcmSMDGSduMVjXWtadDIc3DzMo3eXF6VH1eUnZO5i8ZL4noMtdDt7Zd0uOPWsXXfGUWklrayjDSY++eAKh1XxTHd77G2heJpkIjmZujeleci2urm9aN0lllBO5VGW4rvw2Bs+aZwV8bzKyd2aN3rV9fymWe5diemG4FX9H8Q3lrmE75I8feGTsycAn8cD8aZZeENXvIFl+zNDGSMtJwcH2rvdN8NaT4dUNt+1XZH32GcfT0rsrOlGFmclJVpSujIitfEd/AJruZrO2B4A++4/wrMsLu1tNRnLCV3zhSHzxzkH15xz7VveILrUL2EoH2Q9DGvGfqa46S1NnMjKjYbnIHQ1531mLuont4fAuSvNnXz3FzLZPtTyA68OQTxXIx3d21yLSa6mFsG4IYjNdRZ31xNp671YIh2kkcGoXsbWcMuCyo25HZcH8smsHiGtGbQowg3dGbb+HTPqCyRoZHB3ZJ4IrvNGtnspWGwLkbiPTNYjskVqslp8sq/eNNt764hdXWViD1ye9aU8Q4qzdznrUFU1SSPSbJFOM81riSGGE5wT9a4nStakmdY5PmY+jAD9a34LuCU8sM+ma9ChOLjoeVWpyjLUku9VRCAyunYHGRVO7v41wElV+5AFXptMFzHhc8nPWoV8NSMCciqmquyRMPZ2uzNi1Vo4OoZu4JrV0d01G9gwmSXG5Se3f9Kzbvw/cwZbaW+lbHgzTXF5PdyBgIhtQEEAsepz7Dt/tVFN1OdRki6ns+RyidtRRRXonCFeeeLtB82R0jwCfmQ+3+eK9DrP1e1Fxab8fNFlvw7/AOfasa9JVYcrNaNV05cyPB7vwtql7dgquCnAY8frWhbeE9QSNUlVWI75r0PzIkOAoqWOZG6IM1wrLote9I73mdTZI86j8MXDTrJPCdkZyM9jW0xKxmMryvBrr3dWUqYsg1kz6SWctGcKT3HSplgvZr3NRLGuo/f0OcfTYi2YVEbtyQON1Mit7q2PyzvjupOa0p7WVRmXKsDgGq7SyIfm+bFctSm1qdMKl1YqXOmreowmeQg9MNWM3hKBGEkEsqup45rp1uVJJIxUu+GRRjANEas4bSsTKEXujhp/BonYqz4UtlSOq1vadokFk5GxWZsZkK8sfWtlxx/Wm+Zggbaudac1aTJhShB3iivLFcAFGJ8sYwBUckRkj+UfN1rbimXgMSMgjI9KetpE3Ss3Tcuo1U5ehyN0kuQ8wL54OaqT6elwgiAATqTjkV2sumRyjB6VU/sYRsG3cdKxlRnfQ6IYmKOMexaEtCkm5O+O9S28Cgbf0ro59MxKdqZ7fWq8mlSeYrKCMdawdOVzf26aMprbyzgL8pHIqJ7NjFsAAYHIOOoroRYseoOe1TJZRE4cY9a1jBvQzdZIwrayk8vKuVYVcebULRd5A6cDrmtVYDC3ADKRjpUMsILYAI+tdEE4o55VOZ6kNhr1/E3mfd4xg9K2rPxYxfbMWA9VrJW1J7cgZxilay+UPtHNdNOtVjszCdOnLodha+IrWdgrFiP9quqtoVgiwq7Sx3MPeuA8J6OLvVxcOD5VthyM9W/hHX15/D3r0WvSoTlOPNI8+tCMJWiFFFFbGQUUUUAcpq2lm2uzJGD5MhyOOFPp/n+lQW8QGBjv1rrp4I7iIxyLkH9D6isVrZ4LkR7V2c5J6n0IpKIXZLDAhHSpTbJ6U6NcLxSkmrsSV5LKGQEMoIPrWTd+GYJsmNtpPatzNITUSpxlui4zlHZnHT+GrmKPaqh+ex7VmTaNfx/dicfhmvQ80vFc8sFTZ0Rxc4nm/k30GAYmbPUFTShrgSYlspMf3lFejlEPVQfwo8uM/wAC/lWTwC6Mv66+qOCGNudrflSrdFWwqMa7OTTraQ5KDNRtpNqRwmDU/U5LZj+tRe6ObSV2IG05NXEtnkwCvNa0emRIxLAMO3tVuOFE5Vea1hhf5jKWI7GD/Zrq5ZiSDjAx0oms2jGSmQR1xXQ4FBUEYI4rR4SHQj6xLqcp5ZHRBgVFJEHbG0jPoOK6aSxiY5VcU3+z4iOmKx+qM0+sHKmAqwxjA65FIUDHnFdFLpZwSm0n0NUpbEKSCuDWUqEo7mka0WzLChTmlSBrqZIIULyOcKoq79kUMA+ce1dJomjR2aC5lQm4Ycbh9we3uf8A63rSp0XJ2HOqoq5f06wj06yS2jOccs2MFj3P+e2KtUUV6SSSsjibu7sKKKKYgooooAKingWePaeGH3W9KlooA52XU0srhra6/dyDkZ6MPUHuKmTULeUfLIv51oajpdpqkIjuo923OxgcMpPof8jgVxN5oF3pjEsxaLdhJF7/AIdqlykulzSMYSW9mdWsqN0YU7INcPA9/wCZtSQ496unUb+1GZORQqqfQHSa2Z1eKSN0kUsjBgCVyPUHB/UVzcPiB8jevFX4tdgfqcVanFkODW6NekJxVNNTtn6OKsCaNxkMKZBJk0ZpFIPQilNMAzRmkzziilcApSaSimAueKSiigEBOBn0pAFdQSMg+opyqWYKoyTV2C2CAM4y/X6UnYFe5Da6fGjCV1BYcqD2q9RRUlBRRRQAUUUUAFFFFABRRRQAUhAZSrAEEYIPelooAy5tFgLb4VCn+7WdeaSZFKHINdLSMoYYYZFFkBxS6K0PGMiq82luCSEI+lds9op+6cexqs9s46x5HTjmp5EPnZwZtpQ+0Kw9zSst3C+RKdpAwo65+v5V2ht4jkFRmq02l28siyFFLrnacdM0lTsV7Tuc6l5fwJuJJHvU8Wuz4+ZM1rS6YHj2g1QOkPGcKMinaS2FdMdF4gQ8MpFTx61E8gXpk96y5tMePJCk/SoobV3nVCpwTzxS5pp2C0WtDqoZhMgZTxUtNtbVkiRY0OMdccVbSzY8udvsOa1M7Fap47WR+W+Ue/WrccEcX3Rz6nrUlK47DI4liGFH1PrT6KKQwooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAEZVcYZQR7iozbxMclB+HFFFAB9mi/ufqaalvEyklO57n1oooADaQHrGD+Jp6wQo25YkVh3CgGiigCSiiigAooooAKKKKACiiigAooooAKKKKAP//Z'"
6 |         }
7 |     ]   
8 | }


--------------------------------------------------------------------------------
/pytok/api/video.py:
--------------------------------------------------------------------------------
  1 | from __future__ import annotations
  2 | 
  3 | import asyncio
  4 | from datetime import datetime
  5 | import logging
  6 | import json
  7 | from urllib import parse as url_parsers
  8 | from typing import TYPE_CHECKING, ClassVar, Optional
  9 | 
 10 | import brotli
 11 | import requests
 12 | from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 13 | 
 14 | if TYPE_CHECKING:
 15 |     from ..tiktok import PyTok
 16 |     from .user import User
 17 |     from .sound import Sound
 18 |     from .hashtag import Hashtag
 19 | 
 20 | from .base import Base
 21 | from ..helpers import extract_tag_contents, edit_url, extract_video_id_from_url, extract_user_id_from_url
 22 | from .. import exceptions
 23 | 
 24 | logger = logging.getLogger("pytok.api.video")
 25 | 
 26 | class Counter:
 27 |     def __init__(self):
 28 |         self._counter = 0
 29 | 
 30 |     def add(self, n):
 31 |         self._counter += n
 32 | 
 33 |     def get(self):
 34 |         return self._counter
 35 | 
 36 | class Video(Base):
 37 |     """
 38 |     A TikTok Video class
 39 | 
 40 |     Example Usage
 41 |     ```py
 42 |     video = api.video(id='7041997751718137094')
 43 |     ```
 44 |     """
 45 | 
 46 |     parent: ClassVar[PyTok]
 47 | 
 48 |     id: Optional[str]
 49 |     """TikTok's ID of the Video"""
 50 |     create_time: Optional[datetime]
 51 |     """The creation time of the Video"""
 52 |     stats: Optional[dict]
 53 |     """TikTok's stats of the Video"""
 54 |     author: Optional[User]
 55 |     """The User who created the Video"""
 56 |     sound: Optional[Sound]
 57 |     """The Sound that is associated with the Video"""
 58 |     hashtags: Optional[list[Hashtag]]
 59 |     """A List of Hashtags on the Video"""
 60 |     as_dict: dict
 61 |     """The raw data associated with this Video."""
 62 | 
 63 |     def __init__(
 64 |             self,
 65 |             id: Optional[str] = None,
 66 |             username: Optional[str] = None,
 67 |             url: Optional[str] = None,
 68 |             data: Optional[dict] = None,
 69 |     ):
 70 |         """
 71 |         You must provide the id or a valid url, else this will fail.
 72 |         """
 73 |         self.id = id
 74 |         self.username = username
 75 |         if data is not None:
 76 |             self.as_dict = data
 77 |             self.__extract_from_data()
 78 |         elif url is not None:
 79 |             self.id = extract_video_id_from_url(url)
 80 |             self.username = extract_user_id_from_url(url)
 81 | 
 82 |         if self.id is None and url is None:
 83 |             raise TypeError("You must provide id or url parameter.")
 84 | 
 85 |     async def info(self, **kwargs) -> dict:
 86 |         """
 87 |         Returns a dictionary of all data associated with a TikTok Video.
 88 | 
 89 |         Example Usage
 90 |         ```py
 91 |         video_data = api.video(id='7041997751718137094').info()
 92 |         ```
 93 |         """
 94 |         if not hasattr(self, 'as_dict'):
 95 |             try:
 96 |                 video_data = await self._info_api(**kwargs)
 97 |             except Exception as ex:
 98 |                 self.parent.logger.debug(f"API info fetch failed with exception: {ex}, falling back to scraping")
 99 |                 video_data = await self._info_scraping(**kwargs)
100 |                 
101 |             self.as_dict = video_data
102 |         else:
103 |             video_data = self.as_dict
104 | 
105 |         return video_data
106 |     
107 |     async def _info_api(self, **kwargs) -> dict:
108 |         video_obj = self.parent.tiktok_api.video(id=self.id, url=self._get_url())
109 |         video_data = await video_obj.info()
110 |         return video_data
111 |     
112 |     async def _info_scraping(self, **kwargs) -> dict:
113 |         url = self._get_url()
114 |         page = self.parent._page
115 |         if page.url != url:
116 |             await self.view()
117 | 
118 |         await self.check_and_resolve_login_popup()
119 | 
120 |         # get initial html data
121 |         initial_html_response = self.get_responses(url)[-1]
122 |         html_body = await self.get_response_body(initial_html_response)
123 |         contents = extract_tag_contents(html_body)
124 |         res = json.loads(contents)
125 | 
126 |         video_detail = res['__DEFAULT_SCOPE__']['webapp.video-detail']
127 |         if video_detail['statusCode'] != 0:
128 |             raise exceptions.NotAvailableException(
129 |                 f"Content is not available with status message: {video_detail['statusMsg']}")
130 |         video_data = video_detail['itemInfo']['itemStruct']
131 |         return video_data
132 | 
133 |     async def network_info(self, **kwargs) -> dict:
134 |         """
135 |         Returns a dictionary of all network data associated with a TikTok Video.
136 | 
137 |         Example Usage
138 |         ```py
139 |         video_data = api.video(id='7041997751718137094').network_data()
140 |         ```
141 |         """
142 |         url = self._get_url()
143 |         page = self.parent._page
144 |         if page.url != url:
145 |             await self.view()
146 |         initial_html_response = self.get_responses(url)[-1]
147 |         network_info = {}
148 |         network_info['server_addr'] = await initial_html_response.server_addr()
149 |         network_info['headers'] = await initial_html_response.all_headers()
150 |         return network_info
151 | 
152 |     async def bytes_network_info(self, **kwargs) -> dict:
153 |         """
154 |         Returns a dictionary of all network data associated with a TikTok Video.
155 | 
156 |         Example Usage
157 |         ```py
158 |         video_data = api.video(id='7041997751718137094').bytes_network_data()
159 |         ```
160 |         """
161 |         play_path = url_parsers.urlparse(self.as_dict['video']['playAddr']).path
162 |         reqs = self.get_requests(play_path)
163 |         if len(reqs) == 0:
164 |             # TODO load page and pull
165 |             raise Exception("No requests found for video")
166 |         for req in reqs:
167 |             try:
168 |                 res = await req.response()
169 |                 network_info = {}
170 |                 network_info['server_addr'] = await res.server_addr()
171 |                 network_info['headers'] = await res.all_headers()
172 |                 return network_info
173 |             except Exception:
174 |                 continue
175 |         else:
176 |             raise Exception("Failed to get video bytes")
177 | 
178 |     def _get_url(self) -> str:
179 |         if self.username is not None:
180 |             return f"https://www.tiktok.com/@{self.username}/video/{self.id}"
181 |         else:
182 |             # will autoresolve to correct username
183 |             return f"https://www.tiktok.com/@user/video/{self.id}"
184 | 
185 |     async def view(self, **kwargs) -> None:
186 |         """
187 |         Opens the TikTok Video in your default browser.
188 | 
189 |         Example Usage
190 |         ```py
191 |         api.video(id='7041997751718137094').view()
192 |         ```
193 |         """
194 |         page = self.parent._page
195 |         url = self._get_url()
196 |         if page.url == url:
197 |             return
198 |         try:
199 |             async with page.expect_request(url) as event:
200 |                 await page.goto(url)
201 |                 request = await event.value
202 |                 response = await request.response()
203 |                 if response.status >= 300:
204 |                     raise exceptions.NotAvailableException("Content is not available")
205 |             # no need to check for captcha, because video data is in the html regardless
206 |             await self.wait_for_content_or_unavailable('[id="main-content-video_detail"]', 'Video currently unavailable')
207 |         except exceptions.NotAvailableException as e:
208 |             # fetch specific error message
209 |             initial_html_response = self.get_responses(url)[-1]
210 |             html_body = await self.get_response_body(initial_html_response)
211 |             contents = extract_tag_contents(html_body)
212 |             res = json.loads(contents)
213 | 
214 |             video_detail = res['__DEFAULT_SCOPE__']['webapp.video-detail']
215 |             raise exceptions.NotAvailableException(f"Content is not available with status message: {video_detail['statusMsg']}")
216 |         except PlaywrightTimeoutError as e:
217 |             raise exceptions.TimeoutException(str(e))
218 |         
219 |     async def _related_videos(self, counter, count=20):
220 |         data_request_path = "api/related/item_list"
221 |         data_requests = self.get_requests(data_request_path)
222 |         for req in data_requests:
223 |             # parse params from url
224 |             url_parsed = url_parsers.urlparse(req.url)
225 |             params = url_parsers.parse_qs(url_parsed.query)
226 |             if params['itemID'][0] != self.id:
227 |                 continue
228 |             res = await req.response()
229 |             if res is None:
230 |                 continue
231 |             body = await res.body()
232 |             if len(body) == 0:
233 |                 continue
234 |             d = await res.json()
235 |             for v in d.get('itemList', []):
236 |                 yield v
237 |                 counter.add(1)
238 |             if counter.get() >= count:
239 |                 break
240 | 
241 |     async def related_videos(self, count=20) -> list[dict]:
242 |         """
243 |         Returns a list of related
244 |         TikTok Videos to the current Video.
245 |         
246 |         """
247 |         try:
248 |             async for video in self._related_videos_api(count=count):
249 |                 yield video
250 |         except Exception as ex:
251 |             self.parent.logger.debug(f"API related videos fetch failed with exception: {ex}, falling back to scraping")
252 |             async for video in self._related_videos_scraping(count=count):
253 |                 yield video
254 | 
255 |     async def _related_videos_api(self, count=20) -> list[dict]:
256 |         video_obj = self.parent.tiktok_api.video(id=self.id, url=self._get_url())
257 |         async for video in video_obj.related_videos(count=count):
258 |             yield video
259 | 
260 |     async def _related_videos_scraping(self, count=20) -> list[dict]:
261 |         counter = Counter()
262 |         async for video in self._related_videos(counter, count=count):
263 |             yield video
264 | 
265 |         # get via scroll
266 |         # solve captcha if necessary
267 |         if counter.get() == 0:
268 |             await self.check_and_wait_for_captcha()
269 |             await self.parent._page.reload()
270 |             await asyncio.sleep(5)
271 |             async for video in self._related_videos(counter, count=count):
272 |                 yield video
273 | 
274 |     async def bytes(self, **kwargs) -> bytes:
275 |         """
276 |         Returns the bytes of a TikTok Video.
277 | 
278 |         Example Usage
279 |         ```py
280 |         video_bytes = api.video(id='7041997751718137094').bytes()
281 | 
282 |         # Saving The Video
283 |         with open('saved_video.mp4', 'wb') as output:
284 |             output.write(video_bytes)
285 |         ```
286 |         """
287 |         bytes_url = self.as_dict['video']['playAddr']
288 |         if len(bytes_url) == 0:
289 |             raise exceptions.NotAvailableException("Post does not have a video")
290 |         play_path = url_parsers.urlparse(bytes_url).path
291 |         resps = self.get_responses(play_path)
292 |         if len(resps) > 0:
293 |             for res in resps:
294 |                 if hasattr(res, '_body'):
295 |                     if len(res._body) > 0:
296 |                         return res._body
297 |         # if we don't have the bytes in the response, we need to get it from the server
298 | 
299 |         # send the request ourselves
300 |         try:
301 |             return await asyncio.wait_for(self._request_bytes(bytes_url), timeout=10)
302 |         except TimeoutError:
303 |             raise exceptions.TimeoutException("Failed to get video bytes in time")
304 | 
305 |     async def _request_bytes(self, bytes_url):
306 |         bytes_headers = {
307 |             'sec-ch-ua': '"HeadlessChrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"', 
308 |             'referer': 'https://www.tiktok.com/', 
309 |             'accept-encoding': 'identity;q=1, *;q=0', 
310 |             'sec-ch-ua-mobile': '?0', 
311 |             'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.4 Safari/537.36', 
312 |             'range': 'bytes=0-', 
313 |             'sec-ch-ua-platform': '"Windows"'
314 |         }
315 |         cookies = await self.parent._context.cookies()
316 |         cookies = {cookie['name']: cookie['value'] for cookie in cookies}
317 |         r = requests.get(bytes_url, headers=bytes_headers, cookies=cookies)
318 |         if r.content is not None or len(r.content) > 0:
319 |             return r.content
320 |         raise Exception("Failed to get video bytes")
321 | 
322 |     async def _get_comments_and_req(self, count):
323 |         # get request
324 |         data_request_path = "api/comment/list"
325 |         data_responses = self.get_responses(data_request_path)
326 | 
327 |         amount_yielded = 0
328 |         all_comments = []
329 |         processed_urls = []
330 | 
331 |         valid_data_request = None
332 |         for data_response in data_responses:
333 |             try:
334 |                 res = await data_response.json()
335 | 
336 |                 self.parent.request_cache['comments'] = data_response.request
337 | 
338 |                 processed_urls.append(data_response.url)
339 | 
340 |                 comments = res.get("comments", [])
341 | 
342 |                 amount_yielded += len(comments)
343 |                 all_comments += comments
344 | 
345 |                 if amount_yielded > count:
346 |                     return all_comments, processed_urls, True
347 | 
348 |                 has_more = res.get("has_more")
349 |                 if has_more != 1:
350 |                     self.parent.logger.info(
351 |                         "TikTok isn't sending more TikToks beyond this point."
352 |                     )
353 |                     return all_comments, processed_urls, True
354 |             except Exception:
355 |                 pass
356 | 
357 |         return all_comments, processed_urls, False
358 | 
359 |     async def _get_comment_replies(self, comment, batch_size):
360 |         if 'comments' not in self.parent.request_cache:
361 |             return
362 |         data_request = self.parent.request_cache['comments']
363 |         num_already_fetched = len(
364 |             comment.get('reply_comment', []) if comment.get('reply_comment', []) is not None else [])
365 |         num_comments_to_fetch = comment['reply_comment_total'] - num_already_fetched
366 |         while num_comments_to_fetch > 0:
367 | 
368 |             url_parsed = url_parsers.urlparse(data_request.url)
369 |             params = url_parsers.parse_qs(url_parsed.query)
370 |             params['cursor'] = num_already_fetched
371 |             del params['aweme_id']
372 |             params['count'] = min(num_comments_to_fetch, batch_size)
373 |             params['item_id'] = comment['aweme_id']
374 |             params['comment_id'] = comment['cid']
375 |             params['focus_state'] = 'true'
376 |             url_path = url_parsed.path.replace("api/comment/list", "api/comment/list/reply")
377 |             next_url = f"{url_parsed.scheme}://{url_parsed.netloc}{url_path}?{url_parsers.urlencode(params, doseq=True)}"
378 |             cookies = await self.parent._context.cookies()
379 |             cookies = {cookie['name']: cookie['value'] for cookie in cookies}
380 |             r = requests.get(next_url, headers=data_request.headers, cookies=cookies)
381 | 
382 |             if not r.content:
383 |                 return
384 | 
385 |             res = r.json()
386 | 
387 |             reply_comments = res.get("comments", [])
388 | 
389 |             if reply_comments:
390 |                 comment['reply_comment'] = comment['reply_comment'] + reply_comments if comment[
391 |                     'reply_comment'] else reply_comments
392 | 
393 |             has_more = res.get("has_more")
394 |             if has_more != 1:
395 |                 self.parent.logger.info(
396 |                     "TikTok isn't sending more TikToks beyond this point."
397 |                 )
398 |                 break
399 | 
400 |             await self.parent.request_delay()
401 | 
402 |             num_already_fetched = len(comment['reply_comment'])
403 |             num_comments_to_fetch = comment['reply_comment_total'] - num_already_fetched
404 | 
405 |     async def comments(self, count=200, batch_size=100):
406 |         if (self.id and self.username) or self.as_dict:
407 |             await self.view()
408 |             await self.wait_for_content_or_unavailable_or_captcha('css=[data-e2e=comment-level-1]',
409 |                                                                   'Be the first to comment!')
410 |             # TODO allow multi layer comment fetch
411 | 
412 |             amount_yielded = 0
413 |             all_comments, processed_urls, finished = await self._get_comments_and_req(count)
414 | 
415 |             for comment in all_comments:
416 |                 await self._get_comment_replies(comment, batch_size)
417 | 
418 |             amount_yielded += len(all_comments)
419 |             for comment in all_comments:
420 |                 yield comment
421 | 
422 |             if finished:
423 |                 return
424 | 
425 |             # so that we don't re-yield any comments previously yielded
426 |             comment_ids = set(comment['cid'] for comment in all_comments)
427 |             try:
428 |                 async for comment in self._get_api_comments(count, batch_size, comment_ids):
429 |                     yield comment
430 |             except exceptions.ApiFailedException as e:
431 |                 async for comment in self._get_scroll_comments(count, amount_yielded, processed_urls):
432 |                     yield comment
433 |         else:
434 |             # if we only have the video id, we need to entirely rely on the api
435 |             async for comment in self._get_api_comments(count, batch_size, set()):
436 |                 yield comment
437 | 
438 |     async def _get_scroll_comments(self, count, amount_yielded, processed_urls):
439 |         page = self.parent._page
440 |         if page.url != self._get_url():
441 |             await self.view()
442 |         tries = 0
443 | 
444 |         data_request_path = "api/comment/list"
445 |         while amount_yielded < count:
446 |             # scroll down to induce request
447 |             await self.scroll_to(10000)
448 |             await self.slight_scroll_up()
449 |             await self.check_and_wait_for_captcha()
450 |             await self.check_and_close_signin()
451 | 
452 |             data_responses = self.get_responses(data_request_path)
453 |             data_responses = [data_response for data_response in data_responses if
454 |                               data_response.url not in processed_urls]
455 | 
456 |             if len(data_responses) == 0:
457 |                 if tries > 5:
458 |                     logger.debug(f"Not sending anymore!")
459 |                     break
460 |                 tries += 1
461 | 
462 |             for data_response in data_responses:
463 |                 try:
464 |                     res = await data_response.json()
465 | 
466 |                     processed_urls.append(data_response.url)
467 | 
468 |                     comments = res.get("comments", [])
469 | 
470 |                     for comment in comments:
471 |                         await self._get_comment_replies(comment, 100)
472 | 
473 |                     amount_yielded += len(comments)
474 |                     for comment in comments:
475 |                         yield comment
476 | 
477 |                     if amount_yielded > count:
478 |                         return
479 | 
480 |                     has_more = res.get("has_more")
481 |                     if has_more != 1:
482 |                         self.parent.logger.info(
483 |                             "TikTok isn't sending more TikToks beyond this point."
484 |                         )
485 |                         return
486 |                 except Exception as e:
487 |                     processed_urls.append(data_response.url)
488 | 
489 |     async def _get_comments_via_requests(self, count, cursor, data_request):
490 |         ms_tokens = await self.parent.get_ms_tokens()
491 |         next_url = edit_url(data_request.url, {'count': count, 'cursor': cursor, 'aweme_id': self.id})
492 |         cookies = await self.parent._context.cookies()
493 |         cookies = {cookie['name']: cookie['value'] for cookie in cookies}
494 |         headers = await data_request.all_headers()
495 |         headers = {k: v for k, v in headers.items() if not k.startswith(':')}
496 |         headers['referer'] = None
497 |         r = requests.get(next_url, headers=headers, cookies=cookies)
498 | 
499 |         if r.status_code != 200:
500 |             raise Exception(f"Failed to get comments with status code {r.status_code}")
501 | 
502 |         if len(r.content) == 0:
503 |             logger.debug("Failed to get comments from API, switching to scroll")
504 |             raise exceptions.ApiFailedException("No content in response")
505 | 
506 |         try:
507 |             res = r.json()
508 |         except Exception:
509 |             res = json.loads(brotli.decompress(r.content).decode())
510 | 
511 |         return res
512 | 
513 |     async def _get_api_comments(self, count, batch_size, comment_ids):
514 | 
515 |         data_request = self.parent.request_cache['comments']
516 | 
517 |         try:
518 |             amount_yielded = 0
519 |             cursor = 0
520 |             while amount_yielded < count:
521 |                 # try directly requesting through browser
522 |                 url = edit_url(data_request.url,
523 |                                {'count': 20, 'cursor': cursor, 'aweme_id': self.id})  # , 'msToken': ms_tokens[-1]})
524 |                 page = self.parent._page
525 |                 async with page.expect_request(url) as event:
526 |                     await page.goto(url)
527 |                     request = await event.value
528 |                     response = await request.response()
529 |                     if response.status >= 300:
530 |                         raise exceptions.NotAvailableException("Content is not available")
531 | 
532 |                 if response.status != 200:
533 |                     raise Exception(f"Failed to get comments with status code {response.status}")
534 | 
535 |                 content = await response.body()
536 |                 if len(content) == 0:
537 |                     raise Exception("No content in response")
538 | 
539 |                 res = await response.json()
540 |                 cursor = res.get("cursor", 0)
541 | 
542 |                 comments = res.get("comments", [])
543 |                 amount_yielded += len(comments)
544 |                 for comment in comments:
545 |                     if comment['cid'] not in comment_ids:
546 |                         try:
547 |                             await self._get_comment_replies(comment, batch_size)
548 |                         except Exception:
549 |                             pass
550 |                         yield comment
551 |         except Exception as e:
552 |             try:
553 |                 # try getting all at once
554 |                 retries = 5
555 |                 for _ in range(retries):
556 |                     try:
557 |                         cursor = '0'
558 |                         res = await self._get_comments_via_requests(count, cursor, data_request)
559 | 
560 |                         comments = res.get("comments", [])
561 |                         for comment in comments:
562 |                             if comment['cid'] not in comment_ids:
563 |                                 try:
564 |                                     await self._get_comment_replies(comment, batch_size)
565 |                                 except Exception:
566 |                                     pass
567 |                                 yield comment
568 | 
569 |                         return
570 |                     except Exception as e:
571 |                         pass
572 |                 else:
573 |                     logger.debug("Failed to get all comments at once, trying batched...")
574 |                     raise Exception("Failed to get comments")
575 |             except Exception as e:
576 | 
577 |                 amount_yielded = len(comment_ids)
578 |                 cursor = 0
579 |                 while amount_yielded < count:
580 |                     res = await self._get_comments_via_requests(20, cursor, data_request)
581 | 
582 |                     if res.get('type') == 'verify':
583 |                         # force new request for cache
584 |                         self._get_comments_and_req()
585 | 
586 |                     cursor = res.get("cursor", 0)
587 |                     comments = res.get("comments", [])
588 | 
589 |                     if comments:
590 |                         for comment in comments:
591 |                             await self._get_comment_replies(comment, batch_size)
592 | 
593 |                         amount_yielded += len(comments)
594 |                         for comment in comments:
595 |                             yield comment
596 | 
597 |                     has_more = res.get("has_more")
598 |                     if has_more != 1:
599 |                         self.parent.logger.info(
600 |                             "TikTok isn't sending more TikToks beyond this point."
601 |                         )
602 |                         return
603 | 
604 |                     await self.parent.request_delay()
605 | 
606 |     def __extract_from_data(self) -> None:
607 |         data = self.as_dict
608 |         keys = data.keys()
609 | 
610 |         if "author" in keys:
611 |             self.id = data["id"]
612 |             self.username = data["author"]["uniqueId"]
613 |             self.create_time = datetime.fromtimestamp(int(data["createTime"]))
614 |             self.stats = data["stats"]
615 |             self.author = self.parent.user(data=data["author"])
616 |             self.sound = self.parent.sound(data=data["music"])
617 | 
618 |             self.hashtags = [
619 |                 self.parent.hashtag(data=hashtag)
620 |                 for hashtag in data.get("challenges", [])
621 |             ]
622 | 
623 |         if self.id is None:
624 |             Video.parent.logger.error(
625 |                 f"Failed to create Video with data: {data}\nwhich has keys {data.keys()}"
626 |             )
627 | 
628 |     def __repr__(self):
629 |         return self.__str__()
630 | 
631 |     def __str__(self):
632 |         return f"PyTok.video(id='{self.id}')"
633 | 
634 |     # def __getattr__(self, name):
635 |     #     # Handle author, sound, hashtags, as_dict
636 |     #     if name in ["author", "sound", "hashtags", "stats", "create_time", "as_dict"]:
637 |     #         self.as_dict = self.info()
638 |     #         self.__extract_from_data()
639 |     #         return self.__getattribute__(name)
640 | 
641 |     #     raise AttributeError(f"{name} doesn't exist on PyTok.api.Video")
642 | 


--------------------------------------------------------------------------------