├── .github └── workflows │ └── auto-yapf.yml ├── .gitignore ├── LICENSE ├── README.md ├── config ├── monitoring.json.template └── token.json.template ├── cqhttp_notifier.py ├── discord_notifier.py ├── following_monitor.py ├── graphql_api.py ├── like_monitor.py ├── login.py ├── main.py ├── monitor_base.py ├── notifier_base.py ├── profile_monitor.py ├── requirements.txt ├── status_tracker.py ├── telegram_notifier.py ├── tweet_monitor.py ├── twitter_watcher.py └── utils.py /.github/workflows/auto-yapf.yml: -------------------------------------------------------------------------------- 1 | name: Auto YAPF 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.8"] 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Set up Python ${{ matrix.python-version }} 14 | uses: actions/setup-python@v3 15 | with: 16 | python-version: ${{ matrix.python-version }} 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | pip install yapf 21 | - name: Formatting the code 22 | run: | 23 | yapf --style '{based_on_style: google, column_limit: 120, allow_multiline_lambdas: True}' --recursive --in-place . 24 | git config --global user.name 'github-actions' 25 | git config --global user.email 'github-actions@github.com' 26 | git diff --quiet && git diff --staged --quiet || git commit -am 'Auto YAPF with Google python style' && git push 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # This repo 132 | cache 133 | log 134 | cookies 135 | config/*.json 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ion 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Twitter Monitor 2 | 3 | Monitor the `following`, `tweet`, ~~`like`~~ and `profile` of a Twitter user and send the changes to the telegram channel. 4 | 5 | Data is crawled from Twitter web’s GraphQL API. 6 | 7 | (Due to the unstable return results of Twitter Web's `following` API, accounts with more than 100 followings are not recommended to use `following` monitor.) 8 | 9 | ## Deployed channel sample 10 | 11 | https://t.me/twitter_monitor_menu 12 | 13 | ## Usage 14 | 15 | ### Setup 16 | 17 | (Requires **python >= 3.10**) 18 | 19 | Clone code and install dependent pip packages 20 | 21 | ```bash 22 | git clone https://github.com/ionic-bond/twitter-monitor.git 23 | cd twitter-monitor 24 | pip3 install -r ./requirements.txt 25 | ``` 26 | 27 | ### Prepare required tokens 28 | 29 | - Create a Telegram bot and get it's token: 30 | 31 | https://t.me/BotFather 32 | 33 | - Unofficial Twitter account auth: 34 | 35 | You need to prepare one or more normal twitter accounts, and then use the following command to generate auth cookies 36 | 37 | ```bash 38 | python3 main.py generate-auth-cookie --username "{username}" --password "{password}" 39 | ``` 40 | 41 | ### Fill in config 42 | 43 | - First make a copy from the config templates 44 | 45 | ```bash 46 | cp ./config/token.json.template ./config/token.json 47 | cp ./config/monitoring.json.template ./config/monitoring.json 48 | ``` 49 | 50 | - Edit `config/token.json` 51 | 52 | 1. Fill in `telegram_bot_token` 53 | 54 | 2. Fill in `twitter_auth_username_list` according to your prepared Twitter account auth 55 | 56 | 3. Now you can test whether the tokens can be used by 57 | ```bash 58 | python3 main.py check-tokens 59 | ``` 60 | 61 | - Edit `config/monitoring.json` 62 | 63 | (You need to fill in some telegram chat id here, you can get them from https://t.me/userinfobot and https://t.me/myidbot) 64 | 65 | 1. If you need to view monitor health information (starting summary, daily summary, alert), fill in `maintainer_chat_id` 66 | 67 | 2. Fill in one or more user to `monitoring_user_list`, and their notification telegram chat id, weight, which monitors to enable. The greater the weight, the higher the query frequency. The **profile monitor** is forced to enable (because it triggers the other 3 monitors), and the other 3 monitors are free to choose whether to enable or not 68 | 69 | 3. You can check if your telegram token and chat id are correct by 70 | ```bash 71 | python3 main.py check-tokens --telegram_chat_id {your_chat_id} 72 | ``` 73 | 74 | ### Run 75 | 76 | ```bash 77 | python3 main.py run 78 | ``` 79 | | Flag | Default | Description | 80 | | :-------------------: | :-----: | :-------------------------------------------------------: | 81 | | --interval | 15 | Monitor run interval | 82 | | --confirm | False | Confirm with the maintainer during initialization | 83 | | --listen_exit_command | False | Liten the "exit" command from telegram maintainer chat id | 84 | | --send_daily_summary | False | Send daily summary to telegram maintainer | 85 | 86 | ## Contact me 87 | 88 | Telegram: [@ionic_bond](https://t.me/ionic_bond) 89 | 90 | ## Donate 91 | 92 | [PayPal Donate](https://www.paypal.com/donate/?hosted_button_id=D5DRBK9BL6DUA) or [PayPal](https://paypal.me/ionicbond3) 93 | -------------------------------------------------------------------------------- /config/monitoring.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "maintainer_chat_id": 1034977773, 3 | "monitoring_user_list": [ 4 | { 5 | "username": "Twitter", 6 | "telegram_chat_id_list": [1034977773], 7 | "discord_webhook_url_list": [], 8 | "monitoring_following": true, 9 | "monitoring_tweet": true, 10 | "monitoring_following_count": false, 11 | "monitoring_like_count": true, 12 | "monitoring_tweet_count": true 13 | }, 14 | { 15 | "username": "TwitterDev", 16 | "telegram_chat_id_list": [1034977773], 17 | "monitoring_following": false, 18 | "monitoring_tweet": true, 19 | "monitoring_following_count": true, 20 | "monitoring_like_count": false, 21 | "monitoring_tweet_count": false 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /config/token.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "telegram_bot_token": "", 3 | "twitter_auth_username_list": [ 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /cqhttp_notifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | 4 | from typing import List, Union 5 | 6 | from notifier_base import Message, NotifierBase 7 | 8 | 9 | def _remove_http(text: str) -> str: 10 | # QQ will intercept http links 11 | text = text.replace(r'https://', '') 12 | text = text.replace(r'http://', '') 13 | return text 14 | 15 | 16 | class CqhttpMessage(Message): 17 | 18 | def __init__(self, 19 | url_list: List[str], 20 | text: str, 21 | photo_url_list: Union[List[str], None] = None, 22 | video_url_list: Union[List[str], None] = None): 23 | super().__init__(text, photo_url_list, video_url_list) 24 | self.url_list = url_list 25 | 26 | 27 | class CqhttpNotifier(NotifierBase): 28 | notifier_name = 'Cqhttp' 29 | 30 | @classmethod 31 | def init(cls, token: str, logger_name: str): 32 | cls.headers = {'Authorization': 'Bearer {}'.format(token)} if token else None 33 | cls.logger = logging.getLogger('{}'.format(logger_name)) 34 | cls.logger.info('Init cqhttp notifier succeed.') 35 | super().init() 36 | 37 | @classmethod 38 | def _post_request_to_cqhttp(cls, url: str, data: dict): 39 | response = requests.post(url, headers=cls.headers, data=data, timeout=60) 40 | if response.status_code != 200 or response.json().get('status', '') != 'ok': 41 | raise RuntimeError('Post request error: {}, {}\nurl: {}\ndata: {}'.format( 42 | response.status_code, response.text, url, str(data))) 43 | 44 | @classmethod 45 | def _send_text_to_single_chat(cls, url: str, text: str): 46 | data = {'message': _remove_http(text)} 47 | cls._post_request_to_cqhttp(url, data) 48 | 49 | @classmethod 50 | def _send_photo_to_single_chat(cls, url: str, photo_url: str): 51 | data = {'message': '[CQ:image,file={}]'.format(photo_url)} 52 | cls._post_request_to_cqhttp(url, data) 53 | 54 | @classmethod 55 | def _send_video_to_single_chat(cls, url: str, video_url: str): 56 | data = {'message': '[CQ:video,file={}]'.format(video_url)} 57 | cls._post_request_to_cqhttp(url, data) 58 | 59 | @classmethod 60 | def send_message(cls, message: CqhttpMessage): 61 | assert cls.initialized 62 | assert isinstance(message, CqhttpMessage) 63 | for url in message.url_list: 64 | cls._send_text_to_single_chat(url, message.text) 65 | if message.photo_url_list: 66 | for photo_url in message.photo_url_list: 67 | cls._send_photo_to_single_chat(url, photo_url) 68 | if message.video_url_list: 69 | for video_url in message.video_url_list: 70 | cls._send_video_to_single_chat(url, video_url) 71 | -------------------------------------------------------------------------------- /discord_notifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Union 3 | 4 | import requests 5 | 6 | from notifier_base import Message, NotifierBase 7 | 8 | 9 | class DiscordMessage(Message): 10 | 11 | def __init__(self, 12 | webhook_url_list: List[str], 13 | text: str, 14 | photo_url_list: Union[List[str], None] = None, 15 | video_url_list: Union[List[str], None] = None): 16 | super().__init__(text, photo_url_list, video_url_list) 17 | self.webhook_url_list = webhook_url_list 18 | 19 | 20 | class DiscordNotifier(NotifierBase): 21 | notifier_name = 'Discord' 22 | 23 | @classmethod 24 | def init(cls, logger_name: str): 25 | cls.logger = logging.getLogger('{}'.format(logger_name)) 26 | cls.logger.info('Init discord notifier succeed.') 27 | super().init() 28 | 29 | @classmethod 30 | def _post_request_to_discord(cls, url: str, data: dict): 31 | response = requests.post(url, json=data, timeout=60) 32 | if response.status_code != 204: # Discord webhook returns 204 No Content on success 33 | raise RuntimeError('Post request error: {}, {}\nurl: {}\ndata: {}'.format( 34 | response.status_code, response.text, url, str(data))) 35 | 36 | @classmethod 37 | def _send_text_to_discord(cls, url: str, text: str): 38 | data = {'content': text} 39 | cls._post_request_to_discord(url, data) 40 | 41 | @classmethod 42 | def send_message(cls, message: DiscordMessage): 43 | assert cls.initialized 44 | assert isinstance(message, DiscordMessage) 45 | for url in message.webhook_url_list: 46 | cls._send_text_to_discord(url, message.text) 47 | if message.photo_url_list: 48 | for photo_url in message.photo_url_list: 49 | cls._send_text_to_discord(url, photo_url) 50 | if message.video_url_list: 51 | for video_url in message.video_url_list: 52 | cls._send_text_to_discord(url, video_url) -------------------------------------------------------------------------------- /following_monitor.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import List, Union, Tuple, Dict 3 | 4 | from monitor_base import MonitorBase 5 | from utils import find_all, find_one, get_cursor, get_content 6 | 7 | 8 | class FollowingMonitor(MonitorBase): 9 | monitor_type = 'Following' 10 | 11 | def __init__(self, username: str, title: str, token_config: dict, user_config: dict, cookies_dir: str): 12 | super().__init__(monitor_type=self.monitor_type, 13 | username=username, 14 | title=title, 15 | token_config=token_config, 16 | user_config=user_config, 17 | cookies_dir=cookies_dir) 18 | 19 | self.following_dict = self.get_all_following(self.user_id) 20 | 21 | self.logger.info('Init following monitor succeed.\nUser id: {}\nFollowing {} users: {}'.format( 22 | self.user_id, len(self.following_dict), 23 | [find_one(following, 'screen_name') for following in self.following_dict.values()])) 24 | 25 | def get_all_following(self, user_id: int) -> Dict[str, dict]: 26 | api_name = 'Following' 27 | params = {'userId': user_id, 'includePromotedContent': True, 'count': 1000} 28 | following_dict = dict() 29 | 30 | while True: 31 | json_response = self.twitter_watcher.query(api_name, params) 32 | following_list = find_all(json_response, 'user_results') 33 | while not following_list and not find_one(json_response, 'result'): 34 | import json 35 | self.logger.error(json.dumps(json_response, indent=2)) 36 | time.sleep(10) 37 | json_response = self.twitter_watcher.query(api_name, params) 38 | following_list = find_all(json_response, 'user_results') 39 | 40 | for following in following_list: 41 | user_id = find_one(following, 'rest_id') 42 | following_dict[user_id] = following 43 | 44 | cursor = get_cursor(json_response) 45 | if not cursor or cursor.startswith('-1|') or cursor.startswith('0|'): 46 | break 47 | params['cursor'] = cursor 48 | 49 | return following_dict 50 | 51 | def parse_user_details(self, user: int) -> Tuple[str, Union[str, None]]: 52 | content = get_content(user) 53 | details_str = 'Name: {}'.format(content.get('name', '')) 54 | details_str += '\nBio: {}'.format(content.get('description', '')) 55 | details_str += '\nWebsite: {}'.format( 56 | content.get('entities', {}).get('url', {}).get('urls', [{}])[0].get('expanded_url', '')) 57 | details_str += '\nJoined at: {}'.format(content.get('created_at', '')) 58 | details_str += '\nFollowing: {}'.format(content.get('friends_count', -1)) 59 | details_str += '\nFollowers: {}'.format(content.get('followers_count', -1)) 60 | details_str += '\nTweets: {}'.format(content.get('statuses_count', -1)) 61 | return details_str, content.get('profile_image_url_https', '').replace('_normal', '') 62 | 63 | def detect_changes(self, old_following_dict: set, new_following_dict: set) -> bool: 64 | if old_following_dict.keys() == new_following_dict.keys(): 65 | return True 66 | max_changes = max(len(old_following_dict) / 2, 10) 67 | dec_user_id_list = old_following_dict.keys() - new_following_dict.keys() 68 | inc_user_id_list = new_following_dict.keys() - old_following_dict.keys() 69 | if len(dec_user_id_list) > max_changes or len(inc_user_id_list) > max_changes: 70 | return False 71 | if dec_user_id_list: 72 | self.logger.info('Unfollow: {}'.format(dec_user_id_list)) 73 | for dec_user_id in dec_user_id_list: 74 | message = 'Unfollow: @{}'.format(find_one(old_following_dict[dec_user_id], 'screen_name')) 75 | details_str, profile_image_url = self.parse_user_details(old_following_dict[dec_user_id]) 76 | if details_str: 77 | message += '\n{}'.format(details_str) 78 | self.send_message(message=message, photo_url_list=[profile_image_url] if profile_image_url else []) 79 | if inc_user_id_list: 80 | self.logger.info('Follow: {}'.format(inc_user_id_list)) 81 | for inc_user_id in inc_user_id_list: 82 | message = 'Follow: @{}'.format(find_one(new_following_dict[inc_user_id], 'screen_name')) 83 | details_str, profile_image_url = self.parse_user_details(new_following_dict[inc_user_id]) 84 | if details_str: 85 | message += '\n{}'.format(details_str) 86 | self.send_message(message=message, photo_url_list=[profile_image_url] if profile_image_url else []) 87 | return True 88 | 89 | def watch(self) -> bool: 90 | following_dict = self.get_all_following(self.user_id) 91 | if not self.detect_changes(self.following_dict, following_dict): 92 | return False 93 | self.following_dict = following_dict 94 | self.update_last_watch_time() 95 | return True 96 | 97 | def status(self) -> str: 98 | return 'Last: {}, number: {}'.format(self.get_last_watch_time(), len(self.following_dict)) 99 | -------------------------------------------------------------------------------- /graphql_api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import bs4 5 | import requests 6 | from x_client_transaction.utils import generate_headers, handle_x_migration, get_ondemand_file_url 7 | from x_client_transaction import ClientTransaction 8 | 9 | from utils import check_initialized 10 | 11 | 12 | class GraphqlAPI(): 13 | initialized = False 14 | 15 | def __new__(cls): 16 | raise Exception('Do not instantiate this class!') 17 | 18 | @classmethod 19 | def init(cls) -> None: 20 | cls.logger = logging.getLogger('api') 21 | while not cls.update_api_data(): 22 | time.sleep(10) 23 | cls.initialized = True 24 | 25 | @classmethod 26 | def update_api_data(cls): 27 | response = requests.get( 28 | 'https://github.com/ionic-bond/TwitterInternalAPIDocument/raw/master/docs/json/API.json', timeout=300) 29 | if response.status_code != 200: 30 | cls.logger.error('Request returned an error: {} {}.'.format(response.status_code, response.text)) 31 | return False 32 | json_data = response.json() 33 | 34 | if not json_data.get('graphql', {}): 35 | cls.logger.error('Can not get Graphql API data from json') 36 | return False 37 | if not json_data.get('header', {}): 38 | cls.logger.error('Can not get header data from json') 39 | return False 40 | 41 | cls.graphql_api_data = json_data['graphql'] 42 | cls.headers = json_data['header'] 43 | cls.init_client_transaction() 44 | cls.logger.info('Pull GraphQL API data success, API number: {}'.format(len(cls.graphql_api_data))) 45 | return True 46 | 47 | @classmethod 48 | def init_client_transaction(cls): 49 | session = requests.Session() 50 | session.headers = generate_headers() 51 | home_page = session.get(url="https://x.com") 52 | home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser') 53 | ondemand_file_url = get_ondemand_file_url(response=home_page_response) 54 | ondemand_file = session.get(url=ondemand_file_url) 55 | ondemand_file_response = bs4.BeautifulSoup(ondemand_file.content, 'html.parser') 56 | cls.ct = ClientTransaction(home_page_response=home_page_response, ondemand_file_response=ondemand_file_response) 57 | 58 | @classmethod 59 | def get_clint_transaction_id(cls, method: str, url: str): 60 | return cls.ct.generate_transaction_id(method=method, 61 | path=url.replace('https://x.com', '').replace('https://twitter.com', '')) 62 | 63 | @classmethod 64 | @check_initialized 65 | def get_api_data(cls, api_name): 66 | if api_name not in cls.graphql_api_data: 67 | raise ValueError('Unkonw API name: {}'.format(api_name)) 68 | 69 | api_data = cls.graphql_api_data[api_name] 70 | headers = cls.headers.copy() 71 | transaction_id = cls.get_clint_transaction_id(api_data['method'], api_data['url']) 72 | headers['x-client-transaction-id'] = transaction_id 73 | 74 | return api_data['url'], api_data['method'], headers, api_data['features'] 75 | 76 | 77 | GraphqlAPI.init() 78 | -------------------------------------------------------------------------------- /like_monitor.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from typing import List, Union, Set 4 | 5 | from monitor_base import MonitorBase 6 | from utils import parse_media_from_tweet, parse_text_from_tweet, find_all, find_one 7 | 8 | 9 | def _get_like_id(like: dict) -> str: 10 | return find_one(like, 'rest_id') 11 | 12 | 13 | def _get_like_id_set(like_list: list) -> Set[str]: 14 | return set(_get_like_id(like) for like in like_list) 15 | 16 | 17 | def _filter_advertisers(like_list: list) -> list: 18 | result = [] 19 | for like in like_list: 20 | if find_one(like, 'card'): 21 | continue 22 | if find_one(like, 'userLabelType') == 'BusinessLabel': 23 | continue 24 | if find_one(like, '__typename') == 'TweetWithVisibilityResultss': 25 | continue 26 | source = find_one(like, 'source') 27 | if source: 28 | if 'Advertiser' in source or 'advertiser' in source: 29 | continue 30 | result.append(like) 31 | return result 32 | 33 | 34 | class LikeMonitor(MonitorBase): 35 | monitor_type = 'Like' 36 | like_id_set_max_size = 1000 37 | 38 | def __init__(self, username: str, title: str, token_config: dict, user_config: dict, cookies_dir: str): 39 | super().__init__(monitor_type=self.monitor_type, 40 | username=username, 41 | title=title, 42 | token_config=token_config, 43 | user_config=user_config, 44 | cookies_dir=cookies_dir) 45 | 46 | like_list = self.get_like_list() 47 | while like_list is None: 48 | time.sleep(60) 49 | like_list = self.get_like_list() 50 | self.existing_like_id_set = _get_like_id_set(like_list) 51 | 52 | self.logger.info('Init like monitor succeed.\nUser id: {}\nExisting {} likes: {}'.format( 53 | self.user_id, len(self.existing_like_id_set), self.existing_like_id_set)) 54 | 55 | def get_like_list(self) -> Union[list, None]: 56 | api_name = 'Likes' 57 | params = {'userId': self.user_id, 'includePromotedContent': True, 'count': 1000} 58 | json_response = self.twitter_watcher.query(api_name, params) 59 | if json_response is None: 60 | return None 61 | return _filter_advertisers(find_all(json_response, 'tweet_results')) 62 | 63 | def watch(self) -> bool: 64 | like_list = self.get_like_list() 65 | if like_list is None: 66 | return False 67 | 68 | new_like_list = [] 69 | for like in like_list: 70 | like_id = _get_like_id(like) 71 | if like_id in self.existing_like_id_set: 72 | break 73 | self.existing_like_id_set.add(like_id) 74 | new_like_list.append(like) 75 | 76 | for like in reversed(new_like_list): 77 | photo_url_list, video_url_list = parse_media_from_tweet(like) 78 | text = parse_text_from_tweet(like) 79 | user = find_one(like, 'user_results') 80 | username = find_one(user, 'screen_name') 81 | self.send_message('@{}: {}'.format(username, text), photo_url_list, video_url_list) 82 | 83 | self.update_last_watch_time() 84 | return True 85 | 86 | def status(self) -> str: 87 | return 'Last: {}, num: {}'.format(self.get_last_watch_time(), len(self.existing_like_id_set)) 88 | -------------------------------------------------------------------------------- /login.py: -------------------------------------------------------------------------------- 1 | # Reference: https://github.com/trevorhobenshield/twitter-api-client/blob/main/twitter/login.py 2 | import sys 3 | 4 | from httpx import Client 5 | 6 | from utils import find_all 7 | from graphql_api import GraphqlAPI 8 | 9 | 10 | def update_token(client: Client, key: str, url: str, **kwargs) -> Client: 11 | caller_name = sys._getframe(1).f_code.co_name 12 | try: 13 | headers = { 14 | 'x-guest-token': client.cookies.get('guest_token', ''), 15 | 'x-csrf-token': client.cookies.get('ct0', ''), 16 | 'x-twitter-auth-type': 'OAuth2Client' if client.cookies.get('auth_token') else '', 17 | 'x-client-transaction-id': GraphqlAPI.get_clint_transaction_id(method='POST', url=url) 18 | } 19 | client.headers.update(headers) 20 | r = client.post(url, **kwargs) 21 | if r.status_code != 200: 22 | print(f'[error] {r.status_code} {r.text}') 23 | info = r.json() 24 | 25 | for task in info.get('subtasks', []): 26 | if task.get('enter_text', {}).get('keyboard_type') == 'email': 27 | print(f"[warning] {' '.join(find_all(task, 'text'))}") 28 | client.cookies.set('confirm_email', 'true') # signal that email challenge must be solved 29 | 30 | if task.get('subtask_id') == 'LoginAcid': 31 | if task['enter_text']['hint_text'].casefold() == 'confirmation code': 32 | print(f"[warning] email confirmation code challenge.") 33 | client.cookies.set('confirmation_code', 'true') 34 | 35 | client.cookies.set(key, info[key]) 36 | 37 | except KeyError as e: 38 | client.cookies.set('flow_errors', 'true') # signal that an error occurred somewhere in the flow 39 | print(f'[error] failed to update token at {caller_name}\n{e}') 40 | return client 41 | 42 | 43 | def init_guest_token(client: Client) -> Client: 44 | return update_token(client, 'guest_token', 'https://api.x.com/1.1/guest/activate.json') 45 | 46 | 47 | def flow_start(client: Client) -> Client: 48 | return update_token(client, 49 | 'flow_token', 50 | 'https://api.x.com/1.1/onboarding/task.json', 51 | params={'flow_name': 'login'}, 52 | json={ 53 | "input_flow_data": { 54 | "flow_context": { 55 | "debug_overrides": {}, 56 | "start_location": { 57 | "location": "splash_screen" 58 | } 59 | } 60 | }, 61 | "subtask_versions": {} 62 | }) 63 | 64 | 65 | def flow_instrumentation(client: Client) -> Client: 66 | return update_token(client, 67 | 'flow_token', 68 | 'https://api.x.com/1.1/onboarding/task.json', 69 | json={ 70 | "flow_token": 71 | client.cookies.get('flow_token'), 72 | "subtask_inputs": [{ 73 | "subtask_id": "LoginJsInstrumentationSubtask", 74 | "js_instrumentation": { 75 | "response": "{}", 76 | "link": "next_link" 77 | } 78 | }], 79 | }) 80 | 81 | 82 | def flow_username(client: Client) -> Client: 83 | return update_token(client, 84 | 'flow_token', 85 | 'https://api.x.com/1.1/onboarding/task.json', 86 | json={ 87 | "flow_token": 88 | client.cookies.get('flow_token'), 89 | "subtask_inputs": [{ 90 | "subtask_id": "LoginEnterUserIdentifierSSO", 91 | "settings_list": { 92 | "setting_responses": [{ 93 | "key": "user_identifier", 94 | "response_data": { 95 | "text_data": { 96 | "result": client.cookies.get('username') 97 | } 98 | } 99 | }], 100 | "link": "next_link" 101 | } 102 | }], 103 | }) 104 | 105 | 106 | def flow_password(client: Client) -> Client: 107 | return update_token(client, 108 | 'flow_token', 109 | 'https://api.x.com/1.1/onboarding/task.json', 110 | json={ 111 | "flow_token": 112 | client.cookies.get('flow_token'), 113 | "subtask_inputs": [{ 114 | "subtask_id": "LoginEnterPassword", 115 | "enter_password": { 116 | "password": client.cookies.get('password'), 117 | "link": "next_link" 118 | } 119 | }] 120 | }) 121 | 122 | 123 | def flow_finish(client: Client) -> Client: 124 | return update_token(client, 125 | 'flow_token', 126 | 'https://api.x.com/1.1/onboarding/task.json', 127 | json={ 128 | "flow_token": client.cookies.get('flow_token'), 129 | "subtask_inputs": [], 130 | }) 131 | 132 | 133 | def confirm_email(client: Client) -> Client: 134 | return update_token(client, 135 | 'flow_token', 136 | 'https://api.x.com/1.1/onboarding/task.json', 137 | json={ 138 | "flow_token": 139 | client.cookies.get('flow_token'), 140 | "subtask_inputs": [{ 141 | "subtask_id": "LoginAcid", 142 | "enter_text": { 143 | "text": client.cookies.get('email'), 144 | "link": "next_link" 145 | } 146 | }] 147 | }) 148 | 149 | 150 | def solve_confirmation_challenge(client: Client, confirmation_code, **kwargs) -> Client: 151 | return update_token(client, 152 | 'flow_token', 153 | 'https://api.x.com/1.1/onboarding/task.json', 154 | json={ 155 | "flow_token": 156 | client.cookies.get('flow_token'), 157 | 'subtask_inputs': [{ 158 | 'subtask_id': 'LoginAcid', 159 | 'enter_text': { 160 | 'text': confirmation_code, 161 | 'link': 'next_link', 162 | }, 163 | },], 164 | }) 165 | 166 | 167 | def execute_login_flow(client: Client, confirmation_code, **kwargs) -> Client: 168 | client = init_guest_token(client) 169 | for fn in [flow_start, flow_instrumentation, flow_username, flow_password]: 170 | client = fn(client) 171 | 172 | # solve email challenge 173 | if client.cookies.get('confirm_email') == 'true': 174 | client = confirm_email(client) 175 | 176 | # solve confirmation challenge (Proton Mail only) 177 | if client.cookies.get('confirmation_code') == 'true': 178 | if not confirmation_code: 179 | print(f'[warning] Please check your email for a confirmation code and fill it to --confirmation_code') 180 | return None 181 | client = solve_confirmation_challenge(client, confirmation_code, **kwargs) 182 | 183 | client = flow_finish(client) 184 | 185 | return client 186 | 187 | 188 | def login(username: str, password: str, confirmation_code: str = None, **kwargs) -> Client: 189 | client = Client(cookies={ 190 | "username": username, 191 | "password": password, 192 | "guest_token": None, 193 | "flow_token": None, 194 | }, 195 | headers=GraphqlAPI.headers | { 196 | 'content-type': 'application/json', 197 | 'x-twitter-active-user': 'yes', 198 | 'x-twitter-client-language': 'en', 199 | 'User-Agent': 'Mozilla/5.0 (platform; rv:geckoversion) Gecko/geckotrail Firefox/firefoxversion', 200 | }, 201 | follow_redirects=True) 202 | client = execute_login_flow(client, confirmation_code, **kwargs) 203 | if not client or client.cookies.get('flow_errors') == 'true': 204 | raise Exception(f'[error] {username} login failed') 205 | return client 206 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import json 4 | import logging 5 | import os 6 | import sys 7 | 8 | import click 9 | from apscheduler.executors.pool import ThreadPoolExecutor 10 | from apscheduler.schedulers.background import BlockingScheduler 11 | 12 | from cqhttp_notifier import CqhttpNotifier 13 | from discord_notifier import DiscordNotifier 14 | from following_monitor import FollowingMonitor 15 | from graphql_api import GraphqlAPI 16 | from like_monitor import LikeMonitor 17 | from login import login 18 | from monitor_base import MonitorManager 19 | from profile_monitor import ProfileMonitor 20 | from status_tracker import StatusTracker 21 | from telegram_notifier import TelegramMessage, TelegramNotifier, send_alert 22 | from tweet_monitor import TweetMonitor 23 | from twitter_watcher import TwitterWatcher 24 | 25 | CONFIG_FIELD_TO_MONITOR = { 26 | 'monitoring_profile': ProfileMonitor, 27 | 'monitoring_following': FollowingMonitor, 28 | 'monitoring_like': LikeMonitor, 29 | 'monitoring_tweet': TweetMonitor 30 | } 31 | 32 | 33 | def _setup_logger(name: str, log_file_path: str, level=logging.INFO): 34 | file_handler = logging.FileHandler(log_file_path) 35 | file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) 36 | logger = logging.getLogger(name) 37 | logger.setLevel(level) 38 | logger.addHandler(file_handler) 39 | 40 | 41 | def _send_summary(telegram_chat_id: str, monitors: dict, watcher: TwitterWatcher): 42 | for modoule, data in monitors.items(): 43 | monitor_status = {} 44 | for username, monitor in data.items(): 45 | monitor_status[username] = monitor.status() 46 | TelegramNotifier.put_message_into_queue( 47 | TelegramMessage(chat_id_list=[telegram_chat_id], 48 | text='{}: {}'.format(modoule, json.dumps(monitor_status, indent=4)))) 49 | tokens_status = watcher.check_tokens() 50 | TelegramNotifier.put_message_into_queue( 51 | TelegramMessage(chat_id_list=[telegram_chat_id], 52 | text='Tokens status: {}'.format(json.dumps(tokens_status, indent=4)))) 53 | 54 | 55 | def _check_monitors_status(telegram_token: str, telegram_chat_id: int, monitors: dict): 56 | alerts = StatusTracker.check() 57 | for title, monitor in monitors[ProfileMonitor.monitor_type].items(): 58 | if monitor.username.element != monitor.original_username: 59 | alerts.append('{} username changed to {}'.format(title, monitor.username.element)) 60 | if alerts: 61 | send_alert(token=telegram_token, chat_id=telegram_chat_id, message='Alert: \n{}'.format('\n'.join(alerts))) 62 | 63 | 64 | def _check_tokens_status(telegram_token: str, telegram_chat_id: int, watcher: TwitterWatcher): 65 | tokens_status = watcher.check_tokens() 66 | failed_tokens = [token for token, status in tokens_status.items() if status == False] 67 | if failed_tokens: 68 | send_alert(token=telegram_token, 69 | chat_id=telegram_chat_id, 70 | message='Some tokens failed: {}'.format(json.dumps(tokens_status, indent=4))) 71 | 72 | 73 | @click.group() 74 | def cli(): 75 | pass 76 | 77 | 78 | @cli.command(context_settings={'show_default': True}) 79 | @click.option('--log_dir', default=os.path.join(sys.path[0], 'log')) 80 | @click.option('--cookies_dir', default=os.path.join(sys.path[0], 'cookies')) 81 | @click.option('--token_config_path', default=os.path.join(sys.path[0], 'config/token.json')) 82 | @click.option('--monitoring_config_path', default=os.path.join(sys.path[0], 'config/monitoring.json')) 83 | @click.option('--interval', default=15, help="Monitor run interval") 84 | @click.option('--confirm', is_flag=True, default=False, help="Confirm with the maintainer during initialization") 85 | @click.option('--listen_exit_command', 86 | is_flag=True, 87 | default=False, 88 | help="Liten the \"exit\" command from telegram maintainer chat id") 89 | @click.option('--send_daily_summary', is_flag=True, default=False, help="Send daily summary to telegram maintainer") 90 | def run(log_dir, cookies_dir, token_config_path, monitoring_config_path, interval, confirm, listen_exit_command, 91 | send_daily_summary): 92 | os.makedirs(log_dir, exist_ok=True) 93 | logging.basicConfig(filename=os.path.join(log_dir, 'main'), 94 | format='%(asctime)s - %(levelname)s - %(message)s', 95 | level=logging.WARNING) 96 | _setup_logger('api', os.path.join(log_dir, 'twitter-api')) 97 | _setup_logger('status', os.path.join(log_dir, 'status-tracker')) 98 | 99 | with open(os.path.join(token_config_path), 'r') as token_config_file: 100 | token_config = json.load(token_config_file) 101 | telegram_bot_token = token_config.get('telegram_bot_token', '') 102 | twitter_auth_username_list = token_config.get('twitter_auth_username_list', []) 103 | assert twitter_auth_username_list 104 | with open(os.path.join(monitoring_config_path), 'r') as monitoring_config_file: 105 | monitoring_config = json.load(monitoring_config_file) 106 | assert monitoring_config['monitoring_user_list'] 107 | 108 | _setup_logger('telegram', os.path.join(log_dir, 'telegram')) 109 | _setup_logger('cqhttp', os.path.join(log_dir, 'cqhttp')) 110 | _setup_logger('discord', os.path.join(log_dir, 'discord')) 111 | TelegramNotifier.init(token=telegram_bot_token, logger_name='telegram') 112 | CqhttpNotifier.init(token=token_config.get('cqhttp_access_token', ''), logger_name='cqhttp') 113 | DiscordNotifier.init(logger_name='discord') 114 | 115 | monitors = dict() 116 | for monitor_cls in CONFIG_FIELD_TO_MONITOR.values(): 117 | monitors[monitor_cls.monitor_type] = dict() 118 | executors = {'default': ThreadPoolExecutor(len(monitoring_config['monitoring_user_list']))} 119 | scheduler = BlockingScheduler(executors=executors) 120 | for monitoring_user in monitoring_config['monitoring_user_list']: 121 | username = monitoring_user['username'] 122 | title = monitoring_user.get('title', username) 123 | for config_field, monitor_cls in CONFIG_FIELD_TO_MONITOR.items(): 124 | if monitoring_user.get(config_field, False) or monitor_cls is ProfileMonitor: 125 | monitor_type = monitor_cls.monitor_type 126 | logger_name = '{}-{}'.format(title, monitor_type) 127 | _setup_logger(logger_name, os.path.join(log_dir, logger_name)) 128 | monitors[monitor_type][title] = monitor_cls(username, title, token_config, monitoring_user, cookies_dir) 129 | if monitor_cls is ProfileMonitor: 130 | scheduler.add_job(monitors[monitor_type][title].watch, trigger='interval', seconds=interval) 131 | _setup_logger('monitor-caller', os.path.join(log_dir, 'monitor-caller')) 132 | MonitorManager.init(monitors=monitors) 133 | 134 | scheduler.add_job(GraphqlAPI.update_api_data, trigger='cron', hour='*') 135 | 136 | if monitoring_config['maintainer_chat_id']: 137 | # maintainer_chat_id should be telegram chat id. 138 | maintainer_chat_id = monitoring_config['maintainer_chat_id'] 139 | twitter_watcher = TwitterWatcher(twitter_auth_username_list, cookies_dir) 140 | _send_summary(maintainer_chat_id, monitors, twitter_watcher) 141 | scheduler.add_job(_check_monitors_status, 142 | trigger='cron', 143 | hour='*', 144 | args=[telegram_bot_token, maintainer_chat_id, monitors]) 145 | scheduler.add_job(_check_tokens_status, 146 | trigger='cron', 147 | hour='*', 148 | args=[telegram_bot_token, maintainer_chat_id, twitter_watcher]) 149 | if send_daily_summary: 150 | scheduler.add_job(_send_summary, 151 | trigger='cron', 152 | hour='6', 153 | args=[maintainer_chat_id, monitors, twitter_watcher]) 154 | if confirm: 155 | if not TelegramNotifier.confirm( 156 | TelegramMessage(chat_id_list=[maintainer_chat_id], 157 | text='Please confirm the initialization information')): 158 | TelegramNotifier.put_message_into_queue( 159 | TelegramMessage(chat_id_list=[maintainer_chat_id], text='Monitor will exit now.')) 160 | raise RuntimeError('Initialization information confirm error') 161 | TelegramNotifier.put_message_into_queue( 162 | TelegramMessage(chat_id_list=[maintainer_chat_id], text='Monitor initialization succeeded.')) 163 | if listen_exit_command: 164 | TelegramNotifier.listen_exit_command(maintainer_chat_id) 165 | 166 | scheduler.start() 167 | 168 | 169 | @cli.command(context_settings={'show_default': True}) 170 | @click.option('--cookies_dir', default=os.path.join(sys.path[0], 'cookies')) 171 | @click.option('--token_config_path', default=os.path.join(sys.path[0], 'config/token.json')) 172 | @click.option('--telegram_chat_id') 173 | @click.option('--test_username', default='X') 174 | @click.option('--output_response', is_flag=True, default=False) 175 | def check_tokens(cookies_dir, token_config_path, telegram_chat_id, test_username, output_response): 176 | with open(os.path.join(token_config_path), 'r') as token_config_file: 177 | token_config = json.load(token_config_file) 178 | telegram_bot_token = token_config.get('telegram_bot_token', '') 179 | twitter_auth_username_list = token_config.get('twitter_auth_username_list', []) 180 | assert twitter_auth_username_list 181 | twitter_watcher = TwitterWatcher(twitter_auth_username_list, cookies_dir) 182 | result = json.dumps(twitter_watcher.check_tokens(test_username, output_response), indent=4) 183 | print(result) 184 | if telegram_chat_id: 185 | TelegramNotifier.init(telegram_bot_token, '') 186 | TelegramNotifier.send_message(TelegramMessage(chat_id_list=[telegram_chat_id], text=result)) 187 | 188 | 189 | @cli.command(context_settings={'show_default': True}) 190 | @click.option('--cookies_dir', default=os.path.join(sys.path[0], 'cookies')) 191 | @click.option('--username', required=True) 192 | @click.option('--password', required=True) 193 | @click.option('--confirmation_code', required=False, default=None) 194 | def generate_auth_cookie(cookies_dir, username, password, confirmation_code): 195 | os.makedirs(cookies_dir, exist_ok=True) 196 | client = login(username=username, password=password, confirmation_code=confirmation_code) 197 | cookies = client.cookies 198 | dump_path = os.path.join(cookies_dir, '{}.json'.format(username)) 199 | with open(dump_path, 'w') as f: 200 | f.write(json.dumps(dict(cookies), indent=2)) 201 | print('Saved to {}'.format(dump_path)) 202 | 203 | 204 | if __name__ == '__main__': 205 | cli() 206 | -------------------------------------------------------------------------------- /monitor_base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from abc import ABC, abstractmethod 3 | from datetime import datetime 4 | from typing import List, Union 5 | 6 | from cqhttp_notifier import CqhttpMessage, CqhttpNotifier 7 | from discord_notifier import DiscordMessage, DiscordNotifier 8 | from status_tracker import StatusTracker 9 | from telegram_notifier import TelegramMessage, TelegramNotifier 10 | from twitter_watcher import TwitterWatcher 11 | 12 | 13 | class MonitorBase(ABC): 14 | 15 | def __init__(self, monitor_type: str, username: str, title: str, token_config: dict, user_config: dict, 16 | cookies_dir: str): 17 | logger_name = '{}-{}'.format(title, monitor_type) 18 | self.logger = logging.getLogger(logger_name) 19 | self.twitter_watcher = TwitterWatcher(token_config.get('twitter_auth_username_list', []), cookies_dir) 20 | self.username = username 21 | self.user_id = self.twitter_watcher.get_id_by_username(username) 22 | if not self.user_id: 23 | raise RuntimeError('Initialization error, please check if username {} exists'.format(username)) 24 | self.telegram_chat_id_list = user_config.get('telegram_chat_id_list', None) 25 | self.cqhttp_url_list = user_config.get('cqhttp_url_list', None) 26 | self.discord_webhook_url_list = user_config.get('discord_webhook_url_list', None) 27 | self.message_prefix = '[{}][{}]'.format(username, monitor_type) 28 | self.update_last_watch_time() 29 | 30 | def update_last_watch_time(self): 31 | StatusTracker.update_monitor_status(self.monitor_type, self.username) 32 | 33 | def get_last_watch_time(self): 34 | return StatusTracker.get_monitor_status(self.monitor_type, self.username) 35 | 36 | def send_message(self, 37 | message: str, 38 | photo_url_list: Union[List[str], None] = None, 39 | video_url_list: Union[List[str], None] = None): 40 | message = '{} {}'.format(self.message_prefix, message) 41 | self.logger.info('Sending message: {}\n'.format(message)) 42 | if photo_url_list: 43 | photo_url_list = [photo_url for photo_url in photo_url_list if photo_url] 44 | if video_url_list: 45 | video_url_list = [video_url for video_url in video_url_list if video_url] 46 | if photo_url_list: 47 | self.logger.info('Photo: {}'.format(', '.join(photo_url_list))) 48 | if video_url_list: 49 | self.logger.info('Video: {}'.format(', '.join(video_url_list))) 50 | if self.telegram_chat_id_list: 51 | TelegramNotifier.put_message_into_queue( 52 | TelegramMessage(chat_id_list=self.telegram_chat_id_list, 53 | text=message, 54 | photo_url_list=photo_url_list, 55 | video_url_list=video_url_list)) 56 | if self.cqhttp_url_list: 57 | CqhttpNotifier.put_message_into_queue( 58 | CqhttpMessage(url_list=self.cqhttp_url_list, 59 | text=message, 60 | photo_url_list=photo_url_list, 61 | video_url_list=video_url_list)) 62 | if self.discord_webhook_url_list: 63 | DiscordNotifier.put_message_into_queue( 64 | DiscordMessage(webhook_url_list=self.discord_webhook_url_list, 65 | text=message, 66 | photo_url_list=photo_url_list, 67 | video_url_list=video_url_list)) 68 | 69 | @abstractmethod 70 | def watch(self) -> bool: 71 | pass 72 | 73 | @abstractmethod 74 | def status(self) -> str: 75 | pass 76 | 77 | 78 | class MonitorManager(): 79 | monitors = None 80 | 81 | def __new__(self): 82 | raise Exception('Do not instantiate this class!') 83 | 84 | @classmethod 85 | def init(cls, monitors: dict): 86 | cls.monitors = monitors 87 | cls.logger = logging.getLogger('monitor-caller') 88 | 89 | @classmethod 90 | def get(cls, monitor_type: str, username: str) -> Union[MonitorBase, None]: 91 | assert cls.monitors is not None 92 | monitors_by_type = cls.monitors.get(monitor_type, None) 93 | assert monitors_by_type is not None 94 | monitor = monitors_by_type.get(username, None) 95 | return monitor 96 | 97 | @classmethod 98 | def call(cls, monitor_type: str, username: str) -> bool: 99 | monitor = cls.get(monitor_type, username) 100 | if not monitor: 101 | return True 102 | return monitor.watch() 103 | -------------------------------------------------------------------------------- /notifier_base.py: -------------------------------------------------------------------------------- 1 | import queue 2 | import threading 3 | from abc import ABC, abstractmethod 4 | from typing import List, Union 5 | 6 | from status_tracker import StatusTracker 7 | from utils import check_initialized 8 | 9 | 10 | class Message: 11 | 12 | def __init__(self, 13 | text: str, 14 | photo_url_list: Union[List[str], None] = None, 15 | video_url_list: Union[List[str], None] = None): 16 | self.text = text 17 | self.photo_url_list = photo_url_list 18 | self.video_url_list = video_url_list 19 | 20 | 21 | class NotifierBase(ABC): 22 | initialized = False 23 | 24 | def __new__(self): 25 | raise Exception('Do not instantiate this class!') 26 | 27 | @classmethod 28 | @abstractmethod 29 | def init(cls): 30 | cls.message_queue = queue.SimpleQueue() 31 | StatusTracker.set_notifier_status(cls.notifier_name, True) 32 | cls.initialized = True 33 | cls.work_start() 34 | 35 | @classmethod 36 | @abstractmethod 37 | @check_initialized 38 | def send_message(cls, message: Message): 39 | pass 40 | 41 | @classmethod 42 | @check_initialized 43 | def _work(cls): 44 | while True: 45 | message = cls.message_queue.get() 46 | try: 47 | StatusTracker.set_notifier_status(cls.notifier_name, False) 48 | cls.send_message(message) 49 | StatusTracker.set_notifier_status(cls.notifier_name, True) 50 | except Exception as e: 51 | print(e) 52 | cls.logger.error(e) 53 | 54 | @classmethod 55 | @check_initialized 56 | def work_start(cls): 57 | threading.Thread(target=cls._work, daemon=True).start() 58 | 59 | @classmethod 60 | @check_initialized 61 | def put_message_into_queue(cls, message: Message): 62 | cls.message_queue.put(message) 63 | -------------------------------------------------------------------------------- /profile_monitor.py: -------------------------------------------------------------------------------- 1 | import time 2 | from datetime import datetime, timedelta 3 | from functools import cached_property 4 | from typing import List, Union 5 | 6 | from following_monitor import FollowingMonitor 7 | from like_monitor import LikeMonitor 8 | from monitor_base import MonitorBase, MonitorManager 9 | from tweet_monitor import TweetMonitor 10 | from utils import find_one, get_content 11 | 12 | MESSAGE_TEMPLATE = '{} changed\nOld: {}\nNew: {}' 13 | SUB_MONITOR_LIST = [FollowingMonitor, LikeMonitor, TweetMonitor] 14 | 15 | 16 | class ProfileParser(): 17 | 18 | def __init__(self, json_response: dict): 19 | self.content = get_content(find_one(json_response, 'user')) 20 | self.json_response = json_response 21 | 22 | @cached_property 23 | def name(self) -> str: 24 | return self.content.get('name', '') 25 | 26 | @cached_property 27 | def username(self) -> str: 28 | return self.content.get('screen_name', '') 29 | 30 | @cached_property 31 | def location(self) -> str: 32 | return self.content.get('location', '') 33 | 34 | @cached_property 35 | def bio(self) -> str: 36 | return self.content.get('description', '') 37 | 38 | @cached_property 39 | def website(self) -> str: 40 | return self.content.get('entities', {}).get('url', {}).get('urls', [{}])[0].get('expanded_url', '') 41 | 42 | @cached_property 43 | def followers_count(self) -> int: 44 | return self.content.get('followers_count', 0) 45 | 46 | @cached_property 47 | def following_count(self) -> int: 48 | return self.content.get('friends_count', 0) 49 | 50 | @cached_property 51 | def like_count(self) -> int: 52 | return self.content.get('favourites_count', 0) 53 | 54 | @cached_property 55 | def tweet_count(self) -> int: 56 | return self.content.get('statuses_count', 0) 57 | 58 | @cached_property 59 | def profile_image_url(self) -> str: 60 | return self.content.get('profile_image_url_https', '').replace('_normal', '') 61 | 62 | @cached_property 63 | def profile_banner_url(self) -> str: 64 | return self.content.get('profile_banner_url', '') 65 | 66 | @cached_property 67 | def pinned_tweet(self) -> str: 68 | pinned_tweet = self.content.get('pinned_tweet_ids_str', []) 69 | if not pinned_tweet: 70 | return None 71 | if isinstance(pinned_tweet, list): 72 | return pinned_tweet[0] 73 | return pinned_tweet 74 | 75 | @cached_property 76 | def highlighted_tweet_count(self) -> str: 77 | return find_one(self.json_response, 'highlighted_tweets') 78 | 79 | 80 | class ElementBuffer(): 81 | # For handling unstable twitter API results 82 | 83 | def __init__(self, element, change_threshold: int = 2): 84 | self.element = element 85 | self.change_threshold = change_threshold 86 | self.change_count = 0 87 | 88 | def __str__(self): 89 | return str(self.element) 90 | 91 | def __repr__(self): 92 | return str(self.element) 93 | 94 | def push(self, element) -> Union[dict, None]: 95 | if element == self.element: 96 | self.change_count = 0 97 | return None 98 | self.change_count += 1 99 | if self.change_count >= self.change_threshold: 100 | result = {'old': self.element, 'new': element} 101 | self.element = element 102 | self.change_count = 0 103 | return result 104 | return None 105 | 106 | 107 | class ProfileMonitor(MonitorBase): 108 | monitor_type = 'Profile' 109 | 110 | def __init__(self, username: str, title: str, token_config: dict, user_config: dict, cookies_dir: str): 111 | super().__init__(monitor_type=self.monitor_type, 112 | username=username, 113 | title=title, 114 | token_config=token_config, 115 | user_config=user_config, 116 | cookies_dir=cookies_dir) 117 | 118 | json_response = self.get_user() 119 | while not json_response: 120 | time.sleep(60) 121 | json_response = self.get_user() 122 | parser = ProfileParser(json_response) 123 | self.name = ElementBuffer(parser.name) 124 | self.username = ElementBuffer(parser.username) 125 | self.location = ElementBuffer(parser.location) 126 | self.bio = ElementBuffer(parser.bio) 127 | self.website = ElementBuffer(parser.website) 128 | self.followers_count = ElementBuffer(parser.followers_count) 129 | self.following_count = ElementBuffer(parser.following_count) 130 | self.like_count = ElementBuffer(parser.like_count) 131 | self.tweet_count = ElementBuffer(parser.tweet_count, change_threshold=1) 132 | self.profile_image_url = ElementBuffer(parser.profile_image_url) 133 | self.profile_banner_url = ElementBuffer(parser.profile_banner_url) 134 | self.pinned_tweet = ElementBuffer(parser.pinned_tweet) 135 | self.highlighted_tweet_count = ElementBuffer(parser.highlighted_tweet_count) 136 | 137 | self.monitoring_following_count = user_config.get('monitoring_following_count', False) 138 | self.monitoring_tweet_count = user_config.get('monitoring_tweet_count', False) 139 | self.monitoring_like_count = user_config.get('monitoring_like_count', False) 140 | 141 | self.title = title 142 | self.original_username = username 143 | self.sub_monitor_up_to_date = {} 144 | for sub_monitor in SUB_MONITOR_LIST: 145 | self.sub_monitor_up_to_date[sub_monitor.monitor_type] = True 146 | 147 | self.logger.info('Init profile monitor succeed.\n{}'.format(self.__dict__)) 148 | 149 | def get_user(self) -> Union[dict, None]: 150 | params = {'userId': self.user_id} 151 | json_response = self.twitter_watcher.query('UserByRestId', params) 152 | if not find_one(json_response, 'user'): 153 | return None 154 | return json_response 155 | 156 | def detect_change_and_update(self, user: dict): 157 | parser = ProfileParser(user) 158 | 159 | result = self.name.push(parser.name) 160 | if result: 161 | self.send_message(message=MESSAGE_TEMPLATE.format('Name', result['old'], result['new'])) 162 | 163 | result = self.username.push(parser.username) 164 | if result: 165 | self.send_message(message=MESSAGE_TEMPLATE.format('Username', result['old'], result['new'])) 166 | 167 | result = self.location.push(parser.location) 168 | if result: 169 | self.send_message(message=MESSAGE_TEMPLATE.format('Location', result['old'], result['new'])) 170 | 171 | result = self.bio.push(parser.bio) 172 | if result: 173 | self.send_message(message=MESSAGE_TEMPLATE.format('Bio', result['old'], result['new'])) 174 | 175 | result = self.website.push(parser.website) 176 | if result: 177 | self.send_message(message=MESSAGE_TEMPLATE.format('Website', result['old'], result['new'])) 178 | 179 | result = self.followers_count.push(parser.followers_count) 180 | 181 | result = self.following_count.push(parser.following_count) 182 | if result: 183 | if self.monitoring_following_count: 184 | self.send_message(message=MESSAGE_TEMPLATE.format('Following count', result['old'], result['new'])) 185 | else: 186 | self.logger.info(MESSAGE_TEMPLATE.format('Following count', result['old'], result['new'])) 187 | self.sub_monitor_up_to_date[FollowingMonitor.monitor_type] = False 188 | 189 | result = self.like_count.push(parser.like_count) 190 | if result: 191 | if self.monitoring_like_count: 192 | self.send_message(message=MESSAGE_TEMPLATE.format('Like count', result['old'], result['new'])) 193 | else: 194 | self.logger.info(MESSAGE_TEMPLATE.format('Like count', result['old'], result['new'])) 195 | if result['new'] > result['old']: 196 | self.sub_monitor_up_to_date[LikeMonitor.monitor_type] = False 197 | 198 | result = self.tweet_count.push(parser.tweet_count) 199 | if result: 200 | if self.monitoring_tweet_count: 201 | self.send_message(message=MESSAGE_TEMPLATE.format('Tweet count', result['old'], result['new'])) 202 | else: 203 | self.logger.info(MESSAGE_TEMPLATE.format('Tweet count', result['old'], result['new'])) 204 | if result['new'] > result['old']: 205 | self.sub_monitor_up_to_date[TweetMonitor.monitor_type] = False 206 | 207 | result = self.profile_image_url.push(parser.profile_image_url) 208 | if result: 209 | self.send_message(message=MESSAGE_TEMPLATE.format('Profile image', result['old'], result['new']), 210 | photo_url_list=[result['old'], result['new']]) 211 | 212 | result = self.profile_banner_url.push(parser.profile_banner_url) 213 | if result: 214 | self.send_message(message=MESSAGE_TEMPLATE.format('Profile banner', result['old'], result['new']), 215 | photo_url_list=[result['old'], result['new']]) 216 | 217 | result = self.pinned_tweet.push(parser.pinned_tweet) 218 | if result: 219 | self.send_message(message=MESSAGE_TEMPLATE.format('Pinned tweet', result['old'], result['new'])) 220 | 221 | result = self.highlighted_tweet_count.push(parser.highlighted_tweet_count) 222 | if result: 223 | self.send_message(message=MESSAGE_TEMPLATE.format('Highlighted tweet', result['old'], result['new'])) 224 | 225 | def watch_sub_monitor(self): 226 | for sub_monitor in SUB_MONITOR_LIST: 227 | sub_monitor_type = sub_monitor.monitor_type 228 | sub_monitor_instance = MonitorManager.get(monitor_type=sub_monitor_type, username=self.title) 229 | if sub_monitor_instance: 230 | if not self.sub_monitor_up_to_date[sub_monitor_type]: 231 | self.sub_monitor_up_to_date[sub_monitor_type] = MonitorManager.call(monitor_type=sub_monitor_type, 232 | username=self.title) 233 | else: 234 | sub_monitor_instance.update_last_watch_time() 235 | 236 | def watch(self) -> bool: 237 | user = self.get_user() 238 | if not user: 239 | return False 240 | self.detect_change_and_update(user) 241 | self.watch_sub_monitor() 242 | self.update_last_watch_time() 243 | return True 244 | 245 | def status(self) -> str: 246 | return 'Last: {}, username: {}'.format(self.get_last_watch_time(), self.username.element) 247 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | python-telegram-bot==13.15 3 | retry 4 | tzlocal==2.1 5 | bs4 6 | requests 7 | httpx 8 | XClientTransaction -------------------------------------------------------------------------------- /status_tracker.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime, timedelta, timezone 3 | 4 | 5 | class StatusTracker(): 6 | 7 | def __new__(self): 8 | raise Exception('Do not instantiate this class!') 9 | 10 | monitors_status = dict() 11 | notifiers_status = dict() 12 | 13 | logger = logging.getLogger('status') 14 | 15 | @classmethod 16 | def update_monitor_status(cls, monitor_type: str, username: str): 17 | key = '{}-{}'.format(monitor_type, username) 18 | cls.monitors_status[key] = datetime.now(timezone.utc) 19 | 20 | @classmethod 21 | def get_monitor_status(cls, monitor_type: str, username: str): 22 | key = '{}-{}'.format(monitor_type, username) 23 | return cls.monitors_status.get(key, None) 24 | 25 | @classmethod 26 | def set_notifier_status(cls, notifier: str, status: bool): 27 | cls.notifiers_status[notifier] = status 28 | 29 | @classmethod 30 | def check(cls) -> list: 31 | alerts = [] 32 | 33 | monitor_time_threshold = datetime.now(timezone.utc) - timedelta(minutes=30) 34 | for monitor_name, monitor_status in cls.monitors_status.items(): 35 | cls.logger.info('{}: {}'.format(monitor_name, monitor_status)) 36 | if monitor_status < monitor_time_threshold: 37 | alerts.append('{}: {}'.format(monitor_name, monitor_status)) 38 | 39 | for notifier_name, notifier_status in cls.notifiers_status.items(): 40 | cls.logger.info('{}: {}'.format(notifier_name, notifier_status)) 41 | if notifier_status is False: 42 | alerts.append('{}'.format(notifier_name)) 43 | 44 | return alerts 45 | -------------------------------------------------------------------------------- /telegram_notifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import threading 4 | import time 5 | from datetime import datetime, timezone 6 | from typing import List, Union 7 | 8 | import telegram 9 | from retry import retry 10 | from telegram.error import BadRequest, RetryAfter, TimedOut, NetworkError 11 | 12 | from notifier_base import Message, NotifierBase 13 | 14 | 15 | class TelegramMessage(Message): 16 | 17 | def __init__(self, 18 | chat_id_list: List[int], 19 | text: str, 20 | photo_url_list: Union[List[str], None] = None, 21 | video_url_list: Union[List[str], None] = None): 22 | super().__init__(text, photo_url_list, video_url_list) 23 | self.chat_id_list = chat_id_list 24 | 25 | 26 | class TelegramNotifier(NotifierBase): 27 | notifier_name = 'Telegram' 28 | 29 | @classmethod 30 | def init(cls, token: str, logger_name: str): 31 | assert token 32 | cls.bot = telegram.Bot(token=token, request=telegram.utils.request.Request(con_pool_size=2)) 33 | cls.logger = logging.getLogger('{}'.format(logger_name)) 34 | updates = cls._get_updates() 35 | cls.update_offset = updates[-1].update_id + 1 if updates else None 36 | cls.logger.info('Init telegram notifier succeed.') 37 | super().init() 38 | 39 | @classmethod 40 | @retry((RetryAfter, TimedOut, NetworkError), delay=10, tries=10) 41 | def _send_message_to_single_chat(cls, chat_id: str, text: str, photo_url_list: Union[List[str], None], 42 | video_url_list: Union[List[str], None]): 43 | if video_url_list: 44 | cls.bot.send_video(chat_id=chat_id, video=video_url_list[0], caption=text, timeout=60) 45 | elif photo_url_list: 46 | if len(photo_url_list) == 1: 47 | cls.bot.send_photo(chat_id=chat_id, photo=photo_url_list[0], caption=text, timeout=60) 48 | else: 49 | media_group = [telegram.InputMediaPhoto(media=photo_url_list[0], caption=text)] 50 | for photo_url in photo_url_list[1:10]: 51 | media_group.append(telegram.InputMediaPhoto(media=photo_url)) 52 | cls.bot.send_media_group(chat_id=chat_id, media=media_group, timeout=60) 53 | else: 54 | cls.bot.send_message(chat_id=chat_id, text=text, disable_web_page_preview=True, timeout=60) 55 | 56 | @classmethod 57 | def send_message(cls, message: TelegramMessage): 58 | assert cls.initialized 59 | assert isinstance(message, TelegramMessage) 60 | for chat_id in message.chat_id_list: 61 | try: 62 | cls._send_message_to_single_chat(chat_id, message.text, message.photo_url_list, message.video_url_list) 63 | except BadRequest as e: 64 | # Telegram cannot send some photos/videos for unknown reasons. 65 | cls.logger.error('{}, trying to send message without media.'.format(e)) 66 | cls._send_message_to_single_chat(chat_id, message.text, None, None) 67 | 68 | @classmethod 69 | @retry((RetryAfter, TimedOut, NetworkError), delay=60) 70 | def _get_updates(cls, offset=None) -> List[telegram.Update]: 71 | return cls.bot.get_updates(offset=offset) 72 | 73 | @classmethod 74 | def _get_new_updates(cls) -> List[telegram.Update]: 75 | updates = cls._get_updates(offset=cls.update_offset) 76 | if updates: 77 | cls.update_offset = updates[-1].update_id + 1 78 | return updates 79 | 80 | @staticmethod 81 | def _get_new_update_offset(updates: List[telegram.Update]) -> Union[int, None]: 82 | if not updates: 83 | return None 84 | return updates[-1].update_id + 1 85 | 86 | @classmethod 87 | def confirm(cls, message: TelegramMessage) -> bool: 88 | assert cls.initialized 89 | assert isinstance(message, TelegramMessage) 90 | message.text = '{}\nPlease reply Y/N'.format(message.text) 91 | cls.put_message_into_queue(message) 92 | sending_time = datetime.now(timezone.utc) 93 | while True: 94 | for update in cls._get_new_updates(): 95 | received_message = update.message 96 | if received_message.date < sending_time: 97 | continue 98 | if received_message.chat.id not in message.chat_id_list: 99 | continue 100 | text = received_message.text.upper() 101 | if text == 'Y': 102 | return True 103 | if text == 'N': 104 | return False 105 | time.sleep(10) 106 | 107 | @classmethod 108 | def listen_exit_command(cls, chat_id: str): 109 | 110 | def _listen_exit_command(): 111 | starting_time = datetime.now(timezone.utc) 112 | while True: 113 | for update in cls._get_new_updates(): 114 | received_message = update.message 115 | if received_message.date < starting_time: 116 | continue 117 | if received_message.chat.id != chat_id: 118 | continue 119 | text = received_message.text.upper() 120 | if text == 'EXIT': 121 | if cls.confirm(TelegramMessage([chat_id], 'Do you want to exit the program?')): 122 | cls.put_message_into_queue(TelegramMessage([chat_id], 'Program will exit after 5 sec.')) 123 | cls.logger.error('The program exits by the telegram command') 124 | time.sleep(5) 125 | os._exit(0) 126 | time.sleep(20) 127 | 128 | threading.Thread(target=_listen_exit_command, daemon=True).start() 129 | 130 | 131 | def send_alert(token: str, chat_id: int, message: str): 132 | # The telegram notifier may also be wrong, so initialize the telegram bot separately. 133 | bot = telegram.Bot(token=token) 134 | bot.send_message(chat_id=chat_id, text=message, timeout=60) 135 | -------------------------------------------------------------------------------- /tweet_monitor.py: -------------------------------------------------------------------------------- 1 | import time 2 | from datetime import datetime, timedelta, timezone 3 | 4 | from monitor_base import MonitorBase 5 | from utils import parse_media_from_tweet, parse_text_from_tweet, parse_create_time_from_tweet, find_all, find_one, get_content, convert_html_to_text 6 | 7 | 8 | def _verify_tweet_user_id(tweet: dict, user_id: str) -> bool: 9 | user = find_one(tweet, 'user_results') 10 | return find_one(user, 'rest_id') == user_id 11 | 12 | 13 | class TweetMonitor(MonitorBase): 14 | monitor_type = 'Tweet' 15 | 16 | def __init__(self, username: str, title: str, token_config: dict, user_config: dict, cookies_dir: str): 17 | super().__init__(monitor_type=self.monitor_type, 18 | username=username, 19 | title=title, 20 | token_config=token_config, 21 | user_config=user_config, 22 | cookies_dir=cookies_dir) 23 | 24 | tweet_list = self.get_tweet_list() 25 | while tweet_list is None: 26 | time.sleep(60) 27 | tweet_list = self.get_tweet_list() 28 | 29 | self.last_tweet_id = -1 30 | for tweet in tweet_list: 31 | if _verify_tweet_user_id(tweet, self.user_id): 32 | self.last_tweet_id = max(self.last_tweet_id, int(find_one(tweet, 'rest_id'))) 33 | 34 | self.logger.info('Init tweet monitor succeed.\nUser id: {}\nLast tweet: {}'.format( 35 | self.user_id, self.last_tweet_id)) 36 | 37 | def get_tweet_list(self) -> dict: 38 | api_name = 'UserTweetsAndReplies' 39 | params = {'userId': self.user_id, 'includePromotedContent': True, 'withVoice': True, 'count': 1000} 40 | json_response = self.twitter_watcher.query(api_name, params) 41 | if json_response is None: 42 | return None 43 | return find_all(json_response, 'tweet_results') 44 | 45 | def watch(self) -> bool: 46 | tweet_list = self.get_tweet_list() 47 | if tweet_list is None: 48 | return False 49 | 50 | max_tweet_id = -1 51 | new_tweet_list = [] 52 | time_threshold = datetime.now(timezone.utc) - timedelta(minutes=5) 53 | for tweet in tweet_list: 54 | if not _verify_tweet_user_id(tweet, self.user_id): 55 | continue 56 | tweet_id = int(find_one(tweet, 'rest_id')) 57 | if tweet_id <= self.last_tweet_id: 58 | continue 59 | if parse_create_time_from_tweet(tweet) < time_threshold: 60 | continue 61 | 62 | new_tweet_list.append(tweet) 63 | max_tweet_id = max(max_tweet_id, tweet_id) 64 | 65 | self.last_tweet_id = max(self.last_tweet_id, max_tweet_id) 66 | 67 | for tweet in reversed(new_tweet_list): 68 | text = parse_text_from_tweet(tweet) 69 | retweet = find_one(tweet, 'retweeted_status_result') 70 | quote = find_one(tweet, 'quoted_status_result') 71 | if retweet: 72 | photo_url_list, video_url_list = parse_media_from_tweet(retweet) 73 | else: 74 | photo_url_list, video_url_list = parse_media_from_tweet(tweet) 75 | if quote: 76 | quote_text = get_content(quote).get('full_text', '') 77 | quote_user = find_one(quote, 'user_results') 78 | quote_username = get_content(quote_user).get('screen_name', '') 79 | text += '\n\nQuote: @{}: {}'.format(quote_username, quote_text) 80 | source = find_one(tweet, 'source') 81 | text += '\n\nSource: {}'.format(convert_html_to_text(source)) 82 | tweet_id = find_one(tweet, 'rest_id') 83 | tweet_link = "https://x.com/{}/status/{}".format(self.user_id, tweet_id) 84 | text += f"\nLink: {tweet_link}" 85 | self.send_message(text, photo_url_list, video_url_list) 86 | 87 | self.update_last_watch_time() 88 | return True 89 | 90 | def status(self) -> str: 91 | return 'Last: {}, id: {}'.format(self.get_last_watch_time(), self.last_tweet_id) 92 | -------------------------------------------------------------------------------- /twitter_watcher.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import random 5 | import time 6 | from typing import List, Union 7 | 8 | import requests 9 | 10 | from graphql_api import GraphqlAPI 11 | from utils import find_one 12 | 13 | 14 | def _get_auth_headers(headers, cookies) -> dict: 15 | 16 | authed_headers = headers | { 17 | 'cookie': '; '.join(f'{k}={v}' for k, v in cookies.items()), 18 | 'referer': 'https://twitter.com/', 19 | 'x-csrf-token': cookies.get('ct0', ''), 20 | 'x-guest-token': cookies.get('guest_token', ''), 21 | 'x-twitter-auth-type': 'OAuth2Session' if cookies.get('auth_token') else '', 22 | 'x-twitter-active-user': 'yes', 23 | 'x-twitter-client-language': 'en', 24 | } 25 | return dict(sorted({k.lower(): v for k, v in authed_headers.items()}.items())) 26 | 27 | 28 | def _build_params(params: dict) -> dict: 29 | return {k: json.dumps(v) for k, v in params.items()} 30 | 31 | 32 | class TwitterWatcher: 33 | 34 | def __init__(self, auth_username_list: List[str], cookies_dir: str): 35 | assert auth_username_list 36 | self.token_number = len(auth_username_list) 37 | self.auth_cookie_list = [] 38 | for auth_username in auth_username_list: 39 | auth_cookie_file = os.path.join(cookies_dir, '{}.json'.format(auth_username)) 40 | with open(auth_cookie_file, 'r') as f: 41 | self.auth_cookie_list.append(json.load(f)) 42 | self.auth_cookie_list[-1]['username'] = auth_username 43 | self.current_token_index = random.randrange(self.token_number) 44 | self.logger = logging.getLogger('api') 45 | 46 | def query(self, api_name: str, params: dict) -> Union[dict, list, None]: 47 | url, method, headers, features = GraphqlAPI.get_api_data(api_name) 48 | params = _build_params({"variables": params, "features": features}) 49 | for _ in range(self.token_number): 50 | self.current_token_index = (self.current_token_index + 1) % self.token_number 51 | auth_headers = _get_auth_headers(headers, self.auth_cookie_list[self.current_token_index]) 52 | try: 53 | response = requests.request(method=method, url=url, headers=auth_headers, params=params, timeout=300) 54 | except requests.exceptions.ConnectionError as e: 55 | self.logger.error('{} request error: {}, try next token.'.format(url, e)) 56 | continue 57 | if response.status_code in [200, 404, 403]: 58 | # 404 NOT_FOUND 59 | # 403 CURRENT_USER_SUSPENDED 60 | if not response.text: 61 | self.logger.error('{} response empty {}, try next token.'.format(url, response.status_code)) 62 | continue 63 | json_response = response.json() 64 | if 'errors' in json_response: 65 | self.logger.error('{} request error: {} {}, try next token.'.format( 66 | url, response.status_code, json_response['errors'])) 67 | continue 68 | return json_response 69 | if response.status_code != 429: 70 | # 429 TWEET_RATE_LIMIT_EXCEEDED 71 | self.logger.error('{} request returned an error: {} {}, try next token.'.format( 72 | url, response.status_code, response.text)) 73 | continue 74 | self.logger.error('All tokens are unavailable, query fails: {}\n{}\n{}'.format( 75 | url, json.dumps(auth_headers, indent=2), json.dumps(params, indent=2))) 76 | return None 77 | 78 | def get_user_by_username(self, username: str, params: dict = {}) -> dict: 79 | api_name = 'UserByScreenName' 80 | params['screen_name'] = username 81 | json_response = self.query(api_name, params) 82 | while json_response is None: 83 | time.sleep(60) 84 | json_response = self.query(api_name, params) 85 | return json_response 86 | 87 | def get_user_by_id(self, id: int, params: dict = {}) -> dict: 88 | api_name = 'UserByRestId' 89 | params['userId'] = id 90 | json_response = self.query(api_name, params) 91 | while json_response is None: 92 | time.sleep(60) 93 | json_response = self.query(api_name, params) 94 | return json_response 95 | 96 | def get_id_by_username(self, username: str): 97 | json_response = self.get_user_by_username(username, {}) 98 | return find_one(json_response, 'rest_id') 99 | 100 | def check_tokens(self, test_username: str = 'X', output_response: bool = False): 101 | result = dict() 102 | for auth_cookie in self.auth_cookie_list: 103 | try: 104 | url, method, headers, features = GraphqlAPI.get_api_data('UserByScreenName') 105 | params = _build_params({"variables": {'screen_name': test_username}, "features": features}) 106 | auth_headers = _get_auth_headers(headers, auth_cookie) 107 | response = requests.request(method=method, url=url, headers=auth_headers, params=params, timeout=300) 108 | except requests.exceptions.ConnectionError as e: 109 | result[auth_cookie['username']] = False 110 | print(e) 111 | continue 112 | result[auth_cookie['username']] = (response.status_code == 200) 113 | if output_response: 114 | print(json.dumps(response.json(), indent=2)) 115 | return result 116 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | from datetime import datetime, timezone 3 | from typing import Tuple 4 | 5 | from bs4 import BeautifulSoup 6 | 7 | 8 | def convert_html_to_text(html: str) -> str: 9 | bs = BeautifulSoup(html, "html.parser") 10 | return bs.get_text() 11 | 12 | 13 | def get_photo_url_from_media(media: dict) -> str: 14 | return media.get('media_url_https', '') 15 | 16 | 17 | def get_video_url_from_media(media: dict) -> str: 18 | video_info = media.get('video_info', {}) 19 | variants = video_info.get('variants', []) 20 | max_bitrate = -1 21 | video_url = '' 22 | for variant in variants: 23 | bitrate = variant.get('bitrate', 0) 24 | if bitrate > max_bitrate: 25 | max_bitrate = bitrate 26 | video_url = variant.get('url', '') 27 | return video_url 28 | 29 | 30 | def parse_media_from_tweet(tweet: dict) -> Tuple[list, list]: 31 | photo_url_list = [] 32 | video_url_list = [] 33 | tweet_content = get_content(tweet) 34 | medias = tweet_content.get('extended_entities', {}).get('media', []) 35 | for media in medias: 36 | media_type = media.get('type', '') 37 | if media_type == 'photo': 38 | photo_url_list.append(get_photo_url_from_media(media)) 39 | elif media_type in ['video', 'animated_gif']: 40 | video_url_list.append(get_video_url_from_media(media)) 41 | return photo_url_list, video_url_list 42 | 43 | 44 | def parse_text_from_tweet(tweet: dict) -> str: 45 | tweet_content = get_content(tweet) 46 | return convert_html_to_text(tweet_content.get('full_text', '')) 47 | 48 | 49 | def parse_username_from_tweet(tweet: dict) -> str: 50 | user = find_one(tweet, 'user_results') 51 | return find_one(user, 'rest_id') 52 | 53 | 54 | def parse_create_time_from_tweet(tweet: dict) -> datetime.time: 55 | created_at = find_one(get_content(tweet), 'created_at') 56 | if not created_at: 57 | return datetime.fromtimestamp(0).replace(tzinfo=timezone.utc) 58 | return datetime.strptime(created_at, '%a %b %d %H:%M:%S %z %Y') 59 | 60 | 61 | def find_all(obj: any, key: str) -> list: 62 | # DFS 63 | def dfs(obj: any, key: str, res: list) -> list: 64 | if not obj: 65 | return res 66 | if isinstance(obj, list): 67 | for e in obj: 68 | res.extend(dfs(e, key, [])) 69 | return res 70 | if isinstance(obj, dict): 71 | if key in obj: 72 | res.append(obj[key]) 73 | for v in obj.values(): 74 | res.extend(dfs(v, key, [])) 75 | return res 76 | 77 | return dfs(obj, key, []) 78 | 79 | 80 | def find_one(obj: any, key: str) -> any: 81 | # BFS 82 | que = deque([obj]) 83 | while len(que): 84 | obj = que.popleft() 85 | if isinstance(obj, list): 86 | que.extend(obj) 87 | if isinstance(obj, dict): 88 | if key in obj: 89 | return obj[key] 90 | for v in obj.values(): 91 | que.append(v) 92 | return None 93 | 94 | 95 | def get_content(obj: dict) -> dict: 96 | return find_one(obj, 'legacy') 97 | 98 | 99 | def get_cursor(obj: any) -> str: 100 | entries = find_one(obj, 'entries') 101 | for entry in entries: 102 | entry_id = entry.get('entryId', '') 103 | if entry_id.startswith('cursor-bottom'): 104 | return entry.get('content', {}).get('value', '') 105 | 106 | 107 | def check_initialized(cls_method): 108 | 109 | def wrapper(cls, *args, **kwargs): 110 | if cls.initialized: 111 | return cls_method(cls, *args, **kwargs) 112 | else: 113 | raise RuntimeError('Class has not initialized!') 114 | 115 | return wrapper 116 | --------------------------------------------------------------------------------