vktgbot
2 |
3 | 
4 | 
6 |
Telegram bot for automatic forwarding posts from VK to Telegram.
9 | 10 |
11 |
12 | 
13 |
14 |
73 | Original Creator - [alcortazzo](https://github.com/alcortazzo) 74 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | app: 5 | build: ../ 6 | volumes: 7 | - ../logs:/code/logs 8 | - ../last_id.txt:/code/last_id.txt 9 | -------------------------------------------------------------------------------- /images/code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alcortazzo/vktgbot/e8308d4ead1fcaeaf5cbb385697e97bed2abd801/images/code.png -------------------------------------------------------------------------------- /last_id.txt: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiogram<=2.25.2 2 | requests 3 | loguru 4 | python-dotenv 5 | -------------------------------------------------------------------------------- /vktgbot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alcortazzo/vktgbot/e8308d4ead1fcaeaf5cbb385697e97bed2abd801/vktgbot/__init__.py -------------------------------------------------------------------------------- /vktgbot/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Telegram Bot for automated reposting from VKontakte community pages 3 | to Telegram channels. 4 | 5 | v3.1 6 | by @alcortazzo 7 | """ 8 | 9 | import time 10 | 11 | from loguru import logger 12 | 13 | from config import SINGLE_START, TIME_TO_SLEEP 14 | from start_script import start_script 15 | from tools import prepare_temp_folder 16 | 17 | logger.add( 18 | "./logs/debug.log", 19 | format="{time} {level} {message}", 20 | level="DEBUG", 21 | rotation="1 week", 22 | compression="zip", 23 | ) 24 | 25 | logger.info("Script is started.") 26 | 27 | 28 | @logger.catch 29 | def main(): 30 | start_script() 31 | prepare_temp_folder() 32 | 33 | 34 | while True: 35 | try: 36 | main() 37 | if SINGLE_START: 38 | logger.info("Script has successfully completed its execution") 39 | exit() 40 | else: 41 | logger.info(f"Script went to sleep for {TIME_TO_SLEEP} seconds.") 42 | time.sleep(TIME_TO_SLEEP) 43 | except KeyboardInterrupt: 44 | logger.info("Script is stopped by the user.") 45 | exit() 46 | -------------------------------------------------------------------------------- /vktgbot/api_requests.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import re 4 | import requests 5 | from loguru import logger 6 | 7 | 8 | def get_data_from_vk( 9 | vk_token: str, req_version: float, vk_domain: str, req_filter: str, req_count: int 10 | ) -> Union[dict, None]: 11 | logger.info("Trying to get posts from VK.") 12 | 13 | match = re.search("^(club|public)(\d+)$", vk_domain) 14 | if match: 15 | source_param = {"owner_id": "-" + match.groups()[1]} 16 | else: 17 | source_param = {"domain": vk_domain} 18 | 19 | response = requests.get( 20 | "https://api.vk.com/method/wall.get", 21 | params=dict( 22 | { 23 | "access_token": vk_token, 24 | "v": req_version, 25 | "filter": req_filter, 26 | "count": req_count, 27 | }, 28 | **source_param, 29 | ), 30 | ) 31 | data = response.json() 32 | if "response" in data: 33 | return data["response"]["items"] 34 | elif "error" in data: 35 | logger.error("Error was detected when requesting data from VK: " f"{data['error']['error_msg']}") 36 | return None 37 | 38 | 39 | def get_video_url(vk_token: str, req_version: float, owner_id: str, video_id: str, access_key: str) -> str: 40 | response = requests.get( 41 | "https://api.vk.com/method/video.get", 42 | params={ 43 | "access_token": vk_token, 44 | "v": req_version, 45 | "videos": f"{owner_id}_{video_id}{'' if not access_key else f'_{access_key}'}", 46 | }, 47 | ) 48 | data = response.json() 49 | if "response" in data and data["response"]["items"]: 50 | return data["response"]["items"][0]["files"].get("external", "") 51 | elif "error" in data: 52 | logger.error(f"Error was detected when requesting data from VK: {data['error']['error_msg']}") 53 | return "" 54 | 55 | 56 | def get_group_name(vk_token: str, req_version: float, owner_id) -> str: 57 | response = requests.get( 58 | "https://api.vk.com/method/groups.getById", 59 | params={ 60 | "access_token": vk_token, 61 | "v": req_version, 62 | "group_id": owner_id, 63 | }, 64 | ) 65 | data = response.json() 66 | if "response" in data: 67 | return data["response"][0]["name"] 68 | elif "error" in data: 69 | logger.error(f"Error was detected when requesting data from VK: {data['error']['error_msg']}") 70 | return "" 71 | -------------------------------------------------------------------------------- /vktgbot/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import dotenv 5 | 6 | dotenv.load_dotenv() 7 | 8 | 9 | TG_CHANNEL: str = os.getenv("VAR_TG_CHANNEL", "") 10 | TG_BOT_TOKEN: str = os.getenv("VAR_TG_BOT_TOKEN", "") 11 | VK_TOKEN: str = os.getenv("VAR_VK_TOKEN", "") 12 | VK_DOMAIN: str = os.getenv("VAR_VK_DOMAIN", "") 13 | 14 | REQ_VERSION: float = float(os.getenv("VAR_REQ_VERSION", 5.103)) 15 | REQ_COUNT: int = int(os.getenv("VAR_REQ_COUNT", 3)) 16 | REQ_FILTER: str = os.getenv("VAR_REQ_FILTER", "owner") 17 | 18 | SINGLE_START: bool = os.getenv("VAR_SINGLE_START", "").lower() in ("true",) 19 | TIME_TO_SLEEP: int = int(os.getenv("VAR_TIME_TO_SLEEP", 120)) 20 | SKIP_ADS_POSTS: bool = os.getenv("VAR_SKIP_ADS_POSTS", "").lower() in ("true",) 21 | SKIP_COPYRIGHTED_POST: bool = os.getenv("VAR_SKIP_COPYRIGHTED_POST", "").lower() in ("true") 22 | SKIP_REPOSTS: bool = os.getenv("VAR_SKIP_REPOSTS", "").lower() in ("true") 23 | 24 | WHITELIST: list = json.loads(os.getenv("VAR_WHITELIST", "[]")) 25 | BLACKLIST: list = json.loads(os.getenv("VAR_BLACKLIST", "[]")) 26 | -------------------------------------------------------------------------------- /vktgbot/last_id.py: -------------------------------------------------------------------------------- 1 | from loguru import logger 2 | 3 | 4 | def read_id() -> int: 5 | try: 6 | return int(open("./last_id.txt", "r").read()) 7 | except ValueError: 8 | logger.critical( 9 | "The value of the last identifier is incorrect. Please check the contents of the file 'last_id.txt'." 10 | ) 11 | exit() 12 | 13 | 14 | def write_id(new_id: int) -> None: 15 | open("./last_id.txt", "w").write(str(new_id)) 16 | logger.info(f"New ID, written in the file: {new_id}") 17 | -------------------------------------------------------------------------------- /vktgbot/parse_posts.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Union 3 | 4 | import requests 5 | from loguru import logger 6 | 7 | from api_requests import get_video_url 8 | from config import REQ_VERSION, VK_TOKEN 9 | from tools import add_urls_to_text, prepare_text_for_html, prepare_text_for_reposts, reformat_vk_links 10 | 11 | 12 | def parse_post(item: dict, repost_exists: bool, item_type: str, group_name: str) -> dict: 13 | text = prepare_text_for_html(item["text"]) 14 | if repost_exists: 15 | text = prepare_text_for_reposts(text, item, item_type, group_name) 16 | 17 | text = reformat_vk_links(text) 18 | 19 | urls: list = [] 20 | videos: list = [] 21 | photos: list = [] 22 | docs: list = [] 23 | 24 | if "attachments" in item: 25 | parse_attachments(item["attachments"], text, urls, videos, photos, docs) 26 | 27 | text = add_urls_to_text(text, urls, videos) 28 | logger.info(f"{item_type.capitalize()} parsing is complete.") 29 | return {"text": text, "photos": photos, "docs": docs} 30 | 31 | 32 | def parse_attachments(attachments, text, urls, videos, photos, docs): 33 | for attachment in attachments: 34 | if attachment["type"] == "link": 35 | url = get_url(attachment, text) 36 | if url: 37 | urls.append(url) 38 | elif attachment["type"] == "video": 39 | video = get_video(attachment) 40 | if video: 41 | videos.append(video) 42 | elif attachment["type"] == "photo": 43 | photo = get_photo(attachment) 44 | if photo: 45 | photos.append(photo) 46 | elif attachment["type"] == "doc": 47 | doc = get_doc(attachment["doc"]) 48 | if doc: 49 | docs.append(doc) 50 | 51 | 52 | def get_url(attachment: dict, text: str) -> Union[str, None]: 53 | url = attachment["link"]["url"] 54 | return url if url not in text else None 55 | 56 | 57 | def get_video(attachment: dict) -> str: 58 | owner_id = attachment["video"]["owner_id"] 59 | video_id = attachment["video"]["id"] 60 | video_type = attachment["video"]["type"] 61 | access_key = attachment["video"].get("access_key", "") 62 | 63 | video = get_video_url(VK_TOKEN, REQ_VERSION, owner_id, video_id, access_key) 64 | if video: 65 | return video 66 | elif video_type == "short_video": 67 | return f"https://vk.com/clip{owner_id}_{video_id}" 68 | else: 69 | return f"https://vk.com/video{owner_id}_{video_id}" 70 | 71 | 72 | def get_photo(attachment: dict) -> Union[str, None]: 73 | sizes = attachment["photo"]["sizes"] 74 | types = ["w", "z", "y", "x", "r", "q", "p", "o", "m", "s"] 75 | 76 | for type_ in types: 77 | if next( 78 | (item for item in sizes if item["type"] == type_), 79 | False, 80 | ): 81 | return re.sub( 82 | "&([a-zA-Z]+(_[a-zA-Z]+)+)=([a-zA-Z0-9-_]+)", 83 | "", 84 | next( 85 | (item for item in sizes if item["type"] == type_), 86 | False, 87 | )["url"], 88 | ) 89 | else: 90 | return None 91 | 92 | 93 | def get_doc(doc: dict) -> Union[dict, None]: 94 | if doc["size"] > 50000000: 95 | logger.info(f"The document was skipped due to its size exceeding the 50MB limit: {doc['size']=}.") 96 | return None 97 | else: 98 | response = requests.get(doc["url"]) 99 | 100 | with open(f'./temp/{doc["title"]}', "wb") as file: 101 | file.write(response.content) 102 | 103 | return {"title": doc["title"], "url": doc["url"]} 104 | -------------------------------------------------------------------------------- /vktgbot/send_posts.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from aiogram import Bot, types 4 | from aiogram.utils import exceptions 5 | from loguru import logger 6 | 7 | from tools import split_text 8 | 9 | 10 | async def send_post(bot: Bot, tg_channel: str, text: str, photos: list, docs: list, num_tries: int = 0) -> None: 11 | num_tries += 1 12 | if num_tries > 3: 13 | logger.error("Post was not sent to Telegram. Too many tries.") 14 | return 15 | try: 16 | if len(photos) == 0: 17 | await send_text_post(bot, tg_channel, text) 18 | elif len(photos) == 1: 19 | await send_photo_post(bot, tg_channel, text, photos) 20 | elif len(photos) >= 2: 21 | await send_photos_post(bot, tg_channel, text, photos) 22 | if docs: 23 | await send_docs_post(bot, tg_channel, docs) 24 | except exceptions.RetryAfter as ex: 25 | logger.warning(f"Flood limit is exceeded. Sleep {ex.timeout} seconds. Try: {num_tries}") 26 | await asyncio.sleep(ex.timeout) 27 | await send_post(bot, tg_channel, text, photos, docs, num_tries) 28 | except exceptions.BadRequest as ex: 29 | logger.warning(f"Bad request. Wait 60 seconds. Try: {num_tries}. {ex}") 30 | await asyncio.sleep(60) 31 | await send_post(bot, tg_channel, text, photos, docs, num_tries) 32 | 33 | 34 | async def send_text_post(bot: Bot, tg_channel: str, text: str) -> None: 35 | if not text: 36 | return 37 | 38 | if len(text) < 4096: 39 | await bot.send_message(tg_channel, text, parse_mode=types.ParseMode.HTML) 40 | else: 41 | text_parts = split_text(text, 4084) 42 | prepared_text_parts = ( 43 | [text_parts[0] + " (...)"] 44 | + ["(...) " + part + " (...)" for part in text_parts[1:-1]] 45 | + ["(...) " + text_parts[-1]] 46 | ) 47 | 48 | for part in prepared_text_parts: 49 | await bot.send_message(tg_channel, part, parse_mode=types.ParseMode.HTML) 50 | await asyncio.sleep(0.5) 51 | logger.info("Text post sent to Telegram.") 52 | 53 | 54 | async def send_photo_post(bot: Bot, tg_channel: str, text: str, photos: list) -> None: 55 | if len(text) <= 1024: 56 | await bot.send_photo(tg_channel, photos[0], text, parse_mode=types.ParseMode.HTML) 57 | logger.info("Text post (<=1024) with photo sent to Telegram.") 58 | else: 59 | prepared_text = f' {text}' 60 | if len(prepared_text) <= 4096: 61 | await bot.send_message(tg_channel, prepared_text, parse_mode=types.ParseMode.HTML) 62 | else: 63 | await send_text_post(bot, tg_channel, text) 64 | await bot.send_photo(tg_channel, photos[0]) 65 | logger.info("Text post (>1024) with photo sent to Telegram.") 66 | 67 | 68 | async def send_photos_post(bot: Bot, tg_channel: str, text: str, photos: list) -> None: 69 | media = types.MediaGroup() 70 | for photo in photos: 71 | media.attach_photo(types.InputMediaPhoto(photo)) 72 | 73 | if (len(text) > 0) and (len(text) <= 1024): 74 | media.media[0].caption = text 75 | media.media[0].parse_mode = types.ParseMode.HTML 76 | elif len(text) > 1024: 77 | await send_text_post(bot, tg_channel, text) 78 | await bot.send_media_group(tg_channel, media) 79 | logger.info("Text post with photos sent to Telegram.") 80 | 81 | 82 | async def send_docs_post(bot: Bot, tg_channel: str, docs: list) -> None: 83 | media = types.MediaGroup() 84 | for doc in docs: 85 | media.attach_document(types.InputMediaDocument(open(f"./temp/{doc['title']}", "rb"))) 86 | await bot.send_media_group(tg_channel, media) 87 | logger.info("Documents sent to Telegram.") 88 | -------------------------------------------------------------------------------- /vktgbot/start_script.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from aiogram import Bot, Dispatcher 4 | from aiogram.utils import executor 5 | from loguru import logger 6 | 7 | import config 8 | from api_requests import get_data_from_vk, get_group_name 9 | from last_id import read_id, write_id 10 | from parse_posts import parse_post 11 | from send_posts import send_post 12 | from tools import blacklist_check, prepare_temp_folder, whitelist_check 13 | 14 | 15 | def start_script(): 16 | bot = Bot(token=config.TG_BOT_TOKEN) 17 | dp = Dispatcher(bot) 18 | 19 | last_known_id = read_id() 20 | logger.info(f"Last known ID: {last_known_id}") 21 | 22 | items: Union[dict, None] = get_data_from_vk( 23 | config.VK_TOKEN, 24 | config.REQ_VERSION, 25 | config.VK_DOMAIN, 26 | config.REQ_FILTER, 27 | config.REQ_COUNT, 28 | ) 29 | if not items: 30 | return 31 | 32 | if "is_pinned" in items[0]: 33 | items = items[1:] 34 | logger.info(f"Got a few posts with IDs: {items[-1]['id']} - {items[0]['id']}.") 35 | 36 | new_last_id: int = items[0]["id"] 37 | 38 | if new_last_id > last_known_id: 39 | for item in items[::-1]: 40 | item: dict 41 | if item["id"] <= last_known_id: 42 | continue 43 | logger.info(f"Working with post with ID: {item['id']}.") 44 | if blacklist_check(config.BLACKLIST, item["text"]): 45 | continue 46 | if whitelist_check(config.WHITELIST, item["text"]): 47 | continue 48 | if config.SKIP_ADS_POSTS and item["marked_as_ads"]: 49 | logger.info("Post was skipped as an advertisement.") 50 | continue 51 | if config.SKIP_COPYRIGHTED_POST and "copyright" in item: 52 | logger.info("Post was skipped as an copyrighted post.") 53 | continue 54 | 55 | item_parts = {"post": item} 56 | group_name = "" 57 | if "copy_history" in item and not config.SKIP_REPOSTS: 58 | item_parts["repost"] = item["copy_history"][0] 59 | group_name = get_group_name( 60 | config.VK_TOKEN, 61 | config.REQ_VERSION, 62 | abs(item_parts["repost"]["owner_id"]), 63 | ) 64 | logger.info("Detected repost in the post.") 65 | 66 | for item_part in item_parts: 67 | prepare_temp_folder() 68 | repost_exists: bool = True if len(item_parts) > 1 else False 69 | 70 | logger.info(f"Starting parsing of the {item_part}") 71 | parsed_post = parse_post(item_parts[item_part], repost_exists, item_part, group_name) 72 | logger.info(f"Starting sending of the {item_part}") 73 | executor.start( 74 | dp, 75 | send_post( 76 | bot, 77 | config.TG_CHANNEL, 78 | parsed_post["text"], 79 | parsed_post["photos"], 80 | parsed_post["docs"], 81 | ), 82 | ) 83 | 84 | write_id(new_last_id) 85 | -------------------------------------------------------------------------------- /vktgbot/tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from loguru import logger 5 | 6 | 7 | def blacklist_check(blacklist: list, text: str) -> bool: 8 | if blacklist: 9 | text_lower = text.lower() 10 | for black_word in blacklist: 11 | if black_word.lower() in text_lower: 12 | logger.info(f"Post was skipped due to the detection of blacklisted word: {black_word}.") 13 | return True 14 | 15 | return False 16 | 17 | 18 | def whitelist_check(whitelist: list, text: str) -> bool: 19 | if whitelist: 20 | text_lower = text.lower() 21 | for white_word in whitelist: 22 | if white_word.lower() in text_lower: 23 | return False 24 | logger.info("The post was skipped because no whitelist words were found.") 25 | return True 26 | 27 | return False 28 | 29 | 30 | def prepare_temp_folder(): 31 | if "temp" in os.listdir(): 32 | for root, dirs, files in os.walk("temp"): 33 | for file in files: 34 | os.remove(os.path.join(root, file)) 35 | else: 36 | os.mkdir("temp") 37 | 38 | 39 | def prepare_text_for_reposts(text: str, item: dict, item_type: str, group_name: str) -> str: 40 | if item_type == "post" and text: 41 | from_id = item["copy_history"][0]["from_id"] 42 | id = item["copy_history"][0]["id"] 43 | link_to_repost = f"https://vk.com/wall{from_id}_{id}" 44 | text = f'{text}\n\nREPOST ↓ {group_name}' 45 | if item_type == "repost": 46 | from_id = item["from_id"] 47 | id = item["id"] 48 | link_to_repost = f"https://vk.com/wall{from_id}_{id}" 49 | text = f'REPOST ↓ {group_name}\n\n{text}' 50 | 51 | return text 52 | 53 | 54 | def prepare_text_for_html(text: str) -> str: 55 | return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) 56 | 57 | 58 | def add_urls_to_text(text: str, urls: list, videos: list) -> str: 59 | first_link = True 60 | urls = videos + urls 61 | 62 | if not urls: 63 | return text 64 | 65 | for url in urls: 66 | if url not in text: 67 | if first_link: 68 | text = f' {text}\n\n{url}' if text else url 69 | first_link = False 70 | else: 71 | text += f"\n{url}" 72 | return text 73 | 74 | 75 | def split_text(text: str, fragment_size: int) -> list: 76 | fragments = [] 77 | for fragment in range(0, len(text), fragment_size): 78 | fragments.append(text[fragment : fragment + fragment_size]) 79 | return fragments 80 | 81 | 82 | def reformat_vk_links(text: str) -> str: 83 | match = re.search("\[([\w.]+?)\|(.+?)\]", text) 84 | while match: 85 | left_text = text[: match.span()[0]] 86 | right_text = text[match.span()[1] :] 87 | matching_text = text[match.span()[0] : match.span()[1]] 88 | 89 | link_domain, link_text = re.findall("\[(.+?)\|(.+?)\]", matching_text)[0] 90 | text = left_text + f"""{link_text}""" + right_text 91 | match = re.search("\[([\w.]+?)\|(.+?)\]", text) 92 | 93 | return text 94 | --------------------------------------------------------------------------------