├── tests ├── __init__.py ├── test_manage_worker.py └── test_repost_worker.py ├── Procfile ├── runtime.txt ├── pytest.ini ├── alembic ├── README ├── script.py.mako ├── versions │ ├── 2e8d45ad3ac3_.py │ ├── 215a632cdf23_.py │ ├── f5f69376d382_.py │ └── 97bba2b7506c_.py └── env.py ├── scripts └── migrate ├── app.json ├── vk_channelify ├── __init__.py ├── models │ ├── channel.py │ ├── __init__.py │ ├── disabled_channel.py │ └── time_stamp_mixin.py ├── vk_errors.py ├── metrics.py ├── repost_worker.py └── manage_worker.py ├── requirements.txt ├── Dockerfile ├── README.md ├── docs └── index.md ├── LICENSE ├── app.py ├── alembic.ini └── .gitignore /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: python app.py -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.11.11 -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | -------------------------------------------------------------------------------- /alembic/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /scripts/migrate: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # CWD: vk-channelify repo 4 | 5 | alembic upgrade head 6 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vk-channelify", 3 | "scripts": { 4 | "dokku": { 5 | "predeploy": "PYTHONPATH=. alembic upgrade head" 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /vk_channelify/__init__.py: -------------------------------------------------------------------------------- 1 | from . import models, metrics 2 | from .manage_worker import run_worker as run_manage_worker 3 | from .repost_worker import run_worker as run_repost_worker 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | SQLAlchemy==1.4.54 2 | alembic==1.13.3 3 | psycopg2-binary==2.9.11 4 | requests==2.31.0 5 | python-telegram-bot==13.15 6 | prometheus-client==0.21.1 7 | pytest==8.3.4 8 | pytest-mock==3.14.0 9 | pyhamcrest==2.1.0 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11 2 | 3 | WORKDIR /usr/src/app 4 | 5 | COPY requirements.txt ./ 6 | RUN pip install --no-cache-dir -r requirements.txt 7 | 8 | COPY . . 9 | 10 | EXPOSE 80 11 | ENV PORT=80 12 | ENV PYTHONPATH=/usr/src/app 13 | CMD ["python", "app.py"] 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vk-channelify 2 | 3 | Пересылает новости из группы ВК в Телеграм канал. 4 | 5 | Предполагается, что 1 канал = 1 группа ВК. 6 | 7 | Ссылка: [@vk_channelify_bot](https://t.me/vk_channelify_bot) 8 | 9 | ## Как это работает 10 | 11 | ![](https://imgur.com/5rQ8eY5.png) 12 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # vk-channelify 2 | 3 | Пересылает новости из группы ВК в Телеграм канал. 4 | 5 | Предполагается, что 1 канал = 1 группа ВК. 6 | 7 | Ссылка: [@vk_channelify_bot](https://t.me/vk_channelify_bot) 8 | 9 | ## Как это работает 10 | 11 | ![](https://imgur.com/5rQ8eY5.png) 12 | 13 | --- 14 | 15 | [Telegram](https://t.me/olezhes) 16 | 17 | [Исходный код](https://github.com/reo7sp/vk-channelify) 18 | -------------------------------------------------------------------------------- /vk_channelify/models/channel.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, Integer 2 | 3 | from . import Base 4 | 5 | 6 | class Channel(Base): 7 | __tablename__ = 'channels' 8 | 9 | channel_id = Column(String, primary_key=True, nullable=False) 10 | vk_group_id = Column(String, nullable=False) 11 | last_vk_post_id = Column(Integer, nullable=False, server_default='0', default=0) 12 | owner_id = Column(String, nullable=False) 13 | owner_username = Column(String) 14 | hashtag_filter = Column(String) 15 | -------------------------------------------------------------------------------- /vk_channelify/models/__init__.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | from .time_stamp_mixin import TimeStampMixin 6 | 7 | 8 | class Base(TimeStampMixin): 9 | pass 10 | 11 | 12 | Base = declarative_base(cls=Base) 13 | 14 | from .channel import Channel 15 | from .disabled_channel import DisabledChannel 16 | 17 | 18 | def make_session_maker(url): 19 | engine = create_engine(url) 20 | return sessionmaker(bind=engine) 21 | -------------------------------------------------------------------------------- /vk_channelify/models/disabled_channel.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, String, Integer 2 | 3 | from . import Base 4 | 5 | 6 | class DisabledChannel(Base): 7 | __tablename__ = 'disabled_channels' 8 | 9 | id = Column(Integer, primary_key=True, autoincrement=True) 10 | channel_id = Column(String, nullable=False) 11 | vk_group_id = Column(String, nullable=False) 12 | last_vk_post_id = Column(Integer, nullable=False, server_default='0') 13 | owner_id = Column(String, nullable=False) 14 | owner_username = Column(String) 15 | hashtag_filter = Column(String) 16 | -------------------------------------------------------------------------------- /vk_channelify/vk_errors.py: -------------------------------------------------------------------------------- 1 | class VkError(Exception): 2 | def __init__(self, code, message, request_params): 3 | super(VkError, self).__init__() 4 | self.code = code 5 | self.message = message 6 | self.request_params = request_params 7 | 8 | def __str__(self): 9 | return 'VkError {}: {} (request_params: {})'.format(self.code, self.message, self.request_params) 10 | 11 | 12 | class VkWallAccessDeniedError(VkError): 13 | def __init__(self, code, message, request_params): 14 | super(VkWallAccessDeniedError, self).__init__(code, message, request_params) 15 | -------------------------------------------------------------------------------- /alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /vk_channelify/models/time_stamp_mixin.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from sqlalchemy import Column, DateTime, event 3 | 4 | class TimeStampMixin: 5 | created_at = Column(DateTime, default=datetime.utcnow, nullable=False) 6 | created_at._creation_order = 9998 7 | updated_at = Column(DateTime, default=datetime.utcnow, nullable=False) 8 | updated_at._creation_order = 9998 9 | 10 | @staticmethod 11 | def _updated_at(mapper, connection, target): 12 | target.updated_at = datetime.utcnow() 13 | 14 | @classmethod 15 | def __declare_last__(cls): 16 | event.listen(cls, 'before_update', cls._updated_at) 17 | -------------------------------------------------------------------------------- /alembic/versions/2e8d45ad3ac3_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 2e8d45ad3ac3 4 | Revises: 97bba2b7506c 5 | Create Date: 2017-07-10 01:04:39.286233 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '2e8d45ad3ac3' 14 | down_revision = '97bba2b7506c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | pass 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | pass 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /alembic/versions/215a632cdf23_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 215a632cdf23 4 | Revises: f5f69376d382 5 | Create Date: 2017-07-09 17:04:22.228617 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '215a632cdf23' 14 | down_revision = 'f5f69376d382' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('disabled_channels', sa.Column('channel_id', sa.String(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('disabled_channels', 'channel_id') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2017 Oleg Morozenkov 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import logging 4 | from prometheus_client import start_http_server 5 | 6 | from vk_channelify import models, run_manage_worker, run_repost_worker 7 | 8 | 9 | if __name__ == '__main__': 10 | telegram_token = os.getenv('TELEGRAM_TOKEN') 11 | vk_token = os.getenv('VK_TOKEN') 12 | db_url = os.getenv('DATABASE_URL') 13 | use_webhook = bool(int(os.getenv('USE_WEBHOOK', False))) 14 | webhook_domain = os.getenv('WEBHOOK_DOMAIN', '127.0.0.1') 15 | webhook_port = int(os.getenv('WEBHOOK_PORT', os.getenv('PORT', 80))) 16 | vk_thread_delay = int(os.getenv('REPOST_DELAY', 15 * 60)) # 15 minutes 17 | metrics_port = int(os.getenv('METRICS_PORT', 9090)) 18 | 19 | logging.basicConfig(level=logging.INFO) 20 | logger = logging.getLogger(__name__) 21 | 22 | try: 23 | start_http_server(metrics_port) 24 | logger.info('Prometheus metrics server started on port {}'.format(metrics_port)) 25 | except Exception as e: 26 | logger.warning('Failed to start Prometheus metrics server: {}'.format(e)) 27 | 28 | db_session_maker = models.make_session_maker(db_url) 29 | telegram_updater = run_manage_worker(telegram_token, db_session_maker, use_webhook, webhook_domain, webhook_port) 30 | repost_thread = run_repost_worker(vk_thread_delay, vk_token, telegram_token, db_session_maker) 31 | 32 | telegram_updater.idle() 33 | -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = alembic 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # max length of characters to apply to the 11 | # "slug" field 12 | #truncate_slug_length = 40 13 | 14 | # set to 'true' to run the environment during 15 | # the 'revision' command, regardless of autogenerate 16 | # revision_environment = false 17 | 18 | # set to 'true' to allow .pyc and .pyo files without 19 | # a source .py file to be detected as revisions in the 20 | # versions/ directory 21 | # sourceless = false 22 | 23 | # version location specification; this defaults 24 | # to alembic/versions. When using multiple version 25 | # directories, initial revisions must be specified with --version-path 26 | # version_locations = %(here)s/bar %(here)s/bat alembic/versions 27 | 28 | # the output encoding used when revision files 29 | # are written from script.py.mako 30 | # output_encoding = utf-8 31 | 32 | # sqlalchemy.url = driver://user:pass@localhost/dbname 33 | 34 | 35 | # Logging configuration 36 | [loggers] 37 | keys = root,sqlalchemy,alembic 38 | 39 | [handlers] 40 | keys = console 41 | 42 | [formatters] 43 | keys = generic 44 | 45 | [logger_root] 46 | level = WARN 47 | handlers = console 48 | qualname = 49 | 50 | [logger_sqlalchemy] 51 | level = WARN 52 | handlers = 53 | qualname = sqlalchemy.engine 54 | 55 | [logger_alembic] 56 | level = INFO 57 | handlers = 58 | qualname = alembic 59 | 60 | [handler_console] 61 | class = StreamHandler 62 | args = (sys.stderr,) 63 | level = NOTSET 64 | formatter = generic 65 | 66 | [formatter_generic] 67 | format = %(levelname)-5.5s [%(name)s] %(message)s 68 | datefmt = %H:%M:%S 69 | -------------------------------------------------------------------------------- /alembic/versions/f5f69376d382_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: f5f69376d382 4 | Revises: 5 | Create Date: 2017-07-09 15:30:59.186558 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'f5f69376d382' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('channels', 22 | sa.Column('channel_id', sa.String(), nullable=False), 23 | sa.Column('vk_group_id', sa.String(), nullable=False), 24 | sa.Column('last_vk_post_id', sa.Integer(), server_default='0', nullable=False), 25 | sa.Column('owner_id', sa.String(), nullable=True), 26 | sa.Column('owner_username', sa.String(), nullable=True), 27 | sa.Column('hashtag_filter', sa.String(), nullable=True), 28 | sa.Column('created_at', sa.DateTime(), nullable=False), 29 | sa.Column('updated_at', sa.DateTime(), nullable=False), 30 | sa.PrimaryKeyConstraint('channel_id') 31 | ) 32 | op.create_table('disabled_channels', 33 | sa.Column('id', sa.Integer(), nullable=False), 34 | sa.Column('vk_group_id', sa.String(), nullable=True), 35 | sa.Column('last_vk_post_id', sa.Integer(), nullable=True), 36 | sa.Column('owner_id', sa.String(), nullable=True), 37 | sa.Column('owner_username', sa.String(), nullable=True), 38 | sa.Column('hashtag_filter', sa.String(), nullable=True), 39 | sa.Column('created_at', sa.DateTime(), nullable=False), 40 | sa.Column('updated_at', sa.DateTime(), nullable=False), 41 | sa.PrimaryKeyConstraint('id') 42 | ) 43 | # ### end Alembic commands ### 44 | 45 | 46 | def downgrade(): 47 | # ### commands auto generated by Alembic - please adjust! ### 48 | op.drop_table('disabled_channels') 49 | op.drop_table('channels') 50 | # ### end Alembic commands ### 51 | -------------------------------------------------------------------------------- /alembic/versions/97bba2b7506c_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 97bba2b7506c 4 | Revises: 215a632cdf23 5 | Create Date: 2017-07-09 17:05:36.617297 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '97bba2b7506c' 14 | down_revision = '215a632cdf23' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.alter_column('channels', 'owner_id', 22 | existing_type=sa.VARCHAR(), 23 | nullable=False) 24 | op.alter_column('disabled_channels', 'channel_id', 25 | existing_type=sa.VARCHAR(), 26 | nullable=False) 27 | op.alter_column('disabled_channels', 'last_vk_post_id', 28 | existing_type=sa.INTEGER(), 29 | nullable=False) 30 | op.alter_column('disabled_channels', 'owner_id', 31 | existing_type=sa.VARCHAR(), 32 | nullable=False) 33 | op.alter_column('disabled_channels', 'vk_group_id', 34 | existing_type=sa.VARCHAR(), 35 | nullable=False) 36 | # ### end Alembic commands ### 37 | 38 | 39 | def downgrade(): 40 | # ### commands auto generated by Alembic - please adjust! ### 41 | op.alter_column('disabled_channels', 'vk_group_id', 42 | existing_type=sa.VARCHAR(), 43 | nullable=True) 44 | op.alter_column('disabled_channels', 'owner_id', 45 | existing_type=sa.VARCHAR(), 46 | nullable=True) 47 | op.alter_column('disabled_channels', 'last_vk_post_id', 48 | existing_type=sa.INTEGER(), 49 | nullable=True) 50 | op.alter_column('disabled_channels', 'channel_id', 51 | existing_type=sa.VARCHAR(), 52 | nullable=True) 53 | op.alter_column('channels', 'owner_id', 54 | existing_type=sa.VARCHAR(), 55 | nullable=True) 56 | # ### end Alembic commands ### 57 | -------------------------------------------------------------------------------- /alembic/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import os 4 | 5 | from alembic import context 6 | from logging.config import fileConfig 7 | from sqlalchemy import create_engine 8 | 9 | # this is the Alembic Config object, which provides 10 | # access to the values within the .ini file in use. 11 | config = context.config 12 | 13 | # Interpret the config file for Python logging. 14 | # This line sets up loggers basically. 15 | fileConfig(config.config_file_name) 16 | 17 | # add your model's MetaData object here 18 | # for 'autogenerate' support 19 | # from myapp import mymodel 20 | # target_metadata = mymodel.Base.metadata 21 | from vk_channelify import models 22 | target_metadata = models.Base.metadata 23 | 24 | # other values from the config, defined by the needs of env.py, 25 | # can be acquired: 26 | # my_important_option = config.get_main_option("my_important_option") 27 | # ... etc. 28 | 29 | 30 | def run_migrations_offline(): 31 | """Run migrations in 'offline' mode. 32 | 33 | This configures the context with just a URL 34 | and not an Engine, though an Engine is acceptable 35 | here as well. By skipping the Engine creation 36 | we don't even need a DBAPI to be available. 37 | 38 | Calls to context.execute() here emit the given string to the 39 | script output. 40 | 41 | """ 42 | url = os.getenv('DATABASE_URL') 43 | context.configure( 44 | url=url, target_metadata=target_metadata, literal_binds=True) 45 | 46 | with context.begin_transaction(): 47 | context.run_migrations() 48 | 49 | 50 | def run_migrations_online(): 51 | """Run migrations in 'online' mode. 52 | 53 | In this scenario we need to create an Engine 54 | and associate a connection with the context. 55 | 56 | """ 57 | connectable = create_engine(os.getenv('DATABASE_URL')) 58 | 59 | with connectable.connect() as connection: 60 | context.configure( 61 | connection=connection, 62 | target_metadata=target_metadata 63 | ) 64 | 65 | with context.begin_transaction(): 66 | context.run_migrations() 67 | 68 | if context.is_offline_mode(): 69 | run_migrations_offline() 70 | else: 71 | run_migrations_online() 72 | -------------------------------------------------------------------------------- /vk_channelify/metrics.py: -------------------------------------------------------------------------------- 1 | from prometheus_client import Counter, Gauge, Histogram, Info 2 | 3 | app_info = Info('vk_channelify', 'VK Channelify bot information') 4 | app_info.info({'version': '1.0.0', 'description': 'VK to Telegram channel reposter'}) 5 | 6 | # Repost worker metrics 7 | repost_iterations_total = Counter( 8 | 'vk_channelify_repost_iterations_total', 9 | 'Total number of repost worker iterations' 10 | ) 11 | repost_iteration_duration_seconds = Histogram( 12 | 'vk_channelify_repost_iteration_duration_seconds', 13 | 'Duration of repost worker iterations in seconds', 14 | buckets=(1, 5, 10, 30, 60, 120, 300, 600) 15 | ) 16 | repost_posts_sent_total = Counter( 17 | 'vk_channelify_posts_sent_total', 18 | 'Total number of posts sent to Telegram channels', 19 | ['channel_id', 'vk_group_id'] 20 | ) 21 | repost_errors_total = Counter( 22 | 'vk_channelify_repost_errors_total', 23 | 'Total number of errors during reposting', 24 | ['error_type', 'channel_id', 'vk_group_id'] 25 | ) 26 | vk_api_requests_total = Counter( 27 | 'vk_channelify_vk_api_requests_total', 28 | 'Total number of VK API requests', 29 | ['method', 'status', 'vk_group_id'] 30 | ) 31 | telegram_api_requests_total = Counter( 32 | 'vk_channelify_telegram_api_requests_total', 33 | 'Total number of Telegram API requests', 34 | ['method', 'status', 'channel_id', 'vk_group_id'] 35 | ) 36 | channels_disabled_total = Counter( 37 | 'vk_channelify_channels_disabled_total', 38 | 'Total number of channels disabled', 39 | ['channel_id', 'vk_group_id'] 40 | ) 41 | 42 | # Channel state metrics 43 | active_channels_gauge = Gauge( 44 | 'vk_channelify_active_channels', 45 | 'Current number of active channels' 46 | ) 47 | disabled_channels_gauge = Gauge( 48 | 'vk_channelify_disabled_channels', 49 | 'Current number of disabled channels' 50 | ) 51 | 52 | # Manage worker metrics 53 | telegram_commands_total = Counter( 54 | 'vk_channelify_telegram_commands_total', 55 | 'Total number of Telegram commands received', 56 | ['command'] 57 | ) 58 | telegram_command_duration_seconds = Histogram( 59 | 'vk_channelify_telegram_command_duration_seconds', 60 | 'Duration of Telegram command processing in seconds', 61 | ['command'], 62 | buckets=(0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10) 63 | ) 64 | telegram_conversations_total = Counter( 65 | 'vk_channelify_telegram_conversations_total', 66 | 'Total number of Telegram conversations', 67 | ['type', 'status'] 68 | ) 69 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff: 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/dictionaries 10 | 11 | # Sensitive or high-churn files: 12 | .idea/**/dataSources/ 13 | .idea/**/dataSources.ids 14 | .idea/**/dataSources.xml 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | 20 | # Gradle: 21 | .idea/**/gradle.xml 22 | .idea/**/libraries 23 | 24 | # Mongo Explorer plugin: 25 | .idea/**/mongoSettings.xml 26 | 27 | ## File-based project format: 28 | *.iws 29 | 30 | ## Plugin-specific files: 31 | 32 | # IntelliJ 33 | /out/ 34 | 35 | # mpeltonen/sbt-idea plugin 36 | .idea_modules/ 37 | 38 | # JIRA plugin 39 | atlassian-ide-plugin.xml 40 | 41 | # Crashlytics plugin (for Android Studio and IntelliJ) 42 | com_crashlytics_export_strings.xml 43 | crashlytics.properties 44 | crashlytics-build.properties 45 | fabric.properties 46 | ### VirtualEnv template 47 | # Virtualenv 48 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 49 | .Python 50 | [Bb]in 51 | [Ii]nclude 52 | [Ll]ib 53 | [Ll]ib64 54 | [Ll]ocal 55 | pyvenv.cfg 56 | .venv 57 | pip-selfcheck.json 58 | ### Python template 59 | # Byte-compiled / optimized / DLL files 60 | __pycache__/ 61 | *.py[cod] 62 | *$py.class 63 | 64 | # C extensions 65 | *.so 66 | 67 | # Distribution / packaging 68 | .Python 69 | env/ 70 | build/ 71 | develop-eggs/ 72 | dist/ 73 | downloads/ 74 | eggs/ 75 | .eggs/ 76 | lib/ 77 | lib64/ 78 | parts/ 79 | sdist/ 80 | var/ 81 | wheels/ 82 | *.egg-info/ 83 | .installed.cfg 84 | *.egg 85 | 86 | # PyInstaller 87 | # Usually these files are written by a python script from a template 88 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 89 | *.manifest 90 | *.spec 91 | 92 | # Installer logs 93 | pip-log.txt 94 | pip-delete-this-directory.txt 95 | 96 | # Unit test / coverage reports 97 | htmlcov/ 98 | .tox/ 99 | .coverage 100 | .coverage.* 101 | .cache 102 | nosetests.xml 103 | coverage.xml 104 | *,cover 105 | .hypothesis/ 106 | 107 | # Translations 108 | *.mo 109 | *.pot 110 | 111 | # Django stuff: 112 | *.log 113 | local_settings.py 114 | 115 | # Flask stuff: 116 | instance/ 117 | .webassets-cache 118 | 119 | # Scrapy stuff: 120 | .scrapy 121 | 122 | # Sphinx documentation 123 | docs/_build/ 124 | 125 | # PyBuilder 126 | target/ 127 | 128 | # Jupyter Notebook 129 | .ipynb_checkpoints 130 | 131 | # pyenv 132 | .python-version 133 | 134 | # celery beat schedule file 135 | celerybeat-schedule 136 | 137 | # SageMath parsed files 138 | *.sage.py 139 | 140 | # dotenv 141 | .env 142 | 143 | # virtualenv 144 | .venv 145 | venv/ 146 | ENV/ 147 | 148 | # Spyder project settings 149 | .spyderproject 150 | 151 | # Rope project settings 152 | .ropeproject 153 | 154 | 155 | .idea 156 | -------------------------------------------------------------------------------- /tests/test_manage_worker.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import Mock, patch 3 | from hamcrest import assert_that, equal_to, is_ 4 | 5 | from vk_channelify.manage_worker import ( 6 | start, 7 | new, 8 | new_in_state_asked_vk_group_link, 9 | new_in_state_asked_channel_access, 10 | new_in_state_asked_channel_message, 11 | cancel_new, 12 | del_state, 13 | ASKED_VK_GROUP_LINK_IN_NEW, 14 | ASKED_CHANNEL_ACCESS_IN_NEW, 15 | ASKED_CHANNEL_MESSAGE_IN_NEW 16 | ) 17 | from telegram.ext import ConversationHandler 18 | 19 | 20 | class TestDelState: 21 | def test_deletes_user_state_if_exists(self): 22 | update = Mock() 23 | update.message.from_user.id = 12345 24 | users_state = {12345: {'data': 'value'}} 25 | 26 | del_state(update, users_state) 27 | 28 | assert_that(12345 not in users_state, is_(True)) 29 | 30 | def test_does_nothing_if_state_not_exists(self): 31 | update = Mock() 32 | update.message.from_user.id = 12345 33 | users_state = {} 34 | 35 | del_state(update, users_state) 36 | 37 | assert_that(12345 not in users_state, is_(True)) 38 | 39 | 40 | class TestStart: 41 | def test_start_sends_welcome_message(self): 42 | update = Mock() 43 | context = Mock() 44 | 45 | start(update, context) 46 | 47 | update.message.reply_text.assert_called_once() 48 | call_args = update.message.reply_text.call_args[0][0] 49 | assert_that('/new' in call_args, is_(True)) 50 | 51 | 52 | class TestNew: 53 | @patch('vk_channelify.manage_worker.metrics') 54 | def test_new_starts_conversation(self, mock_metrics): 55 | update = Mock() 56 | context = Mock() 57 | 58 | result = new(update, context) 59 | 60 | assert_that(result, equal_to(ASKED_VK_GROUP_LINK_IN_NEW)) 61 | update.message.reply_text.assert_called_once() 62 | 63 | 64 | class TestNewInStateAskedVkGroupLink: 65 | def test_saves_vk_domain_and_asks_for_channel_access(self): 66 | update = Mock() 67 | context = Mock() 68 | update.message.text = 'https://vk.ru/mygroup' 69 | update.message.from_user.id = 12345 70 | users_state = {} 71 | 72 | result = new_in_state_asked_vk_group_link(update, context, users_state=users_state) 73 | 74 | assert_that(result, equal_to(ASKED_CHANNEL_ACCESS_IN_NEW)) 75 | assert_that(users_state[12345]['vk_domain'], equal_to('mygroup')) 76 | assert_that(update.message.reply_text.call_count, equal_to(3)) 77 | 78 | 79 | class TestNewInStateAskedChannelAccess: 80 | def test_asks_for_channel_message(self): 81 | update = Mock() 82 | context = Mock() 83 | 84 | result = new_in_state_asked_channel_access(update, context) 85 | 86 | assert_that(result, equal_to(ASKED_CHANNEL_MESSAGE_IN_NEW)) 87 | update.message.reply_text.assert_called_once() 88 | 89 | 90 | class TestNewInStateAskedChannelMessage: 91 | @patch('vk_channelify.manage_worker.metrics') 92 | @patch('vk_channelify.manage_worker.Channel') 93 | @patch('vk_channelify.manage_worker.DisabledChannel') 94 | def test_creates_channel_successfully(self, mock_disabled_channel, mock_channel, mock_metrics): 95 | update = Mock() 96 | context = Mock() 97 | update.message.from_user.id = 12345 98 | update.message.from_user.username = 'testuser' 99 | update.message.forward_from_chat.id = -100123456 100 | users_state = {12345: {'vk_domain': 'mygroup'}} 101 | db = Mock() 102 | db_session_maker = Mock(return_value=db) 103 | 104 | result = new_in_state_asked_channel_message(update, context, db_session_maker=db_session_maker, users_state=users_state) 105 | 106 | assert_that(result, equal_to(ConversationHandler.END)) 107 | db.add.assert_called_once() 108 | db.commit.assert_called_once() 109 | context.bot.send_message.assert_called_once() 110 | 111 | @patch('vk_channelify.manage_worker.metrics') 112 | @patch('vk_channelify.manage_worker.Channel') 113 | def test_rolls_back_on_error(self, mock_channel, mock_metrics): 114 | update = Mock() 115 | context = Mock() 116 | update.message.from_user.id = 12345 117 | update.message.from_user.username = 'testuser' 118 | update.message.forward_from_chat.id = -100123456 119 | users_state = {12345: {'vk_domain': 'mygroup'}} 120 | db = Mock() 121 | db.commit.side_effect = Exception('DB Error') 122 | db_session_maker = Mock(return_value=db) 123 | 124 | with pytest.raises(Exception): 125 | new_in_state_asked_channel_message(update, context, db_session_maker=db_session_maker, users_state=users_state) 126 | 127 | db.rollback.assert_called_once() 128 | mock_metrics.telegram_conversations_total.labels.assert_called_with(type='new', status='failed') 129 | 130 | 131 | class TestCancelNew: 132 | @patch('vk_channelify.manage_worker.metrics') 133 | def test_cancel_ends_conversation(self, mock_metrics): 134 | update = Mock() 135 | context = Mock() 136 | update.message.from_user.id = 12345 137 | users_state = {12345: {'vk_domain': 'mygroup'}} 138 | 139 | result = cancel_new(update, context, users_state=users_state) 140 | 141 | assert_that(result, equal_to(ConversationHandler.END)) 142 | assert_that(12345 not in users_state, is_(True)) 143 | -------------------------------------------------------------------------------- /tests/test_repost_worker.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import Mock, patch 3 | import telegram 4 | from hamcrest import assert_that, equal_to, is_, none, has_length 5 | 6 | from vk_channelify.repost_worker import ( 7 | extract_group_id_if_has, 8 | is_passing_hashtag_filter, 9 | fetch_group_posts, 10 | disable_channel, 11 | run_worker_iteration 12 | ) 13 | from vk_channelify.vk_errors import VkError, VkWallAccessDeniedError 14 | 15 | 16 | class TestRunWorkerIteration: 17 | @patch('vk_channelify.repost_worker.telegram.Bot') 18 | @patch('vk_channelify.repost_worker.fetch_group_posts') 19 | @patch('vk_channelify.repost_worker.metrics') 20 | def test_iteration_sends_new_posts(self, mock_metrics, mock_fetch, mock_bot_class): 21 | mock_bot = Mock() 22 | mock_bot_class.return_value = mock_bot 23 | mock_channel = Mock(channel_id='-100123456', vk_group_id='testgroup', last_vk_post_id=10, hashtag_filter=None) 24 | mock_db = Mock() 25 | mock_db.query.return_value.count.return_value = 1 26 | mock_db.query.return_value.__iter__ = Mock(return_value=iter([mock_channel])) 27 | mock_fetch.return_value = [ 28 | {'id': 11, 'owner_id': -123, 'text': 'New post 1'}, 29 | {'id': 12, 'owner_id': -123, 'text': 'New post 2'} 30 | ] 31 | 32 | run_worker_iteration('vk_token', 'tg_token', mock_db) 33 | 34 | assert_that(mock_bot.send_message.call_count, equal_to(2)) 35 | assert_that(mock_channel.last_vk_post_id, equal_to(12)) 36 | 37 | @patch('vk_channelify.repost_worker.telegram.Bot') 38 | @patch('vk_channelify.repost_worker.fetch_group_posts') 39 | @patch('vk_channelify.repost_worker.metrics') 40 | def test_iteration_skips_old_posts(self, mock_metrics, mock_fetch, mock_bot_class): 41 | mock_bot = Mock() 42 | mock_bot_class.return_value = mock_bot 43 | mock_channel = Mock(channel_id='-100123456', vk_group_id='testgroup', last_vk_post_id=10, hashtag_filter=None) 44 | mock_db = Mock() 45 | mock_db.query.return_value.count.return_value = 1 46 | mock_db.query.return_value.__iter__ = Mock(return_value=iter([mock_channel])) 47 | mock_fetch.return_value = [{'id': 9, 'owner_id': -123, 'text': 'Old post'}] 48 | 49 | run_worker_iteration('vk_token', 'tg_token', mock_db) 50 | 51 | mock_bot.send_message.assert_not_called() 52 | 53 | @patch('vk_channelify.repost_worker.telegram.Bot') 54 | @patch('vk_channelify.repost_worker.fetch_group_posts') 55 | @patch('vk_channelify.repost_worker.disable_channel') 56 | @patch('vk_channelify.repost_worker.metrics') 57 | def test_iteration_disables_channel_on_unauthorized(self, mock_metrics, mock_disable, mock_fetch, mock_bot_class): 58 | mock_bot = Mock() 59 | mock_bot_class.return_value = mock_bot 60 | mock_bot.send_message.side_effect = telegram.error.Unauthorized('Unauthorized') 61 | mock_channel = Mock(channel_id='-100123456', vk_group_id='testgroup', last_vk_post_id=10, hashtag_filter=None) 62 | mock_db = Mock() 63 | mock_db.query.return_value.count.return_value = 1 64 | mock_db.query.return_value.__iter__ = Mock(return_value=iter([mock_channel])) 65 | mock_fetch.return_value = [{'id': 11, 'owner_id': -123, 'text': 'New post'}] 66 | 67 | run_worker_iteration('vk_token', 'tg_token', mock_db) 68 | 69 | mock_disable.assert_called_once_with(mock_channel, mock_db, mock_bot) 70 | 71 | 72 | class TestFetchGroupPosts: 73 | @patch('vk_channelify.repost_worker.requests.get') 74 | @patch('vk_channelify.repost_worker.time.sleep') 75 | @patch('vk_channelify.repost_worker.metrics') 76 | def test_fetch_success(self, mock_metrics, mock_sleep, mock_get): 77 | mock_get.return_value.json.return_value = { 78 | 'response': {'items': [{'id': 1, 'text': 'Post 1'}]} 79 | } 80 | 81 | posts = fetch_group_posts('mygroup', 'test_token') 82 | 83 | assert_that(posts, has_length(1)) 84 | assert_that(posts[0]['id'], equal_to(1)) 85 | 86 | @patch('vk_channelify.repost_worker.requests.get') 87 | @patch('vk_channelify.repost_worker.time.sleep') 88 | @patch('vk_channelify.repost_worker.metrics') 89 | def test_fetch_access_denied_error(self, mock_metrics, mock_sleep, mock_get): 90 | mock_get.return_value.json.return_value = { 91 | 'error': {'error_code': 15, 'error_msg': 'Access denied', 'request_params': []} 92 | } 93 | 94 | with pytest.raises(VkWallAccessDeniedError): 95 | fetch_group_posts('mygroup', 'test_token') 96 | 97 | 98 | class TestExtractGroupIdIfHas: 99 | def test_extract_club_id(self): 100 | assert_that(extract_group_id_if_has('club12345'), equal_to('12345')) 101 | 102 | def test_extract_public_id(self): 103 | assert_that(extract_group_id_if_has('public67890'), equal_to('67890')) 104 | 105 | def test_domain_name_returns_none(self): 106 | assert_that(extract_group_id_if_has('mygroup'), is_(none())) 107 | 108 | 109 | class TestIsPassingHashtagFilter: 110 | def test_no_filter_always_passes(self): 111 | assert_that(is_passing_hashtag_filter(None, {'text': 'Any text'}), is_(True)) 112 | 113 | def test_single_hashtag_match(self): 114 | assert_that(is_passing_hashtag_filter('#news', {'text': 'Post with #news'}), is_(True)) 115 | 116 | def test_single_hashtag_no_match(self): 117 | assert_that(is_passing_hashtag_filter('#news', {'text': 'Post with #other'}), is_(False)) 118 | 119 | def test_multiple_hashtags_match(self): 120 | assert_that(is_passing_hashtag_filter('#news, #update', {'text': 'Post with #update'}), is_(True)) 121 | 122 | 123 | class TestDisableChannel: 124 | @patch('vk_channelify.repost_worker.metrics') 125 | def test_disable_channel_success(self, mock_metrics): 126 | mock_channel = Mock(channel_id='-100123456', vk_group_id='testgroup') 127 | mock_db = Mock() 128 | mock_bot = Mock() 129 | 130 | disable_channel(mock_channel, mock_db, mock_bot) 131 | 132 | mock_db.add.assert_called_once() 133 | mock_db.delete.assert_called_once_with(mock_channel) 134 | mock_db.commit.assert_called_once() 135 | 136 | @patch('vk_channelify.repost_worker.metrics') 137 | def test_disable_channel_rollback_on_error(self, mock_metrics): 138 | mock_channel = Mock(channel_id='-100123456', vk_group_id='testgroup') 139 | mock_db = Mock() 140 | mock_db.commit.side_effect = Exception('DB Error') 141 | 142 | with pytest.raises(Exception): 143 | disable_channel(mock_channel, mock_db, Mock()) 144 | 145 | mock_db.rollback.assert_called_once() 146 | -------------------------------------------------------------------------------- /vk_channelify/repost_worker.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | import traceback 4 | from threading import Thread 5 | 6 | import logging 7 | import requests 8 | import telegram 9 | 10 | from vk_channelify.models.disabled_channel import DisabledChannel 11 | from vk_channelify.vk_errors import VkError, VkWallAccessDeniedError 12 | from .models import Channel 13 | from . import metrics 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def run_worker(iteration_delay, vk_service_code, telegram_token, db_session_maker): 19 | thread = Thread(target=run_worker_inside_thread, 20 | args=(iteration_delay, vk_service_code, telegram_token, db_session_maker), 21 | daemon=True) 22 | thread.start() 23 | return thread 24 | 25 | 26 | def run_worker_inside_thread(iteration_delay, vk_service_code, telegram_token, db_session_maker): 27 | while True: 28 | start_time = datetime.datetime.now() 29 | logger.info('New iteration {}'.format(start_time)) 30 | metrics.repost_iterations_total.inc() 31 | 32 | try: 33 | db = db_session_maker() 34 | with metrics.repost_iteration_duration_seconds.time(): 35 | run_worker_iteration(vk_service_code, telegram_token, db) 36 | except Exception as e: 37 | logger.error('Iteration was failed because of {}'.format(e)) 38 | traceback.print_exc() 39 | metrics.repost_errors_total.labels(error_type='iteration_failed', channel_id='', vk_group_id='').inc() 40 | finally: 41 | try: 42 | db.close() 43 | except Exception as e: 44 | logger.error('Iteration has failed db.close() because of {}'.format(e)) 45 | traceback.print_exc() 46 | 47 | end_time = datetime.datetime.now() 48 | logger.info('Finished iteration {} ({})'.format(end_time, end_time - start_time)) 49 | 50 | time.sleep(iteration_delay) 51 | 52 | 53 | def run_worker_iteration(vk_service_code, telegram_token, db): 54 | bot = telegram.Bot(telegram_token) 55 | 56 | active_count = db.query(Channel).count() 57 | disabled_count = db.query(DisabledChannel).count() 58 | metrics.active_channels_gauge.set(active_count) 59 | metrics.disabled_channels_gauge.set(disabled_count) 60 | 61 | for channel in db.query(Channel): 62 | try: 63 | log_id = '{} (id: {})'.format(channel.vk_group_id, channel.channel_id) 64 | metrics_kwargs = {'channel_id': channel.channel_id, 'vk_group_id': channel.vk_group_id} 65 | 66 | posts = fetch_group_posts(channel.vk_group_id, vk_service_code) 67 | posts_sent = 0 68 | 69 | for post in sorted(posts, key=lambda p: p['id']): 70 | if post['id'] <= channel.last_vk_post_id: 71 | continue 72 | if not is_passing_hashtag_filter(channel.hashtag_filter, post): 73 | continue 74 | 75 | post_url = 'https://vk.ru/wall{}_{}'.format(post['owner_id'], post['id']) 76 | text = '{}\n\n{}'.format(post_url, post['text']) 77 | if len(text) > 4000: 78 | text = text[0:4000] + '...' 79 | 80 | try: 81 | bot.send_message(channel.channel_id, text) 82 | metrics.telegram_api_requests_total.labels(method='send_message', status='success', **metrics_kwargs).inc() 83 | posts_sent += 1 84 | metrics.repost_posts_sent_total.labels(**metrics_kwargs).inc() 85 | except telegram.error.TelegramError as send_error: 86 | metrics.telegram_api_requests_total.labels(method='send_message', status='error', **metrics_kwargs).inc() 87 | raise send_error 88 | 89 | try: 90 | channel.last_vk_post_id = post['id'] 91 | db.commit() 92 | except: 93 | db.rollback() 94 | raise 95 | 96 | if posts_sent: 97 | logger.info('Success sent {} posts on channel {}'.format(posts_sent, log_id)) 98 | 99 | except telegram.error.BadRequest as e: 100 | if 'chat not found' in e.message.lower(): 101 | logger.warning('Disabling channel {} because of telegram error: {}'.format(log_id, e)) 102 | traceback.print_exc() 103 | metrics.repost_errors_total.labels(error_type='telegram_chat_not_found', **metrics_kwargs).inc() 104 | disable_channel(channel, db, bot) 105 | else: 106 | metrics.repost_errors_total.labels(error_type='telegram_bad_request', **metrics_kwargs).inc() 107 | raise e 108 | 109 | except telegram.error.Unauthorized as e: 110 | logger.warning('Disabling channel {} because of telegram error: {}'.format(log_id, e)) 111 | traceback.print_exc() 112 | metrics.repost_errors_total.labels(error_type='telegram_unauthorized', **metrics_kwargs).inc() 113 | disable_channel(channel, db, bot) 114 | 115 | except telegram.error.TimedOut as e: 116 | logger.warning('Got telegram TimedOut error on channel {}'.format(log_id)) 117 | metrics.repost_errors_total.labels(error_type='telegram_timeout', **metrics_kwargs).inc() 118 | 119 | except VkWallAccessDeniedError as e: 120 | logger.warning('Disabling channel {} because of vk error: {}'.format(log_id, e)) 121 | traceback.print_exc() 122 | metrics.repost_errors_total.labels(error_type='vk_wall_access_denied', **metrics_kwargs).inc() 123 | disable_channel(channel, db, bot) 124 | 125 | 126 | def fetch_group_posts(group, vk_service_code): 127 | time.sleep(0.35) 128 | 129 | group_id = extract_group_id_if_has(group) 130 | is_group_domain_passed = group_id is None 131 | 132 | if is_group_domain_passed: 133 | url = 'https://api.vk.ru/method/wall.get?domain={}&count=10&access_token={}&v=5.131'.format(group, vk_service_code) 134 | r = requests.get(url) 135 | else: 136 | url = 'https://api.vk.ru/method/wall.get?owner_id=-{}&count=10&access_token={}&v=5.131'.format(group_id, vk_service_code) 137 | r = requests.get(url) 138 | j = r.json() 139 | 140 | if 'response' not in j: 141 | logger.error('VK responded with {}'.format(j)) 142 | metrics.vk_api_requests_total.labels(method='wall.get', status='error', vk_group_id=group).inc() 143 | 144 | error_code = int(j['error']['error_code']) 145 | if error_code in [15, 18, 19, 100]: 146 | raise VkWallAccessDeniedError(error_code, j['error']['error_msg'], j['error']['request_params']) 147 | else: 148 | raise VkError(error_code, j['error']['error_msg'], j['error']['request_params']) 149 | 150 | metrics.vk_api_requests_total.labels(method='wall.get', status='success', vk_group_id=group).inc() 151 | 152 | return j['response']['items'] 153 | 154 | 155 | def extract_group_id_if_has(group_name): 156 | domainless_group_prefixes = ['club', 'public'] 157 | for prefix in domainless_group_prefixes: 158 | if group_name.startswith(prefix): 159 | group_id = group_name[len(prefix):] 160 | if group_id.isdigit(): 161 | return group_id 162 | 163 | return None 164 | 165 | 166 | def is_passing_hashtag_filter(hashtag_filter, post): 167 | if hashtag_filter is None: 168 | return True 169 | 170 | return any(hashtag.strip() in post['text'] for hashtag in hashtag_filter.split(',')) 171 | 172 | 173 | def disable_channel(channel, db, bot): 174 | log_id = '{} (id: {})'.format(channel.vk_group_id, channel.channel_id) 175 | metrics_kwargs = {'channel_id': channel.channel_id, 'vk_group_id': channel.vk_group_id} 176 | 177 | logger.warning('Disabling channel {}'.format(log_id)) 178 | metrics.channels_disabled_total.labels(**metrics_kwargs).inc() 179 | 180 | try: 181 | db.add(DisabledChannel(channel_id=channel.channel_id, 182 | vk_group_id=channel.vk_group_id, 183 | last_vk_post_id=channel.last_vk_post_id, 184 | owner_id=channel.owner_id, 185 | owner_username=channel.owner_username, 186 | hashtag_filter=channel.hashtag_filter)) 187 | db.delete(channel) 188 | db.commit() 189 | except: 190 | db.rollback() 191 | raise 192 | 193 | try: 194 | bot.send_message(channel.owner_id, 'Канал https://vk.ru/{} отключен'.format(channel.vk_group_id)) 195 | bot.send_message(channel.owner_id, 'Так как не удается отправить в него сообщение') 196 | bot.send_message(channel.owner_id, 'ID канала {}'.format(channel.channel_id)) 197 | bot.send_message(channel.owner_id, 'Чтобы восстановить канал, вызовите команду /recover') 198 | except telegram.error.TelegramError: 199 | logger.warning('Cannot send recover message to {} (id: {})'.format(channel.owner_username, channel.owner_id)) 200 | traceback.print_exc() 201 | -------------------------------------------------------------------------------- /vk_channelify/manage_worker.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import time 3 | from functools import partial, wraps 4 | 5 | import logging 6 | import telegram 7 | from telegram import ReplyKeyboardMarkup, ReplyKeyboardRemove 8 | from telegram.ext import CommandHandler, Updater, ConversationHandler, Filters, MessageHandler, RegexHandler 9 | 10 | from . import models, metrics 11 | from .models import Channel, DisabledChannel 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | ASKED_VK_GROUP_LINK_IN_NEW, ASKED_CHANNEL_ACCESS_IN_NEW, ASKED_CHANNEL_MESSAGE_IN_NEW, \ 16 | ASKED_CHANNEL_ID_IN_FILTER_BY_HASHTAG, ASKED_HASHTAGS_IN_FILTER_BY_HASHTAG, \ 17 | ASKED_CHANNEL_ID_IN_RECOVER = list(range(6)) 18 | 19 | 20 | def run_worker(telegram_token, db_session_maker, use_webhook, webhook_domain='', webhook_port=''): 21 | users_state = dict() 22 | 23 | updater = Updater(telegram_token) 24 | 25 | dp = updater.dispatcher 26 | dp.add_error_handler(on_error) 27 | dp.add_handler(CommandHandler('start', start)) 28 | dp.add_handler(ConversationHandler( 29 | entry_points=[CommandHandler('new', new)], 30 | states={ 31 | ASKED_VK_GROUP_LINK_IN_NEW: [ 32 | RegexHandler('^https://vk.ru/', partial(new_in_state_asked_vk_group_link, 33 | users_state=users_state)) 34 | ], 35 | ASKED_CHANNEL_ACCESS_IN_NEW: [ 36 | RegexHandler('^Я сделал$', new_in_state_asked_channel_access) 37 | ], 38 | ASKED_CHANNEL_MESSAGE_IN_NEW: [ 39 | MessageHandler(Filters.forwarded, partial(new_in_state_asked_channel_message, 40 | db_session_maker=db_session_maker, users_state=users_state)) 41 | ] 42 | }, 43 | allow_reentry=True, 44 | fallbacks=[CommandHandler('cancel', partial(cancel_new, users_state=users_state))] 45 | )) 46 | dp.add_handler(ConversationHandler( 47 | entry_points=[CommandHandler('filter_by_hashtag', partial(filter_by_hashtag, 48 | db_session_maker=db_session_maker, users_state=users_state))], 49 | states={ 50 | ASKED_CHANNEL_ID_IN_FILTER_BY_HASHTAG: [ 51 | MessageHandler(Filters.text, partial(filter_by_hashtag_in_state_asked_channel_id, 52 | db_session_maker=db_session_maker, users_state=users_state)) 53 | ], 54 | ASKED_HASHTAGS_IN_FILTER_BY_HASHTAG: [ 55 | MessageHandler(Filters.text, partial(filter_by_hashtag_in_state_asked_hashtags, 56 | db_session_maker=db_session_maker, users_state=users_state)) 57 | ] 58 | }, 59 | allow_reentry=True, 60 | fallbacks=[CommandHandler('cancel', partial(cancel_filter_by_hashtag, 61 | users_state=users_state))] 62 | )) 63 | dp.add_handler(ConversationHandler( 64 | entry_points=[CommandHandler('recover', partial(recover, 65 | db_session_maker=db_session_maker, users_state=users_state))], 66 | states={ 67 | ASKED_CHANNEL_ID_IN_RECOVER: [ 68 | MessageHandler(Filters.text, partial(recover_in_state_asked_channel_id, 69 | db_session_maker=db_session_maker, users_state=users_state)) 70 | ] 71 | }, 72 | allow_reentry=True, 73 | fallbacks=[CommandHandler('cancel', partial(cancel_recover, 74 | users_state=users_state))] 75 | )) 76 | 77 | if use_webhook: 78 | logger.info('Starting webhook at {}:{}'.format(webhook_domain, webhook_port)) 79 | updater.start_webhook('0.0.0.0', webhook_port, telegram_token) 80 | updater.bot.set_webhook('https://{}/{}'.format(webhook_domain, telegram_token)) 81 | else: 82 | logger.info('Starting long poll') 83 | updater.start_polling() 84 | 85 | return updater 86 | 87 | 88 | def del_state(update, users_state): 89 | if update.message.from_user.id in users_state: 90 | del users_state[update.message.from_user.id] 91 | 92 | 93 | def on_error(update, context): 94 | logger.error('Update "{}" caused error "{}"'.format(update, context.error)) 95 | traceback.print_exc() 96 | 97 | if update is not None and hasattr(update, 'message') and update.message is not None: 98 | update.message.reply_text('Внутренняя ошибка') 99 | update.message.reply_text('{}: {}'.format(type(context.error).__name__, str(context.error))) 100 | update.message.reply_text('Сообщите @olezhes') 101 | 102 | 103 | def catch_exceptions(func): 104 | @wraps(func) 105 | def wrapper(update, context, *args, **kwargs): 106 | try: 107 | return func(update, context, *args, **kwargs) 108 | except Exception as e: 109 | logger.error('Exception in {}: {}'.format(func.__name__, e)) 110 | traceback.print_exc() 111 | raise 112 | 113 | return wrapper 114 | 115 | 116 | def observe_metrics(command_name): 117 | def decorator(func): 118 | @wraps(func) 119 | def wrapper(update, context, *args, **kwargs): 120 | metrics.telegram_commands_total.labels(command=command_name).inc() 121 | start_time = time.time() 122 | try: 123 | result = func(update, context, *args, **kwargs) 124 | duration = time.time() - start_time 125 | metrics.telegram_command_duration_seconds.labels(command=command_name).observe(duration) 126 | return result 127 | except Exception as e: 128 | duration = time.time() - start_time 129 | metrics.telegram_command_duration_seconds.labels(command=command_name).observe(duration) 130 | raise e 131 | 132 | return wrapper 133 | 134 | return decorator 135 | 136 | 137 | def make_db_session(func): 138 | @wraps(func) 139 | def wrapper(*args, db_session_maker, **kwargs): 140 | db = db_session_maker() 141 | result = func(*args, **kwargs, db=db) 142 | db.close() 143 | return result 144 | 145 | return wrapper 146 | 147 | 148 | @catch_exceptions 149 | @observe_metrics('start') 150 | def start(update, context): 151 | update.message.reply_text('Команда /new настроит новый канал. В канал будут пересылаться посты из группы ВК') 152 | 153 | 154 | @catch_exceptions 155 | @observe_metrics('new') 156 | def new(update, context): 157 | metrics.telegram_conversations_total.labels(type='new', status='started').inc() 158 | 159 | update.message.reply_text('Отправьте ссылку на группу ВК') 160 | 161 | return ASKED_VK_GROUP_LINK_IN_NEW 162 | 163 | 164 | @catch_exceptions 165 | def new_in_state_asked_vk_group_link(update, context, users_state): 166 | vk_url = update.message.text 167 | vk_domain = vk_url.split('/')[-1] 168 | users_state[update.message.from_user.id] = dict() 169 | users_state[update.message.from_user.id]['vk_domain'] = vk_domain 170 | 171 | update.message.reply_text('Отлично! Теперь:') 172 | update.message.reply_text('1. Создайте новый канал. Можно использовать существующий') 173 | keyboard = [['Я сделал']] 174 | update.message.reply_text('2. Добавьте этого бота (@vk_channelify_bot) в администраторы канала', 175 | reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True)) 176 | 177 | return ASKED_CHANNEL_ACCESS_IN_NEW 178 | 179 | 180 | @catch_exceptions 181 | def new_in_state_asked_channel_access(update, context): 182 | update.message.reply_text('Хорошо. Перешлите любое сообщение из канала', reply_markup=ReplyKeyboardRemove()) 183 | 184 | return ASKED_CHANNEL_MESSAGE_IN_NEW 185 | 186 | 187 | @catch_exceptions 188 | @make_db_session 189 | def new_in_state_asked_channel_message(update, context, db, users_state): 190 | user_id = update.message.from_user.id 191 | username = update.message.from_user.username 192 | channel_id = str(update.message.forward_from_chat.id) 193 | vk_group_id = users_state[user_id]['vk_domain'] 194 | 195 | try: 196 | channel = Channel(channel_id=channel_id, vk_group_id=vk_group_id, owner_id=user_id, owner_username=username) 197 | db.add(channel) 198 | db.commit() 199 | metrics.telegram_conversations_total.labels(type='new', status='completed').inc() 200 | except Exception: 201 | db.rollback() 202 | metrics.telegram_conversations_total.labels(type='new', status='failed').inc() 203 | raise 204 | 205 | try: 206 | db.query(DisabledChannel).filter(DisabledChannel.channel_id == channel_id).delete() 207 | except Exception: 208 | logger.warning('Cannot delete disabled channel of {}'.format(channel_id)) 209 | traceback.print_exc() 210 | 211 | context.bot.send_message(channel_id, 'Канал работает с помощью @vk_channelify_bot') 212 | 213 | update.message.reply_text('Готово!') 214 | update.message.reply_text('Бот будет проверять группу каждые 15 минут') 215 | update.message.reply_text('Настроить фильтр по хештегам можно командой /filter_by_hashtag') 216 | update.message.reply_text('Команда /new настроит новый канал') 217 | 218 | del_state(update, users_state) 219 | 220 | return ConversationHandler.END 221 | 222 | 223 | @catch_exceptions 224 | def cancel_new(update, context, users_state): 225 | metrics.telegram_conversations_total.labels(type='new', status='cancelled').inc() 226 | 227 | update.message.reply_text('Ладно', reply_markup=ReplyKeyboardRemove()) 228 | update.message.reply_text('Команда /new настроит новый канал') 229 | 230 | del_state(update, users_state) 231 | 232 | return ConversationHandler.END 233 | 234 | 235 | @catch_exceptions 236 | @make_db_session 237 | @observe_metrics('filter_by_hashtag') 238 | def filter_by_hashtag(update, context, db, users_state): 239 | user_id = update.message.from_user.id 240 | 241 | metrics.telegram_conversations_total.labels(type='filter_by_hashtag', status='started').inc() 242 | 243 | users_state[user_id] = dict() 244 | users_state[user_id]['channels'] = dict() 245 | keyboard = [] 246 | keyboard_row = [] 247 | for channel in db.query(Channel).filter(Channel.owner_id == str(user_id)).order_by(Channel.created_at.desc()): 248 | try: 249 | channel_chat = context.bot.get_chat(chat_id=channel.channel_id) 250 | users_state[user_id]['channels'][channel_chat.title] = channel.channel_id 251 | keyboard_row.append(channel_chat.title) 252 | if len(keyboard_row) == 2: 253 | keyboard.append(keyboard_row) 254 | keyboard_row = [] 255 | except telegram.TelegramError: 256 | logger.warning('filter_by_hashtag: cannot get title of channel {}'.format(channel.channel_id)) 257 | traceback.print_exc() 258 | if len(keyboard_row) != 0: 259 | keyboard.append(keyboard_row) 260 | 261 | update.message.reply_text('Выберите канал', reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True)) 262 | 263 | return ASKED_CHANNEL_ID_IN_FILTER_BY_HASHTAG 264 | 265 | 266 | @catch_exceptions 267 | @make_db_session 268 | def filter_by_hashtag_in_state_asked_channel_id(update, context, db, users_state): 269 | user_id = update.message.from_user.id 270 | channel_title = update.message.text 271 | channel_id = str(users_state[user_id]['channels'][channel_title]) 272 | channel = db.query(Channel).get(channel_id) 273 | users_state[user_id]['channel'] = channel 274 | 275 | if channel.hashtag_filter is not None: 276 | update.message.reply_text('Текущий фильтр по хештегам:') 277 | update.message.reply_text(channel.hashtag_filter) 278 | update.message.reply_text('Напишите новые хештеги (разделяйте запятой):') 279 | 280 | return ASKED_HASHTAGS_IN_FILTER_BY_HASHTAG 281 | 282 | 283 | @catch_exceptions 284 | @make_db_session 285 | def filter_by_hashtag_in_state_asked_hashtags(update, context, db, users_state): 286 | user_id = update.message.from_user.id 287 | channel = users_state[user_id]['channel'] 288 | 289 | try: 290 | channel.hashtag_filter = ','.join(h.strip() for h in update.message.text.split(',')) 291 | db.commit() 292 | metrics.telegram_conversations_total.labels(type='filter_by_hashtag', status='completed').inc() 293 | except: 294 | db.rollback() 295 | metrics.telegram_conversations_total.labels(type='filter_by_hashtag', status='failed').inc() 296 | raise 297 | 298 | update.message.reply_text('Сохранено!') 299 | 300 | del_state(update, users_state) 301 | 302 | return ConversationHandler.END 303 | 304 | 305 | @catch_exceptions 306 | def cancel_filter_by_hashtag(update, context, users_state): 307 | metrics.telegram_conversations_total.labels(type='filter_by_hashtag', status='cancelled').inc() 308 | 309 | update.message.reply_text('Ладно', reply_markup=ReplyKeyboardRemove()) 310 | update.message.reply_text('Настроить фильтр по хештегам можно командой /filter_by_hashtag') 311 | update.message.reply_text('Команда /new настроит новый канал') 312 | 313 | del_state(update, users_state) 314 | 315 | return ConversationHandler.END 316 | 317 | 318 | @catch_exceptions 319 | @make_db_session 320 | @observe_metrics('recover') 321 | def recover(update, context, db, users_state): 322 | user_id = update.message.from_user.id 323 | 324 | metrics.telegram_conversations_total.labels(type='recover', status='started').inc() 325 | 326 | users_state[user_id] = dict() 327 | users_state[user_id]['channels'] = dict() 328 | keyboard = [] 329 | keyboard_row = [] 330 | for channel in db.query(DisabledChannel).filter(DisabledChannel.owner_id == str(user_id)).order_by(DisabledChannel.created_at.desc()): 331 | title = '{} ({})'.format(channel.vk_group_id, channel.channel_id) 332 | users_state[user_id]['channels'][title] = channel.channel_id 333 | keyboard_row.append(title) 334 | if len(keyboard_row) == 2: 335 | keyboard.append(keyboard_row) 336 | keyboard_row = [] 337 | if len(keyboard_row) != 0: 338 | keyboard.append(keyboard_row) 339 | 340 | if len(keyboard) == 0: 341 | update.message.reply_text('Нет каналов, которые можно восстановить') 342 | del_state(update, users_state) 343 | 344 | return ConversationHandler.END 345 | else: 346 | update.message.reply_text('Выберите канал', reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True)) 347 | 348 | return ASKED_CHANNEL_ID_IN_RECOVER 349 | 350 | 351 | @catch_exceptions 352 | @make_db_session 353 | def recover_in_state_asked_channel_id(update, context, db, users_state): 354 | user_id = update.message.from_user.id 355 | channel_title = update.message.text 356 | channel_id = str(users_state[user_id]['channels'][channel_title]) 357 | disabled_channel = db.query(DisabledChannel).filter(DisabledChannel.channel_id == channel_id).one() 358 | 359 | try: 360 | db.add(Channel(channel_id=disabled_channel.channel_id, 361 | vk_group_id=disabled_channel.vk_group_id, 362 | last_vk_post_id=disabled_channel.last_vk_post_id, 363 | owner_id=disabled_channel.owner_id, 364 | owner_username=disabled_channel.owner_username, 365 | hashtag_filter=disabled_channel.hashtag_filter)) 366 | db.delete(disabled_channel) 367 | db.commit() 368 | metrics.telegram_conversations_total.labels(type='recover', status='completed').inc() 369 | except: 370 | db.rollback() 371 | metrics.telegram_conversations_total.labels(type='recover', status='failed').inc() 372 | raise 373 | 374 | update.message.reply_text('Готово!') 375 | update.message.reply_text('Бот будет проверять группу каждые 15 минут') 376 | update.message.reply_text('Настроить фильтр по хештегам можно командой /filter_by_hashtag') 377 | update.message.reply_text('Команда /new настроит новый канал') 378 | 379 | del_state(update, users_state) 380 | 381 | return ConversationHandler.END 382 | 383 | 384 | @catch_exceptions 385 | def cancel_recover(update, context, users_state): 386 | metrics.telegram_conversations_total.labels(type='recover', status='cancelled').inc() 387 | 388 | update.message.reply_text('Ладно', reply_markup=ReplyKeyboardRemove()) 389 | update.message.reply_text('Команда /new настроит новый канал') 390 | 391 | del_state(update, users_state) 392 | 393 | return ConversationHandler.END 394 | --------------------------------------------------------------------------------