├── .editorconfig ├── .gitignore ├── LICENSE ├── MANiFEST.in ├── README.md ├── README000.md ├── distributed_frame_config000.py ├── function_scheduling_distributed_framework ├── .editorconfig ├── README.md ├── __init__.py ├── beggar_version_implementation │ ├── README.md │ └── beggar_redis_consumer.py ├── concurrent_pool │ ├── __init__.py │ ├── async_helper.py │ ├── async_pool_executor.py │ ├── bounded_processpoolexcutor.py │ ├── bounded_threadpoolexcutor.py │ ├── concurrent_pool_with_multi_process.py │ ├── custom_evenlet_pool_executor.py │ ├── custom_gevent_pool_executor.py │ ├── custom_threadpool_executor.py │ ├── custom_threadpool_executor000.py │ ├── readme.md │ └── single_thread_executor.py ├── constant.py ├── consumers │ ├── __init__.py │ ├── base_consumer.py │ ├── confirm_mixin.py │ ├── http_consumer.py │ ├── http_consumer000.py │ ├── httpsqs_consumer.py │ ├── kafka_consumer.py │ ├── kafka_consumer_manually_commit.py │ ├── kombu_consumer.py │ ├── local_python_queue_consumer.py │ ├── mongomq_consumer.py │ ├── mqtt_consumer.py │ ├── nats_consumer.py │ ├── nsq_consumer.py │ ├── persist_queue_consumer.py │ ├── rabbitmq_amqpstorm_consumer.py │ ├── rabbitmq_pika_consumer.py │ ├── rabbitmq_pika_consumerv0.py │ ├── rabbitmq_rabbitpy_consumer.py │ ├── redis_brpoplpush_consumer.py │ ├── redis_consumer.py │ ├── redis_consumer_ack_able.py │ ├── redis_consumer_simple.py │ ├── redis_filter.py │ ├── redis_stream_consumer.py │ ├── rocketmq_consumer.py │ ├── sqlachemy_consumer.py │ ├── tcp_consumer.py │ ├── txt_file_consumer.py │ ├── udp_consumer.py │ └── zeromq_consumer.py ├── factories │ ├── __init__.py │ ├── consumer_factory.py │ └── publisher_factotry.py ├── frame_config.py ├── function_result_web │ ├── app.py │ ├── functions.py │ ├── static │ │ ├── assets │ │ │ ├── css │ │ │ │ ├── custom.css │ │ │ │ └── jquery.mCustomScrollbar.min.css │ │ │ ├── img │ │ │ │ └── user.jpg │ │ │ └── js │ │ │ │ ├── custom.js │ │ │ │ └── jquery.mCustomScrollbar.concat.min.js │ │ ├── css │ │ │ └── style.css │ │ ├── images │ │ │ ├── bg.jpg │ │ │ ├── password.png │ │ │ ├── tick.png │ │ │ └── user.png │ │ └── js │ │ │ └── jquery-1.11.0.min.js │ └── templates │ │ ├── index.html │ │ └── login.html ├── helpers.py ├── publishers │ ├── __init__.py │ ├── base_publisher.py │ ├── confluent_kafka_publisher.py │ ├── http_publisher.py │ ├── httpsqs_publisher.py │ ├── kafka_publisher.py │ ├── kombu_publisher.py │ ├── local_python_queue_publisher.py │ ├── mongomq_publisher.py │ ├── mqtt_publisher.py │ ├── nats_publisher.py │ ├── nsq_publisher.py │ ├── persist_queue_publisher.py │ ├── rabbitmq_amqpstorm_publisher.py │ ├── rabbitmq_pika_publisher.py │ ├── rabbitmq_rabbitpy_publisher.py │ ├── redis_publisher.py │ ├── redis_publisher_lpush.py │ ├── redis_publisher_simple.py │ ├── redis_stream_publisher.py │ ├── rocketmq_publisher.py │ ├── sqla_queue_publisher.py │ ├── tcp_publisher.py │ ├── txt_file_publisher.py │ ├── udp_publisher.py │ └── zeromq_publisher.py ├── set_frame_config.py ├── timing_job │ └── __init__.py └── utils │ ├── README.md │ ├── __init__.py │ ├── apscheduler_monkey.py │ ├── bulk_operation.py │ ├── custom_pysnooper.py │ ├── decorators.py │ ├── dependency_packages │ ├── __init__.py │ └── mongomq │ │ ├── __init__.py │ │ ├── lock.py │ │ ├── mongomq.py │ │ ├── test.py │ │ └── utils.py │ ├── develop_log.py │ ├── log_manager000.py │ ├── mongo_util.py │ ├── monkey_color_log.py │ ├── mqtt_util.py │ ├── paramiko_util.py │ ├── pysnooper_ydf │ ├── __init__.py │ ├── pycompat.py │ ├── tracer.py │ ├── utils.py │ └── variables.py │ ├── rabbitmq_factory.py │ ├── redis_manager.py │ ├── resource_monitoring.py │ ├── sqla_queue.py │ └── time_util.py ├── nb_log_config.py ├── requirements.txt ├── setup.py ├── test_frame ├── best_simple_example │ ├── README.md │ ├── test_consume.py │ ├── test_publish.py │ └── test_qps_consume.py ├── car_home_crawler_sample │ ├── car_home_consumer.py │ ├── car_home_publisher.py │ ├── carhome.png │ ├── carhome2.png │ ├── carhome3.png │ └── kuaisu.png ├── complex_example │ ├── README.md │ ├── test_consume.py │ ├── test_consume2.py │ └── test_publish.py ├── d3 │ └── t6.py ├── git_fsdf.py ├── git_fsdf_github.py ├── jietu │ ├── 20210428-211001.mp4 │ ├── 20210428-2_clip.gif │ ├── 20210428-2_clip1.gif │ ├── 5种数据库模拟消息队列.png │ ├── QQ图片20190923130527.png │ ├── celery_dir2.png │ ├── celery_proj_dir.png │ ├── linuxgevent.png │ ├── linux上运行使用gevent模式的截图2a.png │ ├── pysnooper改版测试截图.png │ ├── win运行.png │ ├── ~$流程图.docx │ ├── 五彩日志.png │ ├── 任务消费统计曲线.png │ ├── 函数状态持久化.png │ ├── 函数精确控频运行完成100次每秒.png │ ├── 函数结果和运行次数和错误异常查看.png │ ├── 彩色和可跳转演示.png │ ├── 我开发时候的工具和方式.png │ ├── 操作文档图片.png │ ├── 流程图.docx │ └── 运行截图.png ├── log_example.py ├── multi_steps_conusme │ ├── README.md │ ├── multi_steps_example1.py │ └── multi_steps_example2.py ├── other_tests │ ├── test2.py │ ├── test_apscheduler.py │ ├── test_color.py │ ├── test_consume.py │ ├── test_gevent_process.py │ ├── test_import_time.py │ ├── test_monitor.py │ ├── test_nats.py │ ├── test_pulsar.py │ ├── test_pysnooper.py │ ├── test_redis_stream.py │ ├── test_zeromq │ │ ├── test_zeromq_broker.py │ │ ├── test_zeromq_client.py │ │ └── test_zeromq_server.py │ └── tests-redis_performance.py ├── test_apschedual │ ├── README.md │ ├── test_timer.py │ └── test_timer2.py ├── test_async_consumer │ ├── readme_async_test.py │ ├── test_async_consume.py │ ├── test_async_consume2.py │ ├── test_async_rpc.py │ ├── 演示指定async_loop.py │ └── 演示错误调用async函数的方式.py ├── test_auto_run_on_remote │ ├── show.py │ └── test_run.py ├── test_broker │ ├── celery_sqlite3.sqlite │ ├── celery_sqlite3x.sqlite │ ├── dssf_kombu_sqlite2.sqlite │ ├── test_consume.py │ ├── test_publish.py │ └── testtt.py ├── test_broker_kafka │ ├── kafka_cosumer_test.py │ └── kafka_publisher_test.py ├── test_celery │ ├── README.md │ ├── celery_start.bat │ ├── celerybeat-schedule.bak │ ├── celerybeat-schedule.dat │ ├── celerybeat-schedule.dir │ ├── celerybeat.pid │ ├── test_add_task.py │ └── test_celery_app.py ├── test_decorator_run_example │ ├── README.md │ ├── test_common_no_decorator_example.py │ └── test_decorator_task_example.py ├── test_delay_task │ ├── test_delay_consume.py │ └── test_delay_push.py ├── test_fabric_deploy │ └── test_deploy1.py ├── test_frame_using_thread │ ├── README.md │ ├── test_consume.py │ └── test_publish.py ├── test_instead_thread_asyncio │ ├── test_instead_thread.py │ └── test_use_thread.py ├── test_multiprocess_diffrent_linux_and_win │ └── test_start_multiprocess.py ├── test_nb_log │ ├── log_example.py │ └── nb_log_config.py ├── test_qps │ ├── qps_demo_use_frame.py │ └── qps_demo_use_threadpool.py ├── test_rabbitmq │ ├── 333.png │ ├── img.png │ ├── img_1.png │ ├── img_2.png │ ├── test_librabbitmq │ │ └── test_librabbitmq_conusme.py │ ├── test_rabbitmq_consume.py │ └── test_rabbitmq_pubilish.py ├── test_redis_ack_able │ ├── __init__.py │ ├── test_redis_ack_able_consumer.py │ └── test_redis_ack_able_publisher.py ├── test_redis_conn.py ├── test_request_baidu │ ├── baidu_consumer.py │ └── multi_aio_server.py ├── test_rpc │ ├── README.md │ ├── test_consume.py │ └── test_publish.py ├── test_socket │ ├── __init__.py │ ├── test_socket_consumer.py │ ├── test_socket_publisher.py │ ├── test_tcp_client.py │ ├── test_tcp_server.py │ ├── test_udp_server.py │ └── test_uod_client.py ├── test_speed │ ├── speed_test_consume.py │ ├── speed_test_push.py │ ├── speed_test_use_bigger_frame.py │ ├── test_baidu_consume.py │ └── test_baidu_push.py ├── test_timing │ └── test_timming.py ├── test_two_deco │ └── test_2_deco.py ├── test_use_gevent │ ├── README.md │ ├── test_consume.py │ ├── test_publish.py │ └── 运行截图.png ├── test_with_multi_process │ ├── test_consume.py │ └── test_push.py ├── test_xiaoxianrou │ ├── download_xiaoxianrou_pictures.py │ └── download_xiaoxianrou_pictures_三函数版本.py ├── tests_concurrent_pool │ ├── test_concurrent_pool1.py │ ├── test_concurrent_pool2.py │ └── test_cost_time.py ├── tttt └── use_in_flask_tonardo_fastapi │ ├── flask_api.py │ ├── readme.md │ └── run_backedn_tasks.py └── 操作文档.md /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | charset=utf-8 3 | end_of_line=crlf 4 | insert_final_newline=false 5 | indent_style=space 6 | indent_size=4 7 | max_line_length = 400 8 | 9 | 10 | [{.babelrc,.stylelintrc,.eslintrc,jest.config,*.json,*.jsb3,*.jsb2,*.bowerrc}] 11 | indent_style=space 12 | indent_size=2 13 | 14 | [*.csv] 15 | indent_style=tab 16 | tab_width=1 17 | 18 | [{jshint.json,*.jshintrc}] 19 | indent_style=space 20 | indent_size=2 21 | 22 | [{*.jscs.json,*.jscsrc}] 23 | indent_style=space 24 | indent_size=2 25 | 26 | [*.js.map] 27 | indent_style=space 28 | indent_size=2 29 | 30 | [{*.ddl,*.sql}] 31 | indent_style=space 32 | indent_size=2 33 | 34 | [{*.coffee,*.cjsx}] 35 | indent_style=space 36 | indent_size=2 37 | 38 | [{*.yml,*.yaml}] 39 | indent_style=space 40 | indent_size=2 41 | 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea/ 3 | env_hotels/ 4 | henv/ 5 | venv/ 6 | *.pyc 7 | app/apis/logs/ 8 | app/logs/ 9 | *.log.* 10 | *.log 11 | *.lock 12 | *.pytest_cache* 13 | nohup.out 14 | apidoc/ 15 | node_modules/ 16 | hotelApi/ 17 | my_patch_frame_config0000.py 18 | my_patch_frame_config_beifen.py 19 | test_frame/my_patch_frame_config.py 20 | function_result_web0000/ 21 | test_frame/my/ 22 | redis_queue_web/ 23 | test_frame/test_rabbitmq/distributed_frame_config.py 24 | dist/ 25 | *.egg-info/ 26 | not_up_git/ 27 | distributed_frame_config.py 28 | auto_run_on_remote_config.py 29 | /build/ 30 | /test_frame/test_xiaoxianrou/pictures/ 31 | /.vscode/ 32 | -------------------------------------------------------------------------------- /MANiFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | recursive-include function_scheduling_distributed_framework/function_result_web * -------------------------------------------------------------------------------- /distributed_frame_config000.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from function_scheduling_distributed_framework.constant import BrokerEnum 4 | 5 | ''' 6 | 此文件是第一次运行框架自动生成刀项目根目录的，不需要用由户手动创建。 7 | ''' 8 | 9 | ''' 10 | 1）你项目根目录下自动生成的 distributed_frame_config.py 文件中修改配置，会被自动读取到。 11 | 此文件按需修改，例如你使用redis中间件作为消息队列，可以不用管rabbitmq的配置。 12 | 13 | 2）框架使用文档是 https://function-scheduling-distributed-framework.readthedocs.io/zh_CN/latest/ 14 | 15 | 3）使用此框架时候，在同一个python项目中如何连接多个相同种类的消息队列中间件ip地址？ 16 | 见文档中的解答6.13： 17 | https://function-scheduling-distributed-framework.readthedocs.io/zh_CN/latest/articles/c6.html#pythonip 18 | 19 | ''' 20 | 21 | # 如果@task_deco装饰器没有亲自指定beoker_kind入参，则默认使用DEFAULT_BROKER_KIND这个中间件。 22 | # 强烈推荐安装rabbitmq然后使用 BrokerEnum.RABBITMQ_AMQPSTORM 这个中间件, 23 | # 次之 BrokerEnum.REDIS_ACK_ABLE中间件，kafka则推荐 BrokerEnum.CONFLUENT_KAFKA。 24 | # BrokerEnum.PERSISTQUEUE 的优点是基于单机磁盘的消息持久化，不需要安装消息中间件软件就能使用，但不是跨机器的真分布式。 25 | DEFAULT_BROKER_KIND = BrokerEnum.PERSISTQUEUE 26 | 27 | MONGO_CONNECT_URL = f'mongodb://192.168.6.134:27017' 28 | 29 | # RABBITMQ_USER = 'rabbitmq_user' 30 | # RABBITMQ_PASS = 'rabbitmq_pass' 31 | # RABBITMQ_HOST = '127.0.0.1' 32 | # RABBITMQ_PORT = 5672 33 | # RABBITMQ_VIRTUAL_HOST = 'rabbitmq_virtual_host' 34 | 35 | RABBITMQ_USER = 'admin' 36 | RABBITMQ_PASS = 'xxx' 37 | RABBITMQ_HOST = '106.xx.244.xx' 38 | RABBITMQ_PORT = 5672 39 | RABBITMQ_VIRTUAL_HOST = '/' 40 | 41 | 42 | REDIS_HOST = '127.0.0.1' 43 | REDIS_PASSWORD = '' 44 | REDIS_PORT = 6379 45 | REDIS_DB = 7 46 | 47 | NSQD_TCP_ADDRESSES = ['127.0.0.1:4150'] 48 | NSQD_HTTP_CLIENT_HOST = '127.0.0.1' 49 | NSQD_HTTP_CLIENT_PORT = 4151 50 | 51 | KAFKA_BOOTSTRAP_SERVERS = ['192.168.6.134:9092'] 52 | 53 | SQLACHEMY_ENGINE_URL = 'sqlite:////sqlachemy_queues/queues.db' 54 | 55 | # persist_quque中间件时候采用本机sqlite的方式，数据库文件生成的位置。如果linux账号在根目录没权限建文件夹，可以换文件夹。 56 | SQLLITE_QUEUES_PATH = '/sqllite_queues' 57 | 58 | ROCKETMQ_NAMESRV_ADDR = '192.168.199.202:9876' 59 | 60 | MQTT_HOST = '127.0.0.1' 61 | MQTT_TCP_PORT = 1883 62 | 63 | HTTPSQS_HOST = '127.0.0.1' 64 | HTTPSQS_PORT = '1218' 65 | HTTPSQS_AUTH = '123456' 66 | 67 | KOMBU_URL = 'redis://127.0.0.1:6379/0' 68 | # KOMBU_URL = 'sqla+sqlite:////dssf_kombu_sqlite.sqlite' # 4个//// 代表磁盘根目录下生成一个文件。推荐绝对路径。3个///是相对路径。 69 | 70 | 71 | # nb_log包的第几个日志模板，内置了7个模板，可以在你当前项目根目录下的nb_log_config.py文件扩展模板。 72 | NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = 11 # 7是简短的不可跳转，5是可点击跳转的 73 | FSDF_DEVELOP_LOG_LEVEL = 50 # 开发时候的日志，仅供我自己用，所以日志级别跳到最高，用户不需要管。 74 | 75 | TIMEZONE = 'Asia/Shanghai' 76 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | max_line_length = 400 -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/README.md: -------------------------------------------------------------------------------- 1 | 用法见README.md和test_frame的例子。 2 | 3 | 4 | 此项目停止更新，框架名字改成新的funboost。 -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/beggar_version_implementation/README.md: -------------------------------------------------------------------------------- 1 | # 这是乞丐版代码实现 2 | 3 | 主要是砍掉很多功能，大大简化代码行数，演示框架思路是如何分布式执行python 4 | 函数的，不要亲自去使用这里，功能太弱。 5 | 6 | 完整版支持3种并发类型，乞丐版只支持多线程并发。 7 | 8 | 完整版支持15种函数辅助控制，包括控频、超时杀死、消费确认 等15种功能， 9 | 乞丐版为了简化代码演示，全部不支持。 10 | 11 | 完整版支持10种消息队列中间件，这里只演示大家喜欢的redis作为中间件。 -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 9:46 4 | """ 5 | 并发池 包括 6 | 有界队列线程池 加 错误提示 7 | eventlet协程 8 | gevent协程 9 | 自定义的有界队列线程池 加 错误提示，同时线程数量在任务数量少的时候可自动减少。项目中默认使用的并发方式是基于这个。 10 | 11 | 此文件夹包括5种并发池，可以单独用于任何项目，即使没有使用这个函数调度框架。 12 | """ 13 | from .async_pool_executor import * 14 | from .custom_evenlet_pool_executor import CustomEventletPoolExecutor 15 | from .custom_gevent_pool_executor import GeventPoolExecutor 16 | from .bounded_threadpoolexcutor import BoundedThreadPoolExecutor 17 | from .custom_threadpool_executor import CustomThreadPoolExecutor -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/async_helper.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import asyncio 3 | from concurrent.futures import Executor 4 | from function_scheduling_distributed_framework.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble 5 | 6 | # 没有使用内置的concurrent.futures.ThreadpoolExecutor线程池，而是使用智能伸缩线程池。 7 | async_executor_default = ThreadPoolExecutorShrinkAble() 8 | 9 | 10 | async def simple_run_in_executor(f, *args, async_executor: Executor = None, async_loop=None, **kwargs): 11 | """ 12 | 一个很强的函数，使任意同步同步函数f，转化成asyncio异步api语法， 13 | 例如 r = await simple_run_in_executor(block_fun, 20)，可以不阻塞事件循环。 14 | 15 | asyncio.run_coroutine_threadsafe 和 run_in_executor 是一对反义词。 16 | 17 | asyncio.run_coroutine_threadsafe 是在非异步的上下文环境(也就是正常的同步语法的函数里面)下调用异步函数对象（协程）， 18 | 因为当前函数定义没有被async修饰，就不能在函数里面使用await，必须使用这。这个是将asyncio包的future对象转化返回一个concurrent.futures包的future对象。 19 | 20 | run_in_executor 是在异步环境（被async修饰的异步函数）里面，调用同步函数，将函数放到线程池运行防止阻塞整个事件循环的其他任务。 21 | 这个是将 一个concurrent.futures包的future对象 转化为 asyncio包的future对象， 22 | asyncio包的future对象是一个asyncio包的awaitable对象，所以可以被await，concurrent.futures.Future对象不能被await。 23 | 24 | 25 | :param f: f是一个同步的阻塞函数，f前面不能是由async定义的。 26 | :param args: f函数的位置方式入参 27 | :async_executor: 线程池 28 | :param async_loop: async的loop对象 29 | :param kwargs:f函数的关键字方式入参 30 | :return: 31 | """ 32 | loopx = async_loop or asyncio.get_event_loop() 33 | async_executorx = async_executor or async_executor_default 34 | # print(id(loopx)) 35 | result = await loopx.run_in_executor(async_executorx, partial(f, *args, **kwargs)) 36 | return result 37 | 38 | 39 | if __name__ == '__main__': 40 | import time 41 | import requests 42 | 43 | 44 | def block_fun(x): 45 | print(x) 46 | time.sleep(5) 47 | return x * 10 48 | 49 | 50 | async def enter_fun(xx): # 入口函数，模拟一旦异步，必须处处异步。不能直接调用block_fun，否则阻塞其他任务。 51 | await asyncio.sleep(1) 52 | # r = block_fun(xx) # 如果这么用就完蛋了，阻塞事件循环， 运行完所有任务需要更久。 53 | r = await simple_run_in_executor(block_fun, xx) 54 | print(r) 55 | 56 | 57 | loopy = asyncio.get_event_loop() 58 | print(id(loopy)) 59 | tasks = [] 60 | tasks.append(simple_run_in_executor(requests.get, url='http://www.baidu.com', timeout=10)) # 同步变异步用法。 61 | 62 | tasks.append(simple_run_in_executor(block_fun, 1)) 63 | tasks.append(simple_run_in_executor(block_fun, 2)) 64 | tasks.append(simple_run_in_executor(block_fun, 3)) 65 | tasks.append(simple_run_in_executor(time.sleep, 8)) 66 | 67 | tasks.append(enter_fun(4)) 68 | tasks.append(enter_fun(5)) 69 | tasks.append(enter_fun(6)) 70 | 71 | print('开始') 72 | loopy.run_until_complete(asyncio.wait(tasks)) 73 | print('结束') 74 | 75 | time.sleep(200) 76 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/bounded_threadpoolexcutor.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | 一个有界任务队列的thradpoolexcutor 4 | 直接捕获错误日志 5 | """ 6 | from functools import wraps 7 | import queue 8 | from concurrent.futures import ThreadPoolExecutor, Future 9 | # noinspection PyProtectedMember 10 | from concurrent.futures.thread import _WorkItem 11 | 12 | from nb_log import LogManager 13 | 14 | logger = LogManager('BoundedThreadPoolExecutor').get_logger_and_add_handlers() 15 | 16 | 17 | def _deco(f): 18 | @wraps(f) 19 | def __deco(*args, **kwargs): 20 | try: 21 | return f(*args, **kwargs) 22 | except Exception as e: 23 | logger.exception(e) 24 | 25 | return __deco 26 | 27 | 28 | class BoundedThreadPoolExecutor(ThreadPoolExecutor, ): 29 | def __init__(self, max_workers=None, thread_name_prefix=''): 30 | ThreadPoolExecutor.__init__(self, max_workers, thread_name_prefix) 31 | self._work_queue = queue.Queue(max_workers * 2) 32 | 33 | def submit(self, fn, *args, **kwargs): 34 | with self._shutdown_lock: 35 | if self._shutdown: 36 | raise RuntimeError('cannot schedule new futures after shutdown') 37 | f = Future() 38 | fn_deco = _deco(fn) 39 | w = _WorkItem(f, fn_deco, args, kwargs) 40 | self._work_queue.put(w) 41 | self._adjust_thread_count() 42 | return f 43 | 44 | 45 | if __name__ == '__main__': 46 | def fun(): 47 | print(1 / 0) 48 | 49 | 50 | pool = BoundedThreadPoolExecutor(10) 51 | pool.submit(fun) 52 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/concurrent_pool_with_multi_process.py: -------------------------------------------------------------------------------- 1 | import time 2 | import multiprocessing 3 | import threading 4 | import asyncio 5 | import nb_log 6 | import atexit 7 | import os 8 | import typing 9 | from function_scheduling_distributed_framework.concurrent_pool.custom_threadpool_executor import CustomThreadpoolExecutor 10 | from function_scheduling_distributed_framework.concurrent_pool.custom_gevent_pool_executor import GeventPoolExecutor 11 | from function_scheduling_distributed_framework.concurrent_pool.custom_evenlet_pool_executor import CustomEventletPoolExecutor 12 | from function_scheduling_distributed_framework.concurrent_pool.async_pool_executor import AsyncPoolExecutor 13 | 14 | 15 | class ConcurrentPoolWithProcess(nb_log.LoggerMixin): 16 | def _start_a_pool(self, pool_class, max_works): 17 | pool = pool_class(max_works) 18 | while True: 19 | func, args, kwargs = self._multi_process_queue.get() # 结束可以放None，然后这里判断，终止。或者joinable queue 20 | print(func, args, kwargs) 21 | pool.submit(func, *args, **kwargs) 22 | 23 | def __init__(self, pool_class: typing.Type = CustomThreadpoolExecutor, max_works=500, process_num=1): 24 | self._multi_process_queue = multiprocessing.Queue(100) 25 | for _ in range(process_num): 26 | multiprocessing.Process(target=self._start_a_pool, args=(pool_class, max_works), daemon=False).start() 27 | 28 | # noinspection PyUnusedLocal 29 | def _queue_call_back(self, result): 30 | self._multi_process_queue.task_done() 31 | 32 | def submit(self, func, *args, **kwargs): 33 | self._multi_process_queue.put((func, args, kwargs)) 34 | 35 | def shutdown(self, wait=True): 36 | pass 37 | 38 | 39 | def test_f(x): 40 | time.sleep(1) 41 | print(x * 10, os.getpid()) 42 | 43 | 44 | async def async_f(x): 45 | await asyncio.sleep(1) 46 | print(x * 10) 47 | 48 | 49 | if __name__ == '__main__': 50 | pool = ConcurrentPoolWithProcess(AsyncPoolExecutor, 20, 2) 51 | # pool = GeventPoolExecutor(200,) 52 | 53 | # time.sleep(15) 54 | for i in range(1000): 55 | time.sleep(0.1) 56 | pool.submit(async_f, i) 57 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/custom_evenlet_pool_executor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/7/3 10:35 4 | import atexit 5 | import time 6 | import warnings 7 | from eventlet import greenpool, monkey_patch, patcher, Timeout 8 | 9 | from nb_log import LogManager, nb_print 10 | 11 | 12 | def check_evenlet_monkey_patch(raise_exc=True): 13 | if not patcher.is_monkey_patched('socket'): # 随便选一个检测标志 14 | if raise_exc: 15 | warnings.warn(f'检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True) ') 16 | raise Exception('检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True)') 17 | else: 18 | return 1 19 | 20 | 21 | logger_evenlet_timeout_deco = LogManager('evenlet_timeout_deco').get_logger_and_add_handlers() 22 | 23 | 24 | def evenlet_timeout_deco(timeout_t): 25 | def _evenlet_timeout_deco(f): 26 | def __evenlet_timeout_deco(*args, **kwargs): 27 | timeout = Timeout(timeout_t, ) 28 | # timeout.start() # 与gevent不一样,直接start了。 29 | result = None 30 | try: 31 | result = f(*args, **kwargs) 32 | except Timeout as t: 33 | logger_evenlet_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒') 34 | if t is not timeout: 35 | nb_print(t) 36 | # raise # not my timeout 37 | finally: 38 | timeout.cancel() 39 | return result 40 | 41 | return __evenlet_timeout_deco 42 | 43 | return _evenlet_timeout_deco 44 | 45 | 46 | class CustomEventletPoolExecutor(greenpool.GreenPool): 47 | def __init__(self, *args, **kwargs): 48 | super().__init__(*args, **kwargs) 49 | check_evenlet_monkey_patch() # basecomer.py中检查。 50 | atexit.register(self.shutdown) 51 | 52 | def submit(self, *args, **kwargs): # 保持为一直的公有用法。 53 | # nb_print(args) 54 | self.spawn_n(*args, **kwargs) 55 | # self.spawn_n(*args, **kwargs) 56 | 57 | def shutdown(self): 58 | self.waitall() 59 | 60 | 61 | if __name__ == '__main__': 62 | # greenpool.GreenPool.waitall() 63 | monkey_patch(all=True) 64 | 65 | 66 | def f2(x): 67 | 68 | time.sleep(2) 69 | nb_print(x) 70 | 71 | 72 | pool = CustomEventletPoolExecutor(4) 73 | 74 | for i in range(15): 75 | nb_print(f'放入{i}') 76 | pool.submit(evenlet_timeout_deco(8)(f2), i) 77 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/readme.md: -------------------------------------------------------------------------------- 1 | #### 这个里面是实现各种并发池，框架使用不同种类的并发池从而使用不同的并发模式来执行函数任务。 2 | 3 | 4 | ```python 5 | 6 | ''' 7 | 各种并发池的api都实现了submit，然后就自动执行函数。类似concurrent.futures包的api 8 | ''' 9 | 10 | 11 | def fun(x): 12 | print(x) 13 | 14 | pool = Pool(50) 15 | pool.submit(fun,1) 16 | 17 | 18 | ``` 19 | 20 | ``` 21 | 实现的池包括 22 | 23 | 24 | gevent 25 | 26 | eventlet 27 | 28 | asyncio 29 | 30 | custom_threadpool_executor.py 可变有界线程池,可变是指线程池嫩自动扩大，最厉害的是能自动缩小线程数量，官方不具备此功能。 31 | 32 | 33 | ``` -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/concurrent_pool/single_thread_executor.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | 4 | class SoloExecutor: 5 | # noinspection PyUnusedLocal 6 | def __init__(self, max_workers: int = 1): 7 | pass 8 | 9 | # noinspection PyMethodMayBeStatic 10 | def submit(self, fn: Callable, *args, **kwargs): 11 | return fn(*args, **kwargs) 12 | 13 | # noinspection PyMethodMayBeStatic 14 | def shutdown(self, wait=True): 15 | pass 16 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/constant.py: -------------------------------------------------------------------------------- 1 | # coding= utf-8 2 | class BrokerEnum: 3 | RABBITMQ_AMQPSTORM = 0 # 使用 amqpstorm 包操作rabbitmq 作为 分布式消息队列，支持消费确认.推荐这个。 4 | 5 | RABBITMQ_RABBITPY = 1 # 使用 rabbitpy 包操作rabbitmq 作为 分布式消息队列，支持消费确认。 6 | 7 | REDIS = 2 # 使用 redis 的 list结构，brpop 作为分布式消息队列。随意重启和关闭会丢失大量消息，不支持消费确认。注重性能不在乎丢失消息可以选这个redis方案。 8 | 9 | LOCAL_PYTHON_QUEUE = 3 # 使用python queue.Queue实现的基于当前python进程的消息队列，不支持跨进程 跨脚本 跨机器共享任务，不支持持久化，适合一次性短期简单任务。 10 | MEMORY_QUEUE = LOCAL_PYTHON_QUEUE # 别名，python本地queue就是基于python自带的语言的queue.Queue，消息存在python程序的内存中，不支持重启断点接续。 11 | 12 | RABBITMQ_PIKA = 4 # 使用pika包操作rabbitmq 作为 分布式消息队列。 13 | 14 | MONGOMQ = 5 # 使用mongo的表中的行模拟的 作为分布式消息队列，支持消费确认。 15 | 16 | PERSISTQUEUE = 6 # 使用基于sqlite3模拟消息队列，支持消费确认和持久化，但不支持跨机器共享任务，可以基于本机单机跨脚本和跨进程共享任务，好处是不需要安装中间件。 17 | SQLITE_QUEUE = PERSISTQUEUE # PERSISTQUEUE的别名 18 | 19 | NSQ = 7 # 基于nsq作为分布式消息队列，支持消费确认。 20 | 21 | KAFKA = 8 # 基于kafka作为分布式消息队列，如果随意重启会丢失消息，建议使用BrokerEnum.CONFLUENT_KAFKA。 22 | 23 | """基于confluent-kafka包，包的性能比kafka-python提升10倍。同时应对反复随意重启部署消费代码的场景，此消费者实现至少消费一次，第8种BrokerEnum.KAFKA是最多消费一次。""" 24 | CONFLUENT_KAFKA = 16 25 | KAFKA_CONFLUENT = CONFLUENT_KAFKA 26 | 27 | REDIS_ACK_ABLE = 9 # 基于redis的 list + 临时unack的set队列，采用了 lua脚本操持了取任务和加到pengding为原子性，随意重启和掉线不会丢失任务。 28 | 29 | SQLACHEMY = 10 # 基于SQLACHEMY 的连接作为分布式消息队列中间件支持持久化和消费确认。支持mysql oracle sqlserver等5种数据库。 30 | 31 | ROCKETMQ = 11 # 基于 rocketmq 作为分布式消息队列，这个中间件必须在linux下运行，win不支持。 32 | 33 | REDIS_STREAM = 12 # 基于redis 5.0 版本以后，使用 stream 数据结构作为分布式消息队列，支持消费确认和持久化和分组消费，是redis官方推荐的消息队列形式，比list结构更适合。 34 | 35 | ZEROMQ = 13 # 基于zeromq作为分布式消息队列，不需要安装中间件，可以支持跨机器但不支持持久化。 36 | 37 | RedisBrpopLpush = 14 # 基于redis的list结构但是采用brpoplpush 双队列形式，和 redis_ack_able的实现差不多，实现上采用了原生命令就不需要lua脚本来实现取出和加入unack了。 38 | 39 | """ 40 | 操作 kombu 包，这个包也是celery的中间件依赖包，这个包可以操作10种中间件(例如rabbitmq redis)，但没包括分布式函数调度框架的kafka nsq zeromq 等。 41 | 同时 kombu 包的性能非常差，可以用原生redis的lpush和kombu的publish测试发布，使用brpop 和 kombu 的 drain_events测试消费，对比差距相差了5到10倍。 42 | 由于性能差，除非是分布式函数调度框架没实现的中间件才选kombu方式(例如kombu支持亚马逊队列 qpid pyro 队列)，否则强烈建议使用此框架的操作中间件方式而不是使用kombu。 43 | """ 44 | KOMBU = 15 45 | 46 | """ 基于emq作为中间件的。这个和上面的中间件有很大不同，服务端不存储消息。所以不能先发布几十万个消息，然后再启动消费。mqtt优点是web前后端能交互， 47 | 前端不能操作redis rabbitmq kafka，但很方便操作mqtt。这种使用场景是高实时的互联网接口。 48 | """ 49 | MQTT = 17 50 | 51 | HTTPSQS = 18 # httpsqs中间件实现的，基于http协议操作，dcoker安装此中间件简单。 52 | 53 | UDP = 21 # 基于socket udp 实现的，需要先启动消费端再启动发布，支持分布式但不支持持久化，好处是不需要安装消息队列中间件软件。 54 | 55 | TCP = 22 # 基于socket tcp 实现的，需要先启动消费端再启动发布，支持分布式但不支持持久化，好处是不需要安装消息队列中间件软件。 56 | 57 | HTTP = 23 # 基于http实现的，发布使用的urllib3，消费服务端使用的aiohttp.server实现的，支持分布式但不支持持久化，好处是不需要安装消息队列中间件软件。 58 | 59 | NATS = 24 # 高性能中间件nats,中间件服务端性能很好,。 60 | 61 | TXT_FILE = 25 # 磁盘txt文件作为消息队列，支持单机持久化，不支持多机分布式 62 | 63 | PULSAR = 20 # 最有潜力的下一代分布式消息系统。5年后会同时取代rabbitmq和kafka。目前python客户端只支持linux，win不行 64 | 65 | 66 | class ConcurrentModeEnum: 67 | THREADING = 1 68 | GEVENT = 2 69 | EVENTLET = 3 70 | ASYNC = 4 # asyncio并发，适用于async def定义的函数。 71 | SINGLE_THREAD = 5 72 | 73 | # is_fsdf_remote_run = 0 74 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:09 4 | """ 5 | 实现基于各种中间件的消费者 6 | """ -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/http_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import asyncio 5 | import json 6 | 7 | from aiohttp import web 8 | from aiohttp.web_request import Request 9 | 10 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 11 | 12 | 13 | class HTTPConsumer(AbstractConsumer, ): 14 | """ 15 | http 实现消息队列，不支持持久化，但不需要安装软件。 16 | """ 17 | BROKER_KIND = 23 18 | 19 | # noinspection PyAttributeOutsideInit 20 | def custom_init(self): 21 | self._ip, self._port = self.queue_name.split(':') 22 | self._port = int(self._port) 23 | 24 | # noinspection DuplicatedCode 25 | def _shedual_task(self): 26 | # flask_app = Flask(__name__) 27 | # 28 | # @flask_app.route('/queue', methods=['post']) 29 | # def recv_msg(): 30 | # msg = request.form['msg'] 31 | # kw = {'body': json.loads(msg)} 32 | # self._submit_task(kw) 33 | # return 'finish' 34 | # 35 | # flask_app.run('0.0.0.0', port=self._port,debug=False) 36 | 37 | routes = web.RouteTableDef() 38 | 39 | # noinspection PyUnusedLocal 40 | @routes.get('/') 41 | async def hello(request): 42 | return web.Response(text="Hello, from function_scheduling_distributed_framework") 43 | 44 | @routes.post('/queue') 45 | async def recv_msg(request: Request): 46 | data = await request.post() 47 | msg = data['msg'] 48 | kw = {'body': json.loads(msg)} 49 | self._submit_task(kw) 50 | return web.Response(text="finish") 51 | 52 | app = web.Application() 53 | app.add_routes(routes) 54 | loop = asyncio.new_event_loop() 55 | asyncio.set_event_loop(loop) 56 | web.run_app(app, host='0.0.0.0', port=self._port, ) 57 | 58 | def _confirm_consume(self, kw): 59 | pass # 没有确认消费的功能。 60 | 61 | def _requeue(self, kw): 62 | pass 63 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/httpsqs_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | import time 6 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 7 | from function_scheduling_distributed_framework.publishers.httpsqs_publisher import HttpsqsPublisher 8 | 9 | 10 | class HttpsqsConsumer(AbstractConsumer): 11 | """ 12 | httpsqs作为中间件 13 | """ 14 | BROKER_KIND = 18 15 | 16 | def custom_init(self): 17 | # noinspection PyAttributeOutsideInit 18 | self.httpsqs_publisher = HttpsqsPublisher(self._queue_name) 19 | 20 | # noinspection DuplicatedCode 21 | def _shedual_task(self): 22 | while True: 23 | text = self.httpsqs_publisher.opt_httpsqs('get') 24 | if text == 'HTTPSQS_GET_END': 25 | time.sleep(0.5) 26 | else: 27 | kw = {'body': json.loads(text)} 28 | self._submit_task(kw) 29 | 30 | def _confirm_consume(self, kw): 31 | pass 32 | 33 | def _requeue(self, kw): 34 | try: 35 | kw['body'].pop('extra') 36 | except KeyError: 37 | pass 38 | self.httpsqs_publisher.publish(kw['body']) 39 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/local_python_queue_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:36 4 | import json 5 | from queue import Queue 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework.publishers import local_python_queue_publisher 9 | 10 | 11 | class LocalPythonQueueConsumer(AbstractConsumer): 12 | """ 13 | python 内置queue对象作为消息队列，这个要求发布和消费必须在同一python解释器内部运行，不支持分布式。 14 | """ 15 | BROKER_KIND = 3 16 | 17 | @property 18 | def local_python_queue(self) -> Queue: 19 | return local_python_queue_publisher.local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] 20 | 21 | def _shedual_task(self): 22 | while True: 23 | task = self.local_python_queue.get() 24 | if isinstance(task, str): 25 | task = json.loads(task) 26 | self._print_message_get_from_broker('当前python解释器内部', task) 27 | # self.logger.debug(f'从当前python解释器内部的 [{self._queue_name}] 队列中 取出的消息是： {json.dumps(task)} ') 28 | kw = {'body': task} 29 | self._submit_task(kw) 30 | 31 | def _confirm_consume(self, kw): 32 | pass 33 | 34 | def _requeue(self, kw): 35 | self.local_python_queue.put(kw['body']) 36 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/mongomq_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:33 4 | import time 5 | 6 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 7 | from function_scheduling_distributed_framework.publishers.mongomq_publisher import MongoMixin, MongoMqPublisher 8 | 9 | 10 | class MongoMqConsumer(AbstractConsumer, MongoMixin): 11 | """ 12 | Mongo queue包实现的基于mongo的消息队列，支持消费确认。 13 | """ 14 | BROKER_KIND = 5 15 | 16 | def _shedual_task(self): 17 | mp = MongoMqPublisher(self.queue_name) 18 | while True: 19 | job = mp.queue.next() 20 | if job is not None: 21 | # self.logger.debug(f'从mongo的 [{self._queue_name}] 队列中 取出的消息是： 消息是： {job.payload} ') 22 | self._print_message_get_from_broker('mongo', job.payload) 23 | kw = {'body': job.payload, 'job': job} 24 | self._submit_task(kw) 25 | else: 26 | time.sleep(0.1) 27 | 28 | def _confirm_consume(self, kw): 29 | kw['job'].complete() 30 | 31 | def _requeue(self, kw): 32 | kw['job'].release() 33 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/mqtt_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | # import time 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework import frame_config 9 | import paho.mqtt.client as mqtt 10 | 11 | 12 | class MqttConsumer(AbstractConsumer): 13 | """ 14 | emq 作为中间件 实现的消费者 ，使用共享订阅。 15 | """ 16 | BROKER_KIND = 17 17 | 18 | # noinspection PyAttributeOutsideInit 19 | def custom_init(self): 20 | # fsdf 表示 function_scheduling_distributed_framework.相当于kafka的消费者组作用。 21 | # 这个是共享订阅，见 https://blog.csdn.net/emqx_broker/article/details/103027813 22 | self._topic_shared = f'$share/fsdf/{self._queue_name}' 23 | 24 | # noinspection DuplicatedCode 25 | def _shedual_task(self): 26 | client = mqtt.Client() 27 | # client.username_pw_set('admin', password='public') 28 | client.on_connect = self._on_connect 29 | client.on_message = self._on_message 30 | client.on_disconnect = self._on_socket_close 31 | client.on_socket_close = self._on_socket_close 32 | client.connect(frame_config.MQTT_HOST, frame_config.MQTT_TCP_PORT, 600) # 600为keepalive的时间间隔 33 | client.subscribe(self._topic_shared, qos=0) # on message 是异把消息丢到线程池，本身不可能失败。 34 | client.loop_forever(retry_first_connection=True) # 保持连接 35 | 36 | def _on_socket_close(self, client, userdata, socket): 37 | self.logger.critical(f'{client, userdata, socket}') 38 | self._shedual_task() 39 | 40 | # noinspection PyPep8Naming 41 | def _on_disconnect(self, client, userdata, reasonCode, properties): 42 | self.logger.critical(f'{client, userdata, reasonCode, properties}') 43 | 44 | def _on_connect(self, client, userdata, flags, rc): 45 | self.logger.info(f'连接mqtt服务端成功, {client, userdata, flags, rc}') 46 | 47 | # noinspection PyUnusedLocal 48 | def _on_message(self, client, userdata, msg): 49 | # print(msg.topic + " " + str(msg.payload)) 50 | kw = {'body': json.loads(msg.payload)} 51 | self._submit_task(kw) 52 | 53 | def _confirm_consume(self, kw): 54 | pass 55 | 56 | def _requeue(self, kw): 57 | pass 58 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/nats_consumer.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pynats import NATSClient, NATSMessage # noqa 3 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 4 | from function_scheduling_distributed_framework import frame_config 5 | 6 | 7 | class NatsConsumer(AbstractConsumer): 8 | """ 9 | nats作为中间件实现的。 10 | """ 11 | BROKER_KIND = 24 12 | 13 | def _shedual_task(self): 14 | # print(88888888888888) 15 | nats_client = NATSClient(frame_config.NATS_URL,socket_timeout=600,socket_keepalive=True) 16 | nats_client.connect() 17 | 18 | def callback(msg: NATSMessage): 19 | # print(type(msg)) 20 | # print(msg.reply) 21 | # print(f"Received a message with subject {msg.subject}: {msg.payload}") 22 | kw = {'body': json.loads(msg.payload)} 23 | self._submit_task(kw) 24 | 25 | nats_client.subscribe(subject=self.queue_name, callback=callback) 26 | nats_client.wait() 27 | 28 | def _confirm_consume(self, kw): 29 | pass # 没有确认消费 30 | 31 | def _requeue(self, kw): 32 | self.publisher_of_same_queue.publish(kw['body']) 33 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/nsq_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | from gnsq import Consumer, Message 6 | 7 | from function_scheduling_distributed_framework import frame_config 8 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 9 | from nb_log import LogManager 10 | 11 | LogManager('gnsq').get_logger_and_add_handlers(20) 12 | 13 | 14 | class NsqConsumer(AbstractConsumer): 15 | """ 16 | nsq作为中间件实现的。 17 | """ 18 | BROKER_KIND = 7 19 | 20 | def _shedual_task(self): 21 | consumer = Consumer(self._queue_name, 'frame_channel', frame_config.NSQD_TCP_ADDRESSES, 22 | max_in_flight=self._concurrent_num, heartbeat_interval=60, timeout=600, ) # heartbeat_interval 不能设置为600 23 | 24 | @consumer.on_message.connect 25 | def handler(consumerx: Consumer, message: Message): 26 | # 第一条消息不能并发，第一条消息之后可以并发。 27 | self._print_message_get_from_broker('nsq', message.body.decode()) 28 | # self.logger.debug(f'从nsq的 [{self._queue_name}] 主题中 取出的消息是： {message.body.decode()}') 29 | message.enable_async() 30 | kw = {'consumer': consumerx, 'message': message, 'body': json.loads(message.body)} 31 | self._submit_task(kw) 32 | 33 | consumer.start() 34 | 35 | def _confirm_consume(self, kw): 36 | kw['message'].finish() 37 | 38 | def _requeue(self, kw): 39 | kw['message'].requeue() 40 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/persist_queue_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:35 4 | import json 5 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 6 | from function_scheduling_distributed_framework.publishers.persist_queue_publisher import PersistQueuePublisher 7 | 8 | 9 | class PersistQueueConsumer(AbstractConsumer): 10 | """ 11 | persist queue包实现的本地持久化消息队列。 12 | """ 13 | BROKER_KIND = 6 14 | 15 | def _shedual_task(self): 16 | pub = PersistQueuePublisher(self.queue_name) 17 | while True: 18 | item = pub.queue.get() 19 | # self.logger.debug(f'从本地持久化sqlite的 [{self._queue_name}] 队列中 取出的消息是： {item} ') 20 | self._print_message_get_from_broker('本地持久化sqlite', item) 21 | kw = {'body': json.loads(item), 'q': pub.queue, 'item': item} 22 | self._submit_task(kw) 23 | 24 | def _confirm_consume(self, kw): 25 | kw['q'].ack(kw['item']) 26 | 27 | def _requeue(self, kw): 28 | kw['q'].nack(kw['item']) 29 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/rabbitmq_amqpstorm_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:30 4 | import json 5 | import amqpstorm 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework.publishers.rabbitmq_amqpstorm_publisher import RabbitmqPublisherUsingAmqpStorm 9 | 10 | 11 | class RabbitmqConsumerAmqpStorm(AbstractConsumer): 12 | """ 13 | 使用AmqpStorm实现的，多线程安全的，不用加锁。 14 | """ 15 | BROKER_KIND = 0 16 | 17 | def _shedual_task(self): 18 | # noinspection PyTypeChecker 19 | def callback(amqpstorm_message: amqpstorm.Message): 20 | body = amqpstorm_message.body 21 | # self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是： {body}') 22 | self._print_message_get_from_broker('rabbitmq', body) 23 | body = json.loads(body) 24 | kw = {'amqpstorm_message': amqpstorm_message, 'body': body} 25 | self._submit_task(kw) 26 | 27 | rp = RabbitmqPublisherUsingAmqpStorm(self.queue_name) 28 | rp.init_broker() 29 | rp.channel_wrapper_by_ampqstormbaic.qos(self._concurrent_num) 30 | rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False) 31 | rp.channel.start_consuming(auto_decode=True) 32 | 33 | def _confirm_consume(self, kw): 34 | # noinspection PyBroadException 35 | try: 36 | kw['amqpstorm_message'].ack() # 确认消费 37 | except Exception as e: 38 | self.logger.error(f'AmqpStorm确认消费失败 {type(e)} {e}') 39 | 40 | def _requeue(self, kw): 41 | kw['amqpstorm_message'].nack(requeue=True) 42 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/rabbitmq_rabbitpy_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:31 4 | import json 5 | import rabbitpy 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework.utils.rabbitmq_factory import RabbitMqFactory 9 | 10 | 11 | class RabbitmqConsumerRabbitpy(AbstractConsumer): 12 | """ 13 | 使用rabbitpy实现的 14 | """ 15 | BROKER_KIND = 1 16 | 17 | def _shedual_task(self): 18 | # noinspection PyTypeChecker 19 | channel = RabbitMqFactory(is_use_rabbitpy=1).get_rabbit_cleint().creat_a_channel() # type: rabbitpy.AMQP # 20 | channel.queue_declare(queue=self._queue_name, durable=True) 21 | channel.basic_qos(prefetch_count=self._concurrent_num) 22 | for message in channel.basic_consume(self._queue_name, no_ack=False): 23 | body = message.body.decode() 24 | # self.logger.debug(f'从rabbitmq {self._queue_name} 队列中 取出的消息是： {body}') 25 | self._print_message_get_from_broker('rabbitmq', body) 26 | kw = {'message': message, 'body': json.loads(message.body.decode())} 27 | self._submit_task(kw) 28 | 29 | def _confirm_consume(self, kw): 30 | kw['message'].ack() 31 | 32 | def _requeue(self, kw): 33 | kw['message'].nack(requeue=True) 34 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/redis_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | # import time 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework.utils import RedisMixin 9 | 10 | 11 | class RedisConsumer(AbstractConsumer, RedisMixin): 12 | """ 13 | redis作为中间件实现的，使用redis list 结构实现的。 14 | 这个如果消费脚本在运行时候随意反复重启或者非正常关闭或者消费宕机，会丢失大批任务。高可靠需要用rabbitmq或者redis_ack_able或者redis_stream的中间件方式。 15 | """ 16 | BROKER_KIND = 2 17 | 18 | # noinspection DuplicatedCode 19 | def _shedual_task000(self): 20 | while True: 21 | result = self.redis_db_frame.blpop(self._queue_name, timeout=60) 22 | if result: 23 | # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是： {result[1].decode()} ') 24 | self._print_message_get_from_broker('reids', result[1].decode()) 25 | task_dict = json.loads(result[1]) 26 | kw = {'body': task_dict} 27 | self._submit_task(kw) 28 | 29 | # noinspection DuplicatedCode 30 | def _shedual_task(self): 31 | while True: 32 | with self.redis_db_frame_version3.pipeline() as p: 33 | get_msg_batch_size = 100 34 | p.lrange(self._queue_name, 0, get_msg_batch_size - 1) 35 | p.ltrim(self._queue_name, get_msg_batch_size, -1) 36 | task_str_list = p.execute()[0] 37 | if task_str_list: 38 | # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是： {task_str_list} ') 39 | self._print_message_get_from_broker('redis', task_str_list) 40 | for task_str in task_str_list: 41 | kw = {'body': json.loads(task_str)} 42 | self._submit_task(kw) 43 | else: 44 | result = self.redis_db_frame.brpop(self._queue_name, timeout=60) 45 | if result: 46 | # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是： {result[1].decode()} ') 47 | self._print_message_get_from_broker('redis', result[1].decode()) 48 | task_dict = json.loads(result[1]) 49 | kw = {'body': task_dict} 50 | self._submit_task(kw) 51 | 52 | def _confirm_consume(self, kw): 53 | pass # redis没有确认消费的功能。 54 | 55 | def _requeue(self, kw): 56 | self.redis_db_frame.rpush(self._queue_name, json.dumps(kw['body'])) 57 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/redis_consumer_simple.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 6 | from function_scheduling_distributed_framework.utils import RedisMixin 7 | 8 | 9 | class RedisConsumer(AbstractConsumer, RedisMixin): 10 | """ 11 | redis作为中间件实现的。 12 | """ 13 | BROKER_KIND = 2 14 | 15 | def _shedual_task(self): 16 | while True: 17 | result = self.redis_db_frame.blpop(self._queue_name,timeout=60) 18 | if result: 19 | self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是： {result[1].decode()} ') 20 | task_dict = json.loads(result[1]) 21 | kw = {'body': task_dict} 22 | self._submit_task(kw) 23 | 24 | def _confirm_consume(self, kw): 25 | pass # redis没有确认消费的功能。 26 | 27 | def _requeue(self, kw): 28 | self.redis_db_frame.rpush(self._queue_name, json.dumps(kw['body'])) 29 | 30 | 31 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/rocketmq_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2020/7/8 0008 13:27 4 | import json 5 | import time 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework import frame_config 9 | from function_scheduling_distributed_framework.publishers.rocketmq_publisher import RocketmqPublisher 10 | 11 | 12 | class RocketmqConsumer(AbstractConsumer): 13 | """ 14 | 安装 15 | """ 16 | BROKER_KIND = 11 17 | 18 | def _shedual_task(self): 19 | try: 20 | from rocketmq.client import PushConsumer 21 | except Exception as e: 22 | # print(traceback.format_exc()) 23 | raise ImportError(f'rocketmq包 只支持linux和mac {e}') 24 | consumer = PushConsumer(f'g-{self._queue_name}') 25 | consumer.set_namesrv_addr(frame_config.ROCKETMQ_NAMESRV_ADDR) 26 | consumer.set_thread_count(1) 27 | consumer.set_message_batch_max_size(self._concurrent_num) 28 | 29 | self._publisher = RocketmqPublisher(self._queue_name) 30 | 31 | def callback(rocketmq_msg): 32 | # self.logger.debug(f'从rocketmq的 [{self._queue_name}] 主题的queue_id {rocketmq_msg.queue_id} 中 取出的消息是：{rocketmq_msg.body}') 33 | self._print_message_get_from_broker('rocketmq', rocketmq_msg.body) 34 | kw = {'body': json.loads(rocketmq_msg.body), 'rocketmq_msg': rocketmq_msg} 35 | self._submit_task(kw) 36 | 37 | consumer.subscribe(self._queue_name, callback) 38 | consumer.start() 39 | 40 | while True: 41 | time.sleep(3600) 42 | 43 | def _confirm_consume(self, kw): 44 | pass 45 | 46 | def _requeue(self, kw): 47 | self._publisher.publish(kw['body']) 48 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/sqlachemy_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:33 4 | import json 5 | 6 | from function_scheduling_distributed_framework import frame_config 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | from function_scheduling_distributed_framework.utils import sqla_queue 9 | 10 | 11 | class SqlachemyConsumer(AbstractConsumer): 12 | """ 13 | sqlachemy实现的操作5种数据库模拟消息队列，支持消费确认。 14 | """ 15 | BROKER_KIND = 10 16 | 17 | def _shedual_task(self): 18 | self.queue = sqla_queue.SqlaQueue(self._queue_name, frame_config.SQLACHEMY_ENGINE_URL) 19 | while True: 20 | sqla_task_dict = self.queue.get() 21 | # self.logger.debug(f'从数据库 {frame_config.SQLACHEMY_ENGINE_URL[:25]}。。 的 [{self._queue_name}] 队列中 取出的消息是： 消息是： {sqla_task_dict}') 22 | self._print_message_get_from_broker(f'从数据库 {frame_config.SQLACHEMY_ENGINE_URL[:25]}',sqla_task_dict) 23 | kw = {'body': json.loads(sqla_task_dict['body']), 'sqla_task_dict': sqla_task_dict} 24 | self._submit_task(kw) 25 | 26 | def _confirm_consume(self, kw): 27 | self.queue.set_success(kw['sqla_task_dict']) 28 | 29 | def _requeue(self, kw): 30 | self.queue.set_task_status(kw['sqla_task_dict'], sqla_queue.TaskStatus.REQUEUE) 31 | 32 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/tcp_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | from threading import Thread 6 | import socket 7 | 8 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 9 | 10 | 11 | class TCPConsumer(AbstractConsumer, ): 12 | """ 13 | socket 实现消息队列，不支持持久化，但不需要安装软件。 14 | """ 15 | BROKER_KIND = 22 16 | 17 | BUFSIZE = 10240 18 | 19 | # noinspection PyAttributeOutsideInit 20 | def custom_init(self): 21 | ip__port_str = self.queue_name.split(':') 22 | ip_port = (ip__port_str[0], int(ip__port_str[1])) 23 | self._ip_port_raw = ip_port 24 | self._ip_port = ('', ip_port[1]) 25 | # ip_port = ('', 9999) 26 | 27 | # noinspection DuplicatedCode 28 | def _shedual_task(self): 29 | """ tcp为消息队列中间件 时候 queue_name 要设置为例如 127.0.0.1:5689""" 30 | server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # tcp协议 31 | server.bind(self._ip_port) 32 | server.listen(128) 33 | self._server = server 34 | while True: 35 | tcp_cli_sock, addr = self._server.accept() 36 | Thread(target=self.__handle_conn, args=(tcp_cli_sock,)).start() # 服务端多线程，可以同时处理多个tcp长链接客户端发来的消息。 37 | 38 | def __handle_conn(self, tcp_cli_sock): 39 | try: 40 | while True: 41 | data = tcp_cli_sock.recv(self.BUFSIZE) 42 | # print('server收到的数据', data) 43 | if not data: 44 | break 45 | self._print_message_get_from_broker(f'udp {self._ip_port_raw}', data.decode()) 46 | tcp_cli_sock.send('has_recived'.encode()) 47 | # tcp_cli_sock.close() 48 | kw = {'body': json.loads(data)} 49 | self._submit_task(kw) 50 | tcp_cli_sock.close() 51 | except ConnectionResetError: 52 | pass 53 | 54 | def _confirm_consume(self, kw): 55 | pass # 没有确认消费的功能。 56 | 57 | def _requeue(self, kw): 58 | pass 59 | 60 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/txt_file_consumer.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from persistqueue import Queue 3 | import json 4 | from persistqueue.serializers import json as json_serializer 5 | 6 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 7 | from function_scheduling_distributed_framework import frame_config 8 | 9 | 10 | class TxtFileConsumer(AbstractConsumer, ): 11 | """ 12 | txt文件作为消息队列 13 | 这个不想做消费确认了,要消费确认请选 SQLITE_QUEUE 、PERSISTQUEUE中间件 14 | """ 15 | BROKER_KIND = 25 16 | 17 | def _shedual_task(self): 18 | queue = Queue(str((Path(frame_config.TXT_FILE_PATH) / self.queue_name).absolute()), autosave=True, serializer=json_serializer) 19 | while True: 20 | item = queue.get() 21 | self._print_message_get_from_broker('txt文件', item) 22 | kw = {'body': json.loads(item), 'q': queue, 'item': item} 23 | self._submit_task(kw) 24 | 25 | def _confirm_consume(self, kw): 26 | pass 27 | # kw['q'].task_done() 28 | 29 | def _requeue(self, kw): 30 | pass 31 | # kw['q'].nack(kw['item']) 32 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/consumers/udp_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:32 4 | import json 5 | import socket 6 | 7 | from function_scheduling_distributed_framework.consumers.base_consumer import AbstractConsumer 8 | 9 | 10 | class UDPConsumer(AbstractConsumer, ): 11 | """ 12 | socket 实现消息队列，不支持持久化，但不需要安装软件。 13 | """ 14 | BROKER_KIND = 21 15 | 16 | BUFSIZE = 10240 17 | 18 | # noinspection PyAttributeOutsideInit 19 | def custom_init(self): 20 | """ udp为消息队列中间件 时候 queue_name 要设置为例如 127.0.0.1:5689""" 21 | self.__udp_client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 22 | ip__port_str = self.queue_name.split(':') 23 | self.__ip_port = (ip__port_str[0], int(ip__port_str[1])) 24 | self.__udp_client.connect(self.__ip_port) 25 | 26 | # noinspection DuplicatedCode 27 | def _shedual_task(self): 28 | ip_port = ('', self.__ip_port[1]) 29 | # ip_port = ('', 9999) 30 | server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # udp协议 31 | server.bind(ip_port) 32 | while True: 33 | data, client_addr = server.recvfrom(self.BUFSIZE) 34 | # print('server收到的数据', data) 35 | self._print_message_get_from_broker(f'udp {ip_port}', data.decode()) 36 | server.sendto('has_recived'.encode(), client_addr) 37 | kw = {'body': json.loads(data)} 38 | self._submit_task(kw) 39 | 40 | def _confirm_consume(self, kw): 41 | pass # 没有确认消费的功能。 42 | 43 | def _requeue(self, kw): 44 | self.__udp_client.send(json.dumps(kw['body']).encode()) 45 | data = self.__udp_client.recv(self.BUFSIZE) 46 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/factories/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:17 4 | 5 | """ 6 | 工厂模式，通过broker_kind来生成不同中间件类型的消费者和发布者。 7 | """ -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/frame_config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from pathlib import Path 4 | from function_scheduling_distributed_framework.constant import BrokerEnum 5 | 6 | ''' 7 | 此文件是第一次运行框架自动生成刀项目根目录的，不需要用由户手动创建。 8 | ''' 9 | 10 | ''' 11 | 你项目根目录下自动生成的 distributed_frame_config.py 文件中修改配置，会被自动读取到。 12 | 13 | 此文件按需修改，例如你使用redis中间件作为消息队列，可以不用管rabbitmq mongodb kafka啥的配置。 14 | 但有3个功能例外，如果你需要使用rpc模式或者分布式控频或者任务过滤功能，无论设置使用何种消息队列中间件都需要把redis连接配置好， 15 | 如果@task_deco装饰器设置is_using_rpc_mode为True或者 is_using_distributed_frequency_control为True或do_task_filtering=True则需要把redis连接配置好，默认是False。 16 | 17 | 18 | 框架使用文档是 https://function-scheduling-distributed-framework.readthedocs.io/zh_CN/latest/ 19 | 20 | ''' 21 | 22 | # 如果@task_deco装饰器没有亲自指定broker_kind入参，则默认使用DEFAULT_BROKER_KIND这个中间件。 23 | # 强烈推荐安装rabbitmq然后使用 BrokerEnum.RABBITMQ_AMQPSTORM 这个中间件, 24 | # 次之 BrokerEnum.REDIS_ACK_ABLE中间件，kafka则推荐 BrokerEnum.CONFLUENT_KAFKA。 25 | # BrokerEnum.PERSISTQUEUE 的优点是基于单机磁盘的消息持久化，不需要安装消息中间件软件就能使用，但不是跨机器的真分布式。 26 | DEFAULT_BROKER_KIND = BrokerEnum.PERSISTQUEUE 27 | 28 | MONGO_CONNECT_URL = f'mongodb://192.168.6.133:27017' # 如果有密码连接 'mongodb://myUserAdmin:8mwTdy1klnSYepNo@192.168.199.202:27016/admin' 29 | 30 | RABBITMQ_USER = 'rabbitmq_user' 31 | RABBITMQ_PASS = 'rabbitmq_pass' 32 | RABBITMQ_HOST = '127.0.0.1' 33 | RABBITMQ_PORT = 5672 34 | RABBITMQ_VIRTUAL_HOST = '/' # my_host # 这个是rabbitmq的虚拟子host用户自己创建的，如果你想直接用rabbitmq的根host而不是使用虚拟子host，这里写 / 即可。 35 | 36 | REDIS_HOST = '127.0.0.1' 37 | REDIS_PASSWORD = '' 38 | REDIS_PORT = 6379 39 | REDIS_DB = 7 40 | 41 | NSQD_TCP_ADDRESSES = ['127.0.0.1:4150'] 42 | NSQD_HTTP_CLIENT_HOST = '127.0.0.1' 43 | NSQD_HTTP_CLIENT_PORT = 4151 44 | 45 | KAFKA_BOOTSTRAP_SERVERS = ['127.0.0.1:9092'] 46 | 47 | SQLACHEMY_ENGINE_URL = 'sqlite:////sqlachemy_queues/queues.db' 48 | 49 | # persist_quque中间件时候采用本机sqlite的方式，数据库文件生成的位置。如果linux账号在根目录没权限建文件夹，可以换文件夹。 50 | SQLLITE_QUEUES_PATH = '/sqllite_queues' 51 | 52 | TXT_FILE_PATH = Path(__file__).parent /'txt_queues' 53 | 54 | ROCKETMQ_NAMESRV_ADDR = '192.168.199.202:9876' 55 | 56 | MQTT_HOST = '127.0.0.1' 57 | MQTT_TCP_PORT = 1883 58 | 59 | HTTPSQS_HOST = '127.0.0.1' 60 | HTTPSQS_PORT = '1218' 61 | HTTPSQS_AUTH = '123456' 62 | 63 | NATS_URL = 'nats://192.168.6.134:4222' 64 | 65 | KOMBU_URL = 'redis://127.0.0.1:6379/0' 66 | # KOMBU_URL = 'sqla+sqlite:////dssf_kombu_sqlite.sqlite' # 4个//// 代表磁盘根目录下生成一个文件。推荐绝对路径。3个///是相对路径。 67 | 68 | 69 | # nb_log包的第几个日志模板，内置了7个模板，可以在你当前项目根目录下的nb_log_config.py文件扩展模板。 70 | NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = 11 # 7是简短的不可跳转，5是可点击跳转的，11是可显示ip 进程 线程的模板。 71 | FSDF_DEVELOP_LOG_LEVEL = 50 # 开发时候的日志，仅供我自己用，所以日志级别跳到最高，用户不需要管。 72 | 73 | TIMEZONE = 'Asia/Shanghai' 74 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/assets/img/user.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/function_result_web/static/assets/img/user.jpg -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/assets/js/custom.js: -------------------------------------------------------------------------------- 1 | jQuery(function($) { 2 | 3 | $(".sidebar-dropdown > a").click(function(){ 4 | $(".sidebar-submenu").slideUp(250); 5 | if ($(this).parent().hasClass("active")){ 6 | $(".sidebar-dropdown").removeClass("active"); 7 | $(this).parent().removeClass("active"); 8 | }else{ 9 | $(".sidebar-dropdown").removeClass("active"); 10 | $(this).next(".sidebar-submenu").slideDown(250); 11 | $(this).parent().addClass("active"); 12 | } 13 | 14 | }); 15 | 16 | $("#toggle-sidebar").click(function(){ 17 | $(".page-wrapper").toggleClass("toggled"); 18 | }); 19 | $(".page-wrapper").toggleClass("toggled"); 20 | if(! /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent) ) { 21 | $(".sidebar-content").mCustomScrollbar({ 22 | axis:"y", 23 | autoHideScrollbar: true, 24 | scrollInertia: 300 25 | }); 26 | $(".sidebar-content").addClass("desktop"); 27 | 28 | } 29 | }); -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/images/bg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/function_result_web/static/images/bg.jpg -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/images/password.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/function_result_web/static/images/password.png -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/images/tick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/function_result_web/static/images/tick.png -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/static/images/user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/function_result_web/static/images/user.png -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/function_result_web/templates/login.html: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | python分布式函数调度框架 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |

python分布式函数调度框架

13 |
14 |

立即登录

15 |
16 | {{ form.csrf_token }} 17 | 18 | 19 | 31 | 32 |
33 | {% with messages = get_flashed_messages(category_filter=["error"]) %} 34 | {% if messages %} 35 | 36 | {% for message in messages %} 37 | {{ message }} 38 | 39 | {% endfor %} 40 | 41 | {% endif %} 42 | {% endwith %} 43 | 44 | 45 | 46 |
47 |
48 |
49 |

© 2019 分布式函数调度框架

50 |
51 | 52 | 53 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 11:50 4 | 5 | """ 6 | 实现各种中间件类型的发布者。 7 | """ -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/http_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | 5 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 6 | from urllib3 import PoolManager 7 | 8 | 9 | class HTTPPublisher(AbstractPublisher, ): 10 | """ 11 | http实现的，不支持持久化。 12 | """ 13 | 14 | BROKER_KIND = 23 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def custom_init(self): 18 | self._http = PoolManager(10) 19 | 20 | def concrete_realization_of_publish(self, msg): 21 | url = self.queue_name + '/queue' 22 | self._http.request('post', url, fields={'msg': msg}) 23 | 24 | def clear(self): 25 | pass # udp没有保存消息 26 | 27 | def get_message_count(self): 28 | return -1 # http模式没有持久化保存消息 29 | 30 | def close(self): 31 | pass 32 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/httpsqs_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | import json 5 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 6 | import http.client 7 | from urllib.parse import quote 8 | from function_scheduling_distributed_framework import frame_config 9 | import urllib3 10 | 11 | """ 12 | http://blog.zyan.cc/httpsqs/ 13 | """ 14 | 15 | 16 | class HttpsqsPublisher(AbstractPublisher): 17 | """ 18 | 使用httpsqs作为中间件 19 | """ 20 | 21 | # noinspection PyAttributeOutsideInit 22 | def custom_init(self): 23 | conn = http.client.HTTPConnection(host=frame_config.HTTPSQS_HOST, port=frame_config.HTTPSQS_PORT) 24 | url = f"/?name={self._queue_name}&opt=maxqueue&num=1000000000&auth={frame_config.HTTPSQS_AUTH}&charset=utf-8" 25 | conn.request("GET", url) 26 | self.logger.info(conn.getresponse().read(1000)) 27 | 28 | self.http = urllib3.PoolManager(20) 29 | 30 | def opt_httpsqs000(self, opt=None, data=''): 31 | data_url_encode = quote(data) 32 | resp = self.http.request('get', url=f'http://{frame_config.HTTPSQS_HOST}:{frame_config.HTTPSQS_PORT}' + 33 | f"/?name={self._queue_name}&opt={opt}&data={data_url_encode}&auth={frame_config.HTTPSQS_AUTH}&charset=utf-8") 34 | return resp.data.decode() 35 | 36 | def opt_httpsqs(self, opt=None, data=''): 37 | conn = http.client.HTTPConnection(host=frame_config.HTTPSQS_HOST, port=frame_config.HTTPSQS_PORT) 38 | data_url_encode = quote(data) 39 | url = f"/?name={self._queue_name}&opt={opt}&data={data_url_encode}&auth={frame_config.HTTPSQS_AUTH}&charset=utf-8" 40 | conn.request("GET", url) 41 | r = conn.getresponse() 42 | resp_text = r.read(1000000).decode() 43 | # print(url,r.status, resp_text) 44 | conn.close() 45 | return resp_text 46 | 47 | def concrete_realization_of_publish(self, msg): 48 | # curl "http://host:port/?name=your_queue_name&opt=put&data=经过URL编码的文本消息&auth=mypass123" 49 | text = self.opt_httpsqs('put', msg) 50 | if text != 'HTTPSQS_PUT_OK': 51 | self.logger.critical(text) 52 | 53 | def clear(self): 54 | # curl "http://host:port/?name=your_queue_name&opt=reset&auth=mypass123" 55 | # HTTPSQS_RESET_OK 56 | text = self.opt_httpsqs('reset') 57 | if text != 'HTTPSQS_RESET_OK': 58 | self.logger.critical(text) 59 | else: 60 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 61 | 62 | def get_message_count(self): 63 | text = self.opt_httpsqs('status_json') 64 | status_dict = json.loads(text) 65 | # print(status_dict) 66 | return status_dict['putpos'] - status_dict['getpos'] 67 | 68 | def close(self): 69 | self.http.clear() 70 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/kafka_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/20 0008 12:12 4 | 5 | # noinspection PyPackageRequirements 6 | import atexit 7 | 8 | # noinspection PyPackageRequirements 9 | from kafka import KafkaProducer, KafkaAdminClient 10 | # noinspection PyPackageRequirements 11 | from kafka.admin import NewTopic 12 | # noinspection PyPackageRequirements 13 | from kafka.errors import TopicAlreadyExistsError 14 | 15 | from function_scheduling_distributed_framework import frame_config 16 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 17 | 18 | 19 | class KafkaPublisher(AbstractPublisher, ): 20 | """ 21 | 使用kafka作为中间件 22 | """ 23 | 24 | # noinspection PyAttributeOutsideInit 25 | def custom_init(self): 26 | self._producer = KafkaProducer(bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) 27 | self._admin_client = KafkaAdminClient(bootstrap_servers=frame_config.KAFKA_BOOTSTRAP_SERVERS) 28 | try: 29 | self._admin_client.create_topics([NewTopic(self._queue_name, 10, 1)]) 30 | # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)}) 31 | except TopicAlreadyExistsError: 32 | pass 33 | except Exception as e: 34 | self.logger.exception(e) 35 | atexit.register(self.close) # 程序退出前不主动关闭，会报错。 36 | 37 | def concrete_realization_of_publish(self, msg): 38 | # noinspection PyTypeChecker 39 | # self.logger.debug(msg) 40 | # print(msg) 41 | self._producer.send(self._queue_name, msg.encode(), ) 42 | 43 | def clear(self): 44 | self.logger.warning('还没开始实现 kafka 清空 消息') 45 | # self._consumer.seek_to_end() 46 | # self.logger.warning(f'将kafka offset 重置到最后位置') 47 | 48 | def get_message_count(self): 49 | # return -1 # 还没找到获取所有分区未消费数量的方法 。 50 | # print(self._admin_client.list_consumer_group_offsets('frame_group')) 51 | # print(self._admin_client.describe_consumer_groups('frame_group')) 52 | return -1 53 | 54 | def close(self): 55 | self._producer.close() 56 | 57 | def _at_exit(self): 58 | self._producer.flush() 59 | super()._at_exit() 60 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/local_python_queue_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:07 4 | from queue import Queue 5 | 6 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 7 | 8 | local_pyhton_queue_name__local_pyhton_queue_obj_map = dict() # 使local queue和其他中间件完全一样的使用方式，使用映射保存队列的名字，使消费和发布通过队列名字能找到队列对象。 9 | 10 | 11 | class LocalPythonQueuePublisher(AbstractPublisher): 12 | """ 13 | 使用python内置queue对象作为中间件。方便测试，每个中间件的消费者类是鸭子类，多态可以互相替换。 14 | """ 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def custom_init(self): 18 | if self._queue_name not in local_pyhton_queue_name__local_pyhton_queue_obj_map: 19 | local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] = Queue() 20 | self.queue = local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] 21 | 22 | def concrete_realization_of_publish(self, msg): 23 | # noinspection PyTypeChecker 24 | self.queue.put(msg) 25 | 26 | def clear(self): 27 | # noinspection PyUnresolvedReferences 28 | self.queue.queue.clear() 29 | self.logger.warning(f'清除 本地队列中的消息成功') 30 | 31 | def get_message_count(self): 32 | return self.queue.qsize() 33 | 34 | def close(self): 35 | pass 36 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/mongomq_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:23 4 | import json 5 | from function_scheduling_distributed_framework.utils.dependency_packages.mongomq import MongoQueue 6 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 7 | from function_scheduling_distributed_framework.utils import time_util 8 | from function_scheduling_distributed_framework.utils.mongo_util import MongoMixin 9 | 10 | 11 | class MongoMqPublisher(AbstractPublisher, MongoMixin): 12 | # 使用mongo-queue包实现的基于mongodb的队列。 队列是一个col，自动存放在consume_queues库中。 13 | # noinspection PyAttributeOutsideInit 14 | def custom_init(self): 15 | self.queue = MongoQueue( 16 | self.mongo_client.get_database('consume_queues').get_collection(self._queue_name), 17 | consumer_id=f"consumer-{time_util.DatetimeConverter().datetime_str}", 18 | timeout=600, 19 | max_attempts=3, 20 | ttl=0) 21 | 22 | def concrete_realization_of_publish(self, msg): 23 | # noinspection PyTypeChecker 24 | self.queue.put(json.loads(msg)) 25 | 26 | def clear(self): 27 | self.queue.clear() 28 | self.logger.warning(f'清除 mongo队列 {self._queue_name} 中的消息成功') 29 | 30 | def get_message_count(self): 31 | return self.queue.size() 32 | 33 | def close(self): 34 | pass 35 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/nats_publisher.py: -------------------------------------------------------------------------------- 1 | from pynats import NATSClient # noqa 2 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 3 | from function_scheduling_distributed_framework import frame_config 4 | 5 | 6 | class NatsPublisher(AbstractPublisher, ): 7 | """ 8 | 使用nats作为中间件 9 | """ 10 | 11 | # noinspection PyAttributeOutsideInit 12 | def custom_init(self): 13 | self.nats_client = NATSClient(frame_config.NATS_URL) 14 | self.nats_client.connect() 15 | 16 | def concrete_realization_of_publish(self, msg): 17 | # print(msg) 18 | self.nats_client.publish(subject=self.queue_name, payload=msg.encode()) 19 | 20 | def clear(self): 21 | pass 22 | 23 | def get_message_count(self): 24 | return -1 25 | 26 | def close(self): 27 | # self.redis_db7.connection_pool.disconnect() 28 | pass 29 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/nsq_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/19 0008 12:12 4 | 5 | from gnsq import Producer, NsqdHTTPClient 6 | from gnsq.errors import NSQHttpError 7 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 8 | from function_scheduling_distributed_framework import frame_config 9 | 10 | 11 | class NsqPublisher(AbstractPublisher, ): 12 | """ 13 | 使用nsq作为中间件 14 | """ 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def custom_init(self): 18 | self._nsqd_cleint = NsqdHTTPClient(frame_config.NSQD_HTTP_CLIENT_HOST, frame_config.NSQD_HTTP_CLIENT_PORT) 19 | self._producer = Producer(frame_config.NSQD_TCP_ADDRESSES) 20 | self._producer.start() 21 | 22 | def concrete_realization_of_publish(self, msg): 23 | # noinspection PyTypeChecker 24 | self._producer.publish(self._queue_name, msg.encode()) 25 | 26 | def clear(self): 27 | try: 28 | self._nsqd_cleint.empty_topic(self._queue_name) 29 | except NSQHttpError as e: 30 | self.logger.exception(e) # 不能清除一个不存在的topoc会报错，和其他消息队列中间件不同。 31 | self.logger.warning(f'清除 {self._queue_name} topic中的消息成功') 32 | 33 | def get_message_count(self): 34 | return -1 35 | 36 | def close(self): 37 | self._producer.close() 38 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/persist_queue_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:05 4 | import json 5 | import sqlite3 6 | 7 | import persistqueue 8 | from function_scheduling_distributed_framework import frame_config 9 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 10 | from function_scheduling_distributed_framework.utils import LogManager 11 | 12 | LogManager('persistqueue').get_logger_and_add_handlers(10) 13 | 14 | 15 | # noinspection PyProtectedMember 16 | class PersistQueuePublisher(AbstractPublisher): 17 | """ 18 | 使用persistqueue实现的本地持久化队列。 19 | 这个是本地持久化，支持本地多个启动的python脚本共享队列任务。与LocalPythonQueuePublisher相比，不会随着python解释器退出，导致任务丢失。 20 | """ 21 | 22 | # noinspection PyAttributeOutsideInit 23 | def custom_init(self): 24 | # noinspection PyShadowingNames 25 | def _my_new_db_connection(self, path, multithreading, timeout): # 主要是改了sqlite文件后缀，方便pycharm识别和打开。 26 | # noinspection PyUnusedLocal 27 | conn = None 28 | if path == self._MEMORY: 29 | conn = sqlite3.connect(path, 30 | check_same_thread=not multithreading) 31 | else: 32 | conn = sqlite3.connect('{}/data.sqlite'.format(path), 33 | timeout=timeout, 34 | check_same_thread=not multithreading) 35 | conn.execute('PRAGMA journal_mode=WAL;') 36 | return conn 37 | 38 | persistqueue.SQLiteAckQueue._new_db_connection = _my_new_db_connection # 打猴子补丁。 39 | # REMIND 官方测试基于sqlite的本地持久化，比基于纯文件的持久化，使用相同固态硬盘和操作系统情况下，速度快3倍以上，所以这里选用sqlite方式。 40 | 41 | self.queue = persistqueue.SQLiteAckQueue(path=frame_config.SQLLITE_QUEUES_PATH, name=self._queue_name, auto_commit=True, serializer=json, multithreading=True) 42 | 43 | def concrete_realization_of_publish(self, msg): 44 | # noinspection PyTypeChecker 45 | self.queue.put(msg) 46 | 47 | def clear(self): 48 | sql = f'{"DELETE"} {"FROM"} ack_queue_{self._queue_name}' 49 | self.logger.info(sql) 50 | self.queue._getter.execute(sql) 51 | self.queue._getter.commit() 52 | self.logger.warning(f'清除 本地持久化队列 {self._queue_name} 中的消息成功') 53 | 54 | def get_message_count(self): 55 | return self.queue._count() 56 | # return self.queue.qsize() 57 | 58 | def close(self): 59 | pass 60 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/rabbitmq_amqpstorm_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:06 4 | import amqpstorm 5 | from amqpstorm.basic import Basic as AmqpStormBasic 6 | from amqpstorm.queue import Queue as AmqpStormQueue 7 | from function_scheduling_distributed_framework import frame_config 8 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher, deco_mq_conn_error 9 | 10 | 11 | class RabbitmqPublisherUsingAmqpStorm(AbstractPublisher): 12 | # 使用amqpstorm包实现的mq操作。 13 | # 实例属性没在init里面写，造成补全很麻烦，写在这里做类属性，方便pycharm补全 14 | connection = amqpstorm.UriConnection 15 | channel = amqpstorm.Channel 16 | channel_wrapper_by_ampqstormbaic = AmqpStormBasic 17 | queue = AmqpStormQueue 18 | 19 | # noinspection PyAttributeOutsideInit 20 | # @decorators.synchronized 21 | def init_broker(self): 22 | # username=app_config.RABBITMQ_USER, password=app_config.RABBITMQ_PASS, host=app_config.RABBITMQ_HOST, port=app_config.RABBITMQ_PORT, virtual_host=app_config.RABBITMQ_VIRTUAL_HOST, heartbeat=60 * 10 23 | self.logger.warning(f'使用AmqpStorm包 链接mq') 24 | self.connection = amqpstorm.UriConnection( 25 | f'amqp://{frame_config.RABBITMQ_USER}:{frame_config.RABBITMQ_PASS}@{frame_config.RABBITMQ_HOST}:{frame_config.RABBITMQ_PORT}/{frame_config.RABBITMQ_VIRTUAL_HOST}?heartbeat={60 * 10}' 26 | ) 27 | self.channel = self.connection.channel() # type:amqpstorm.Channel 28 | self.channel_wrapper_by_ampqstormbaic = AmqpStormBasic(self.channel) 29 | self.queue = AmqpStormQueue(self.channel) 30 | self.queue.declare(queue=self._queue_name, durable=True) 31 | 32 | # @decorators.tomorrow_threads(10) 33 | @deco_mq_conn_error 34 | def concrete_realization_of_publish(self, msg): 35 | self.channel_wrapper_by_ampqstormbaic.publish(exchange='', 36 | routing_key=self._queue_name, 37 | body=msg, 38 | properties={'delivery_mode': 2}, ) 39 | # nb_print(msg) 40 | 41 | @deco_mq_conn_error 42 | def clear(self): 43 | self.queue.purge(self._queue_name) 44 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 45 | 46 | @deco_mq_conn_error 47 | def get_message_count(self): 48 | # noinspection PyUnresolvedReferences 49 | return self.queue.declare(queue=self._queue_name, durable=True)['message_count'] 50 | 51 | # @deco_mq_conn_error 52 | def close(self): 53 | self.channel.close() 54 | self.connection.close() 55 | self.logger.warning('关闭amqpstorm包 链接mq') 56 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/rabbitmq_pika_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:03 4 | from threading import Lock 5 | from pikav1 import BasicProperties 6 | import pikav1 7 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher, deco_mq_conn_error 8 | from function_scheduling_distributed_framework import frame_config 9 | 10 | 11 | class RabbitmqPublisher(AbstractPublisher): 12 | """ 13 | 使用pika实现的。 14 | """ 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def custom_init(self): 18 | self._lock_for_pika = Lock() 19 | 20 | # noinspection PyAttributeOutsideInit 21 | def init_broker(self): 22 | self.logger.warning(f'使用pika 链接mq') 23 | credentials = pikav1.PlainCredentials(frame_config.RABBITMQ_USER, frame_config.RABBITMQ_PASS) 24 | self.connection = pikav1.BlockingConnection(pikav1.ConnectionParameters( 25 | frame_config.RABBITMQ_HOST, frame_config.RABBITMQ_PORT, frame_config.RABBITMQ_VIRTUAL_HOST, credentials, heartbeat=60)) 26 | self.channel = self.connection.channel() 27 | self.queue = self.channel.queue_declare(queue=self._queue_name, durable=True) 28 | 29 | # noinspection PyAttributeOutsideInit 30 | @deco_mq_conn_error 31 | def concrete_realization_of_publish(self, msg): 32 | with self._lock_for_pika: # 亲测pika多线程publish会出错 33 | self.channel.basic_publish(exchange='', 34 | routing_key=self._queue_name, 35 | body=msg, 36 | properties=BasicProperties( 37 | delivery_mode=2, # make message persistent 2(1是非持久化) 38 | ) 39 | ) 40 | 41 | @deco_mq_conn_error 42 | def clear(self): 43 | self.channel.queue_purge(self._queue_name) 44 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 45 | 46 | @deco_mq_conn_error 47 | def get_message_count(self): 48 | with self._lock_for_pika: 49 | queue = self.channel.queue_declare(queue=self._queue_name, durable=True) 50 | return queue.method.message_count 51 | 52 | # @deco_mq_conn_error 53 | def close(self): 54 | self.channel.close() 55 | self.connection.close() 56 | self.logger.warning('关闭pika包 链接') 57 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/rabbitmq_rabbitpy_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:04 4 | import os 5 | import rabbitpy 6 | 7 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher, deco_mq_conn_error 8 | from function_scheduling_distributed_framework.utils.rabbitmq_factory import RabbitMqFactory 9 | 10 | 11 | class RabbitmqPublisherUsingRabbitpy(AbstractPublisher): 12 | """ 13 | 使用rabbitpy包实现的。 14 | """ 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def init_broker(self): 18 | self.logger.warning(f'使用rabbitpy包 链接mq') 19 | self.rabbit_client = RabbitMqFactory(is_use_rabbitpy=1).get_rabbit_cleint() 20 | self.channel = self.rabbit_client.creat_a_channel() 21 | self.queue = self.channel.queue_declare(queue=self._queue_name, durable=True) 22 | self.logger.critical('rabbitpy 快速发布 有问题会丢失大量任务，如果使用 rabbitmq 建议设置中间件为 BrokerEnum.RABBITMQ_AMQPSTORM') 23 | os._exit(444) # noqa 24 | 25 | # @decorators.tomorrow_threads(10) 26 | @deco_mq_conn_error 27 | def concrete_realization_of_publish(self, msg): 28 | # noinspection PyTypeChecker 29 | import time 30 | # time.sleep(0.1) 31 | print(self.channel.basic_publish( 32 | exchange='', 33 | routing_key=self._queue_name, 34 | body=msg, 35 | properties={'delivery_mode': 2}, 36 | )) 37 | 38 | 39 | @deco_mq_conn_error 40 | def clear(self): 41 | self.channel.queue_purge(self._queue_name) 42 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 43 | 44 | @deco_mq_conn_error 45 | def get_message_count(self): 46 | # noinspection PyUnresolvedReferences 47 | ch_raw_rabbity = self.channel.channel 48 | return len(rabbitpy.amqp_queue.Queue(ch_raw_rabbity, self._queue_name, durable=True)) 49 | 50 | # @deco_mq_conn_error 51 | def close(self): 52 | self.channel.close() 53 | self.rabbit_client.connection.close() 54 | self.logger.warning('关闭rabbitpy包 链接mq') 55 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/redis_publisher_lpush.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | 5 | 6 | from function_scheduling_distributed_framework.publishers.redis_publisher import RedisPublisher 7 | 8 | 9 | class RedisPublisherLpush(RedisPublisher): 10 | """ 11 | 使用redis作为中间件, 12 | """ 13 | 14 | _push_method = 'lpush' 15 | 16 | 17 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/redis_publisher_simple.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 5 | from function_scheduling_distributed_framework.utils import RedisMixin 6 | 7 | 8 | class RedisPublisher(AbstractPublisher, RedisMixin): 9 | """ 10 | 使用redis作为中间件 11 | """ 12 | 13 | def concrete_realization_of_publish(self, msg): 14 | self.redis_db_frame.rpush(self._queue_name, msg) 15 | 16 | def clear(self): 17 | self.redis_db_frame.delete(self._queue_name) 18 | self.redis_db_frame.delete(f'{self._queue_name}__unack') 19 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 20 | 21 | def get_message_count(self): 22 | # nb_print(self.redis_db7,self._queue_name) 23 | return self.redis_db_frame.llen(self._queue_name) 24 | 25 | def close(self): 26 | # self.redis_db7.connection_pool.disconnect() 27 | pass 28 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/redis_stream_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2021/4/3 0008 13:32 4 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 5 | from function_scheduling_distributed_framework.utils import RedisMixin 6 | 7 | 8 | class RedisStreamPublisher(AbstractPublisher, RedisMixin): 9 | """ 10 | redis 的 stream 结构 作为中间件实现的。需要redis 5.0以上，redis stream结构 是redis的消息队列，功能远超 list结构。 11 | """ 12 | 13 | _has__check_redis_version = False 14 | 15 | def _check_redis_version(self): 16 | redis_server_info_dict = self.redis_db_frame_version3.info() 17 | if float(redis_server_info_dict['redis_version'][0]) < 5: 18 | raise EnvironmentError('必须是5.0版本以上redis服务端才能支持 stream 数据结构，' 19 | '请升级服务端，否则使用 REDIS_ACK_ABLE 方式使用redis 的 list 结构') 20 | if self.redis_db_frame_version3.type(self._queue_name) == 'list': 21 | raise EnvironmentError(f'检测到已存在 {self._queue_name} 这个键，且类型是list， 必须换个队列名字或者删除这个' 22 | f' list 类型的键。' 23 | f'RedisStreamConsumer 使用的是 stream数据结构') 24 | self._has__check_redis_version = True 25 | 26 | def concrete_realization_of_publish(self, msg): 27 | # redis服务端必须是5.0以上，并且确保这个键的类型是stream不能是list数据结构。 28 | if not self._has__check_redis_version: 29 | self._check_redis_version() 30 | self.redis_db_frame_version3.xadd(self._queue_name, {"": msg}) 31 | 32 | def clear(self): 33 | self.redis_db_frame.delete(self._queue_name) 34 | self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功') 35 | 36 | def get_message_count(self): 37 | # nb_print(self.redis_db7,self._queue_name) 38 | return self.redis_db_frame_version3.xlen(self._queue_name) 39 | 40 | def close(self): 41 | # self.redis_db7.connection_pool.disconnect() 42 | pass 43 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/rocketmq_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2020/7/9 0008 12:12 4 | 5 | import time 6 | from function_scheduling_distributed_framework import frame_config 7 | 8 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 9 | 10 | 11 | 12 | class RocketmqPublisher(AbstractPublisher, ): 13 | group_id__rocketmq_producer = {} 14 | 15 | # noinspection PyAttributeOutsideInit 16 | def custom_init(self): 17 | try: 18 | from rocketmq.client import Producer 19 | except Exception as e: 20 | # print(traceback.format_exc()) 21 | raise ImportError(f'rocketmq包 只支持linux和mac {str(e)}') 22 | 23 | group_id = f'g-{self._queue_name}' 24 | if group_id not in self.__class__.group_id__rocketmq_producer: # 同一个进程中创建多个同组消费者会报错。 25 | producer = Producer(group_id) 26 | producer.set_namesrv_addr(frame_config.ROCKETMQ_NAMESRV_ADDR) 27 | producer.start() 28 | self.__class__.group_id__rocketmq_producer[group_id] = producer 29 | else: 30 | producer = self.__class__.group_id__rocketmq_producer[group_id] 31 | self._producer = producer 32 | 33 | def concrete_realization_of_publish(self, msg): 34 | try: 35 | from rocketmq.client import Message 36 | except Exception as e: 37 | # print(traceback.format_exc()) 38 | raise ImportError(f'rocketmq包 只支持linux和mac {str(e)}') 39 | rocket_msg = Message(self._queue_name) 40 | rocket_msg.set_keys(msg) # 利于检索 41 | # rocket_msg.set_tags('XXX') 42 | rocket_msg.set_body(msg) 43 | # print(msg) 44 | self._producer.send_sync(rocket_msg) 45 | 46 | def clear(self): 47 | self.logger.error('清除队列，python版的rocket包太弱了，没有方法设置偏移量或者删除主题。java才能做到') 48 | 49 | def get_message_count(self): 50 | if time.time() - getattr(self, '_last_warning_count', 0) > 300: 51 | setattr(self, '_last_warning_count', time.time()) 52 | self.logger.warning('获取消息数量，python版的rocket包太弱了，没找到方法。java才能做到。') 53 | return -1 54 | 55 | def close(self): 56 | self._producer.shutdown() 57 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/sqla_queue_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:05 4 | from function_scheduling_distributed_framework.utils import sqla_queue 5 | from function_scheduling_distributed_framework import frame_config 6 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 7 | 8 | 9 | # noinspection PyProtectedMember 10 | class SqlachemyQueuePublisher(AbstractPublisher): 11 | """ 12 | 使用Sqlachemy 操作数据库 ，实现的5种sql 数据库服务器作为 消息队列。包括sqlite mydql microsoftsqlserver postgre oracle 13 | 这个是使用数据库表模拟的消息队列。这不是突发奇想一意孤行，很多包库都实现了这。 14 | """ 15 | 16 | # noinspection PyAttributeOutsideInit 17 | def custom_init(self): 18 | self.queue = sqla_queue.SqlaQueue(self._queue_name, frame_config.SQLACHEMY_ENGINE_URL) 19 | 20 | def concrete_realization_of_publish(self, msg): 21 | self.queue.push(dict(body=msg, status=sqla_queue.TaskStatus.TO_BE_CONSUMED)) 22 | 23 | def clear(self): 24 | self.queue.clear_queue() 25 | self.logger.warning(f'清除 sqlalchemy 数据库队列 {self._queue_name} 中的消息成功') 26 | 27 | def get_message_count(self): 28 | return self.queue.to_be_consumed_count 29 | 30 | def close(self): 31 | pass 32 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/tcp_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | import socket 5 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 6 | 7 | 8 | class TCPPublisher(AbstractPublisher, ): 9 | """ 10 | 使用tco作为中间件,不支持持久化，支持分布式 11 | """ 12 | 13 | BROKER_KIND = 21 14 | 15 | BUFSIZE = 10240 16 | 17 | # noinspection PyAttributeOutsideInit 18 | def custom_init(self): 19 | """ tcp为消息队列中间件 时候 queue_name 要设置为例如 127.0.0.1:5689""" 20 | pass 21 | 22 | # noinspection PyAttributeOutsideInit 23 | def concrete_realization_of_publish(self, msg): 24 | if not hasattr(self, '_tcp_cli_sock'): 25 | ip__port_str = self.queue_name.split(':') 26 | ip_port = (ip__port_str[0], int(ip__port_str[1])) 27 | tcp_cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 28 | tcp_cli_sock.connect(ip_port) 29 | self._tcp_cli_sock = tcp_cli_sock 30 | 31 | self._tcp_cli_sock.send(msg.encode()) 32 | self._tcp_cli_sock.recv(self.BUFSIZE) 33 | 34 | def clear(self): 35 | pass # udp没有保存消息 36 | 37 | def get_message_count(self): 38 | # nb_print(self.redis_db7,self._queue_name) 39 | return -1 # udp没有保存消息 40 | 41 | def close(self): 42 | # self.redis_db7.connection_pool.disconnect() 43 | self._tcp_cli_sock.close() 44 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/txt_file_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | import shutil 5 | from pathlib import Path 6 | 7 | from nb_filelock import FileLock 8 | from persistqueue import Queue 9 | from persistqueue.serializers import json as json_serializer 10 | 11 | from function_scheduling_distributed_framework import frame_config 12 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 13 | 14 | 15 | class TxtFilePublisher(AbstractPublisher, ): 16 | """ 17 | 使用txt文件作为中间件 18 | """ 19 | 20 | # noinspection PyAttributeOutsideInit 21 | def custom_init(self): 22 | self._file_queue_path = str((Path(frame_config.TXT_FILE_PATH) / self.queue_name).absolute()) 23 | # Path(self._file_queue_path).mkdir(exist_ok=True) 24 | self.file_queue = Queue(self._file_queue_path, 25 | autosave=True, serializer=json_serializer) 26 | self._file_lock = FileLock(Path(self._file_queue_path) / '_fsdf_queue.lock') 27 | 28 | def concrete_realization_of_publish(self, msg): 29 | with self._file_lock: 30 | self.file_queue.put(msg) 31 | 32 | def clear(self): 33 | shutil.rmtree(self._file_queue_path, ignore_errors=True) 34 | self.logger.warning(f'清除 {Path(self._file_queue_path).absolute()} 文件夹成功') 35 | 36 | def get_message_count(self): 37 | return self.file_queue.qsize() 38 | 39 | def close(self): 40 | pass 41 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/udp_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 12:12 4 | import socket 5 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 6 | 7 | 8 | class UDPPublisher(AbstractPublisher, ): 9 | """ 10 | 使用udp作为中间件,不支持持久化，支持分布式 11 | """ 12 | 13 | BROKER_KIND = 22 14 | 15 | BUFSIZE = 10240 16 | 17 | # noinspection PyAttributeOutsideInit 18 | def custom_init(self): 19 | """ udp为消息队列中间件 时候 queue_name 要设置为例如 127.0.0.1:5689""" 20 | self.__udp_client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 21 | ip__port_str = self.queue_name.split(':') 22 | self.__ip_port = (ip__port_str[0], int(ip__port_str[1])) 23 | self.__udp_client.connect(self.__ip_port) 24 | 25 | def concrete_realization_of_publish(self, msg): 26 | self.__udp_client.send(msg.encode('utf-8'), ) 27 | self.__udp_client.recv(self.BUFSIZE) 28 | 29 | def clear(self): 30 | pass # udp没有保存消息 31 | 32 | def get_message_count(self): 33 | # nb_print(self.redis_db7,self._queue_name) 34 | return -1 # udp没有保存消息 35 | 36 | def close(self): 37 | # self.redis_db7.connection_pool.disconnect() 38 | self.__udp_client.close() 39 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/publishers/zeromq_publisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | import zmq 4 | from function_scheduling_distributed_framework import frame_config 5 | from function_scheduling_distributed_framework.publishers.base_publisher import AbstractPublisher 6 | 7 | 8 | # noinspection PyProtectedMember 9 | class ZeroMqPublisher(AbstractPublisher): 10 | """ 11 | zeromq 中间件的发布者，zeromq基于socket代码，不会持久化，且不需要安装软件。 12 | """ 13 | def custom_init(self): 14 | context = zmq.Context() 15 | socket = context.socket(zmq.REQ) 16 | socket.connect(f"tcp://localhost:{int(self._queue_name)}") 17 | self.socket =socket 18 | self.logger.warning('框架使用 zeromq 中间件方式，必须先启动消费者(消费者会顺便启动broker) ,只有启动了服务端才能发布任务') 19 | 20 | def concrete_realization_of_publish(self, msg): 21 | self.socket.send(msg.encode()) 22 | self.socket.recv() 23 | 24 | def clear(self): 25 | pass 26 | 27 | def get_message_count(self): 28 | return -1 29 | 30 | def close(self): 31 | pass 32 | 33 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/README.md: -------------------------------------------------------------------------------- 1 | # 这里的utils功能是从已有项目复制出来的，大部分没用到，不要管这里。 2 | 3 | # 最主要的是要看base_consumer.py,框架90%逻辑流程在 AbstractConsumer -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 9:48 4 | import json 5 | from datetime import datetime as _datetime 6 | from datetime import date as _date 7 | 8 | 9 | from nb_log import (LogManager, simple_logger, defaul_logger, LoggerMixin, LoggerLevelSetterMixin, 10 | LoggerMixinDefaultWithFileHandler,nb_print,patch_print,reverse_patch_print,get_logger) 11 | 12 | from function_scheduling_distributed_framework.utils.redis_manager import RedisMixin 13 | 14 | class _CustomEncoder(json.JSONEncoder): 15 | """自定义的json解析器，mongodb返回的字典中的时间格式是datatime，json直接解析出错""" 16 | 17 | def default(self, obj): 18 | if isinstance(obj, _datetime): 19 | return obj.strftime('%Y-%m-%d %H:%M:%S') 20 | elif isinstance(obj, _date): 21 | return obj.strftime('%Y-%m-%d') 22 | else: 23 | return json.JSONEncoder.default(self, obj) 24 | 25 | 26 | # noinspection PyProtectedMember,PyPep8,PyRedundantParentheses 27 | def _dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=_CustomEncoder, indent=None, separators=None, 28 | default=None, sort_keys=False, **kw): 29 | # 全局patch ensure_ascii = False 会引起极少数库不兼容。 30 | if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and default is None and not sort_keys and not kw): 31 | return json._default_encoder.encode(obj) 32 | if cls is None: 33 | cls = json.JSONEncoder 34 | return cls( 35 | skipkeys=skipkeys, ensure_ascii=ensure_ascii, 36 | check_circular=check_circular, allow_nan=allow_nan, indent=indent, 37 | separators=separators, default=default, sort_keys=sort_keys, ).encode(obj) 38 | 39 | 40 | def monkey_patch_json(): 41 | json.dumps = _dumps 42 | 43 | #################以下为打猴子补丁##################### 44 | monkey_patch_json() 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/dependency_packages/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/function_scheduling_distributed_framework/utils/dependency_packages/__init__.py -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/dependency_packages/mongomq/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | __all__ = ['MongoQueue', 'Job', 'MongoLock', 'lock'] 3 | 4 | from .mongomq import MongoQueue, Job 5 | from .lock import MongoLock, lock 6 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/dependency_packages/mongomq/lock.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import time 3 | import uuid 4 | 5 | from pymongo import errors 6 | from datetime import datetime, timedelta 7 | 8 | 9 | @contextlib.contextmanager 10 | def lock(collection, key, wait=30, poll_period=5, lease_period=30): 11 | task_lock = MongoLock(collection, key, lease_period) 12 | try: 13 | task_lock.acquire(wait=wait, poll_period=poll_period) 14 | yield task_lock 15 | finally: 16 | task_lock.release() 17 | 18 | 19 | class MongoLock(object): 20 | 21 | def __init__(self, collection, lock_name, lease=120): 22 | self.collection = collection 23 | self.lock_name = lock_name 24 | self._client_id = uuid.uuid4().get_hex() 25 | self._locked = False 26 | self._lease_time = lease 27 | self._lock_expires = False 28 | 29 | @property 30 | def locked(self): 31 | if not self._locked: 32 | return self._locked 33 | return self._locked and (datetime.now() < self._lock_expires) 34 | 35 | @property 36 | def client_id(self): 37 | return self._client_id 38 | 39 | def acquire(self, wait=None, poll_period=5): 40 | result = self._acquire() 41 | if not wait: 42 | return result 43 | 44 | assert isinstance(wait, int) 45 | max_wait = datetime.now() + timedelta(wait) 46 | while max_wait < datetime.now(): 47 | result = self._acquire() 48 | if result: 49 | return result 50 | time.sleep(poll_period) 51 | 52 | def _acquire(self): 53 | ttl = datetime.now() + timedelta(seconds=self._lease_time) 54 | try: 55 | self.collection.insert({ 56 | '_id': self.lock_name, 57 | 'ttl': ttl, 58 | 'client_id': self._client_id}, 59 | w=1, j=1) 60 | except errors.DuplicateKeyError: 61 | self.collection.remove( 62 | {"_id": self.lock_name, 'ttl': {'$lt': datetime.now()}}) 63 | try: 64 | self.collection.insert( 65 | {'_id': self.lock_name, 66 | 'ttl': ttl, 67 | 'client_id': self._client_id}, w=1, j=1) 68 | except errors.DuplicateKeyError: 69 | self._locked = False 70 | return self._locked 71 | self._lock_expires = ttl 72 | self._locked = True 73 | return self._locked 74 | 75 | def release(self): 76 | if not self._locked: 77 | return False 78 | self.collection.remove( 79 | {"_id": self.lock_name, 'client_id': self._client_id}, j=True, w=1) 80 | self._locked = False 81 | return True 82 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/dependency_packages/mongomq/utils.py: -------------------------------------------------------------------------------- 1 | def enum(name, *sequential, **named): 2 | values = dict(zip(sequential, range(len(sequential))), **named) 3 | 4 | # NOTE: Yes, we *really* want to cast using str() here. 5 | # On Python 2 type() requires a byte string (which is str() on Python 2). 6 | # On Python 3 it does not matter, so we'll use str(), which acts as 7 | # a no-op. 8 | return type(str(name), (), values) -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/develop_log.py: -------------------------------------------------------------------------------- 1 | import nb_log 2 | from function_scheduling_distributed_framework import frame_config 3 | 4 | #开发时候的调试日志，比print方便通过级别一键屏蔽。 5 | develop_logger = nb_log.get_logger('fsdf_develop',log_level_int=frame_config.FSDF_DEVELOP_LOG_LEVEL) 6 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/mongo_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/9/17 0017 15:26 4 | import pymongo 5 | from function_scheduling_distributed_framework import frame_config 6 | from function_scheduling_distributed_framework.utils import decorators 7 | 8 | 9 | class MongoMixin: 10 | """ 11 | mixin类被继承，也可以直接实例化。 12 | """ 13 | @property 14 | @decorators.cached_method_result 15 | def mongo_client(self): 16 | return pymongo.MongoClient(frame_config.MONGO_CONNECT_URL, connect=False) # connect等于False原因见注释 17 | 18 | @property 19 | @decorators.cached_method_result 20 | def mongo_db_task_status(self): 21 | return self.mongo_client.get_database('task_status') -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/mqtt_util.py: -------------------------------------------------------------------------------- 1 | import urllib3 2 | import json 3 | import nb_log 4 | import decorator_libs 5 | 6 | # https://www.cnblogs.com/YrRoom/p/14054282.html 7 | 8 | """ 9 | -p 18083 服务器启动端口 10 | -p 1882 TCP端口 11 | -p 8083 WS端口 12 | -p 8084 WSS端口 13 | -p 8883 SSL端口 14 | """ 15 | 16 | """ 17 | 非常适合 前端订阅唯一uuid的topic 然后表单中带上这个topic名字请求python接口 -> 接口中发布任务到rabbitmq或redis消息队列 -> 18 | 后台消费进程执行任务消费,并将结果发布到mqtt的那个唯一uuid的topic -> mqtt 把结果推送到前端。 19 | 20 | 使用ajax轮训或者后台导入websocket相关的包来做和前端的长耗时任务的交互都是伪命题，没有mqtt好。 21 | """ 22 | 23 | 24 | class MqttHttpHelper(nb_log.LoggerMixin, nb_log.LoggerLevelSetterMixin): 25 | 26 | def __init__(self, mqtt_publish_url='http://127.0.0.1:18083/api/v2/mqtt/publish', user='admin', passwd='public', display_full_msg=False): 27 | """ 28 | :param mqtt_publish_url: mqtt的http接口，这是mqtt中间件自带的，不是重新自己实现的接口。不需要导入paho.mqtt.client,requeests urllib3即可。 29 | :param display_full_msg: 时候打印发布的任务 30 | """ 31 | self._mqtt_publish_url = mqtt_publish_url 32 | self.http = urllib3.PoolManager() 33 | self._headers = urllib3.util.make_headers(basic_auth=f'{user}:{passwd}') 34 | self._headers['Content-Type'] = 'application/json' 35 | self._display_full_msg = display_full_msg 36 | 37 | # @decorator_libs.tomorrow_threads(10) 38 | def pub_message(self, topic, msg): 39 | msg = json.dumps(msg) if isinstance(msg, (dict, list)) else msg 40 | if not isinstance(msg, str): 41 | raise Exception('推送的不是字符串') 42 | post_data = {"qos": 1, "retain": False, "topic": topic, "payload": msg} 43 | try: # UnicodeEncodeError: 'latin-1' codec can't encode character '\u6211' in position 145: Body ('我') is not valid Latin-1. Use body.encode('utf-8') if you want to send it encoded in UTF-8. 44 | resp_dict = json.loads(self.http.request('post', self._mqtt_publish_url, body=json.dumps(post_data), 45 | headers=self._headers).data) 46 | except UnicodeEncodeError as e: 47 | self.logger.warning(e) 48 | post_data['payload'] = post_data['payload'].encode().decode('latin-1') 49 | resp_dict = json.loads(self.http.request('post', self._mqtt_publish_url, body=json.dumps(post_data), 50 | headers=self._headers).data) 51 | if resp_dict['code'] == 0: 52 | self.logger.debug(f' 推送mqtt成功 ，主题名称是:{topic} ，长度是 {len(msg)}， 消息是 {msg if self._display_full_msg else msg[:200]} ') 53 | else: 54 | self.logger.debug(f' 推送mqtt失败,主题名称是:{topic},mqtt返回响应是 {json.dumps(resp_dict)} ， 消息是 {msg if self._display_full_msg else msg[:200]}') 55 | 56 | 57 | if __name__ == '__main__': 58 | with decorator_libs.TimerContextManager(): 59 | mp = MqttHttpHelper('http://192.168.6.130:18083/api/v2/mqtt/publish') 60 | for i in range(2000): 61 | mp.pub_message('/topic_test_uuid123456', 'msg_test3') 62 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/pysnooper_ydf/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Ram Rachum and collaborators. 2 | # This program is distributed under the MIT license. 3 | ''' 4 | 5 | 基于最新的改版的pysnooper，修改成彩色可点击。 6 | PySnooper - Never use print for debugging again 7 | 8 | Usage: 9 | 10 | import pysnooper 11 | 12 | @pysnooper.snoop() 13 | def your_function(x): 14 | ... 15 | 16 | A log will be written to stderr showing the lines executed and variables 17 | changed in the decorated function. 18 | 19 | For more information, see https://github.com/cool-RR/PySnooper 20 | ''' 21 | 22 | from .tracer import Tracer as snoop 23 | from .variables import Attrs, Exploding, Indices, Keys 24 | import collections 25 | 26 | __VersionInfo = collections.namedtuple('VersionInfo', 27 | ('major', 'minor', 'micro')) 28 | 29 | __version__ = '0.2.8' 30 | __version_info__ = __VersionInfo(*(map(int, __version__.split('.')))) 31 | 32 | del collections, __VersionInfo # Avoid polluting the namespace 33 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/pysnooper_ydf/pycompat.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Ram Rachum and collaborators. 2 | # This program is distributed under the MIT license. 3 | '''Python 2/3 compatibility''' 4 | 5 | import abc 6 | import os 7 | import inspect 8 | import sys 9 | import datetime as datetime_module 10 | 11 | PY3 = (sys.version_info[0] == 3) 12 | PY2 = not PY3 13 | 14 | if hasattr(abc, 'ABC'): 15 | ABC = abc.ABC 16 | else: 17 | class ABC(object): 18 | """Helper class that provides a standard way to create an ABC using 19 | inheritance. 20 | """ 21 | __metaclass__ = abc.ABCMeta 22 | __slots__ = () 23 | 24 | 25 | if hasattr(os, 'PathLike'): 26 | PathLike = os.PathLike 27 | else: 28 | class PathLike(ABC): 29 | """Abstract base class for implementing the file system path protocol.""" 30 | 31 | @abc.abstractmethod 32 | def __fspath__(self): 33 | """Return the file system path representation of the object.""" 34 | raise NotImplementedError 35 | 36 | @classmethod 37 | def __subclasshook__(cls, subclass): 38 | return ( 39 | hasattr(subclass, '__fspath__') or 40 | # Make a concession for older `pathlib` versions:g 41 | (hasattr(subclass, 'open') and 42 | 'path' in subclass.__name__.lower()) 43 | ) 44 | 45 | 46 | try: 47 | iscoroutinefunction = inspect.iscoroutinefunction 48 | except AttributeError: 49 | iscoroutinefunction = lambda whatever: False # Lolz 50 | 51 | try: 52 | isasyncgenfunction = inspect.isasyncgenfunction 53 | except AttributeError: 54 | isasyncgenfunction = lambda whatever: False # Lolz 55 | 56 | 57 | if PY3: 58 | string_types = (str,) 59 | text_type = str 60 | else: 61 | string_types = (basestring,) 62 | text_type = unicode 63 | 64 | try: 65 | from collections import abc as collections_abc 66 | except ImportError: # Python 2.7 67 | import collections as collections_abc 68 | 69 | if sys.version_info[:2] >= (3, 6): 70 | time_isoformat = datetime_module.time.isoformat 71 | else: 72 | def time_isoformat(time, timespec='microseconds'): 73 | assert isinstance(time, datetime_module.time) 74 | if timespec != 'microseconds': 75 | raise NotImplementedError 76 | result = '{:02d}:{:02d}:{:02d}.{:06d}'.format( 77 | time.hour, time.minute, time.second, time.microsecond 78 | ) 79 | assert len(result) == 15 80 | return result 81 | 82 | 83 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/rabbitmq_factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 11:51 4 | import pikav0 as pika 5 | import rabbitpy 6 | from pikav0.adapters.blocking_connection import BlockingChannel 7 | 8 | from function_scheduling_distributed_framework import frame_config 9 | 10 | 11 | class RabbitmqClientRabbitPy: 12 | """ 13 | 使用rabbitpy包。 14 | """ 15 | 16 | # noinspection PyUnusedLocal 17 | def __init__(self, username, password, host, port, virtual_host, heartbeat=0): 18 | rabbit_url = f'amqp://{username}:{password}@{host}:{port}/{virtual_host}?heartbeat={heartbeat}' 19 | self.connection = rabbitpy.Connection(rabbit_url) 20 | 21 | def creat_a_channel(self) -> rabbitpy.AMQP: 22 | return rabbitpy.AMQP(self.connection.channel()) # 使用适配器，使rabbitpy包的公有方法几乎接近pika包的channel的方法。 23 | 24 | 25 | class RabbitmqClientPika: 26 | """ 27 | 使用pika包,多线程不安全的包。 28 | """ 29 | 30 | def __init__(self, username, password, host, port, virtual_host, heartbeat=0): 31 | """ 32 | parameters = pika.URLParameters('amqp://guest:guest@localhost:5672/%2F') 33 | 34 | connection = pika.SelectConnection(parameters=parameters, 35 | on_open_callback=on_open) 36 | :param username: 37 | :param password: 38 | :param host: 39 | :param port: 40 | :param virtual_host: 41 | :param heartbeat: 42 | """ 43 | credentials = pika.PlainCredentials(username, password) 44 | self.connection = pika.BlockingConnection(pika.ConnectionParameters( 45 | host, port, virtual_host, credentials, heartbeat=heartbeat)) 46 | # self.connection = pika.SelectConnection(pika.ConnectionParameters( 47 | # host, port, virtual_host, credentials, heartbeat=heartbeat)) 48 | 49 | def creat_a_channel(self) -> BlockingChannel: 50 | return self.connection.channel() 51 | 52 | 53 | class RabbitMqFactory: 54 | def __init__(self, heartbeat=600 , is_use_rabbitpy=0): 55 | """ 56 | :param heartbeat: 57 | :param is_use_rabbitpy: 为0使用pika，多线程不安全。为1使用rabbitpy，多线程安全的包。 58 | """ 59 | if is_use_rabbitpy: 60 | self.rabbit_client = RabbitmqClientRabbitPy(frame_config.RABBITMQ_USER, frame_config.RABBITMQ_PASS, 61 | frame_config.RABBITMQ_HOST, frame_config.RABBITMQ_PORT, 62 | frame_config.RABBITMQ_VIRTUAL_HOST, heartbeat) 63 | else: 64 | self.rabbit_client = RabbitmqClientPika(frame_config.RABBITMQ_USER, frame_config.RABBITMQ_PASS, 65 | frame_config.RABBITMQ_HOST, frame_config.RABBITMQ_PORT, 66 | frame_config.RABBITMQ_VIRTUAL_HOST, heartbeat) 67 | 68 | def get_rabbit_cleint(self): 69 | return self.rabbit_client 70 | -------------------------------------------------------------------------------- /function_scheduling_distributed_framework/utils/redis_manager.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | import redis2 as redis 3 | import redis3 4 | from function_scheduling_distributed_framework import frame_config 5 | from function_scheduling_distributed_framework.utils import decorators 6 | 7 | 8 | class RedisManager(object): 9 | _pool_dict = {} 10 | 11 | def __init__(self, host='127.0.0.1', port=6379, db=0, password='123456'): 12 | if (host, port, db, password) not in self.__class__._pool_dict: 13 | # print ('创建一个连接池') 14 | self.__class__._pool_dict[(host, port, db, password)] = redis.ConnectionPool(host=host, port=port, db=db, 15 | password=password) 16 | self._r = redis.Redis(connection_pool=self._pool_dict[(host, port, db, password)]) 17 | self._ping() 18 | 19 | def get_redis(self): 20 | """ 21 | :rtype :redis.Redis 22 | """ 23 | return self._r 24 | 25 | def _ping(self): 26 | try: 27 | self._r.ping() 28 | except Exception as e: 29 | raise e 30 | 31 | 32 | # noinspection PyArgumentEqualDefault 33 | class RedisMixin(object): 34 | """ 35 | 可以被作为万能mixin能被继承，也可以单独实例化使用。 36 | """ 37 | 38 | @property 39 | @decorators.cached_method_result 40 | def redis_db0(self): 41 | return RedisManager(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=0).get_redis() 42 | 43 | @property 44 | @decorators.cached_method_result 45 | def redis_db8(self): 46 | return RedisManager(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=8).get_redis() 47 | 48 | @property 49 | @decorators.cached_method_result 50 | def redis_db7(self): 51 | return RedisManager(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=7).get_redis() 52 | 53 | @property 54 | @decorators.cached_method_result 55 | def redis_db6(self): 56 | return RedisManager(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=6).get_redis() 57 | 58 | @property 59 | @decorators.cached_method_result 60 | def redis_db_frame(self): 61 | return RedisManager(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=frame_config.REDIS_DB).get_redis() 62 | 63 | @property 64 | @decorators.cached_method_result 65 | def redis_db_frame_version3(self): 66 | return redis3.Redis(host=frame_config.REDIS_HOST, port=frame_config.REDIS_PORT, password=frame_config.REDIS_PASSWORD, db=frame_config.REDIS_DB,decode_responses=True) 67 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ## 这里的不用管，可能没及时更新维护， pip install function_scheduling_distributed_framework 自动安装所有依赖包 2 | 3 | eventlet==0.31.0 4 | gevent==21.1.2 5 | pymongo==3.5.1 6 | AMQPStorm==2.7.1 7 | rabbitpy==2.0.1 8 | gnsq==1.0.1 9 | kafka-python==1.4.6 10 | mongo-mq==0.0.1 11 | persist-queue>=0.4.2 12 | redis 13 | decorator==4.4.0 14 | # pysnooper==0.0.11 15 | tomorrow3==1.1.0 16 | concurrent-log-handler==0.9.19 17 | elasticsearch 18 | requests 19 | psutil 20 | Flask 21 | flask_bootstrap 22 | flask_wtf 23 | wtforms 24 | flask_login 25 | sqlalchemy 26 | sqlalchemy_utils==0.36.1 27 | apscheduler==3.7.0 28 | PyWin32 # windows need because of concurrent-log-handler 29 | # pip install -r requirements.text -i http://mirrors.aliyun.com/pypi/simple/ 30 | pikav0 31 | pikav1 32 | rocketmq 33 | zmq 34 | pyzmq 35 | kombu==4.6.11 36 | confluent_kafka==1.5.0 37 | paho-mqtt 38 | aiohttp 39 | setuptools~=39.0.1 40 | urllib3~=1.22 41 | six~=1.11.0 42 | redis3 43 | fabric2 44 | nats-python 45 | 46 | 47 | ## 这里的不用管，可能没及时更新维护， pip install function_scheduling_distributed_framework 自动安装所有依赖包 -------------------------------------------------------------------------------- /test_frame/best_simple_example/README.md: -------------------------------------------------------------------------------- 1 | 演示框架的使用，演示最简单的基于本地持久化消息队列的用法，运行此例子不需要安装任何中间件，有python环境就可以运行。 2 | 3 | 4 | 自动在当前硬盘下的根目录的\sqllite_queues，创建sqlite数据库文件，可以pycahrm专业版打开数据库查看。 5 | 6 | 7 | 正式使用墙裂推荐安装rabbitmq中间件，使用rabbitmq作为消息队列是最好的。不建议redis，redis的list只是个数组结构，很多消息队列的功能redis的list都没有，连最重要的消费确认都不支持，导致随意反复关闭运行中的程序、 突然关机、突然断电这些情况下，会丢失一部分任务。 8 | 9 | 此例子没有利用到框架中除分布式以外的10种功能。 10 | 11 | 12 | ### 装饰器版本的运行方式，查看test_decorator_run_example的例子。 -------------------------------------------------------------------------------- /test_frame/best_simple_example/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | 6 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum, fabric_deploy 7 | 8 | 9 | # 通过设置broker_kind，一键切换中间件为mq或redis等20种中间件或包。 10 | # 通过设置concurrent_mode，来设置并发方式，改这一个参数能自动支持threading eventlet gevent asyncio并发 11 | # 通过设置qps，能够精确的指定每秒运行多少次函数，无视函数需要耗时多久。 12 | # 通过设置concurrent_num，设置并发大小。此例，设置300的线程池大小，但qps为6，函数需要耗时5秒钟， 13 | # 框架的智能线程池会自动只开30个线程，不会造成启动线程太多切换影响效率性能，所以可以自信大胆的设置线程数量。 14 | # 智能线程池能自动扩大也能自动缩小线程数量，例如某段时间函数耗时大，会增大线程数量来达到qps，如果函数耗时变小了，会自动缩小线程数量，框架不需要提前知道函数的确定耗时，会自动调节并发数量的。 15 | # 还有其他30种函数运行控制参数，看代码里面的函数入参说明，说的非常详细了。 16 | 17 | # @task_deco('queue_test2', ) # @task_deco必须参数只有一个。 18 | @task_deco('queue_test2', qps=6, broker_kind=BrokerEnum.PERSISTQUEUE) 19 | def f2(a, b): 20 | sleep_time = 7 21 | result = a + b 22 | print(f'消费此消息 {a} + {b} 中。。。。。,此次需要消耗 {sleep_time} 秒') 23 | time.sleep(sleep_time) # 模拟做某事需要阻塞n秒种，必须用并发绕过此阻塞。 24 | print(f'{a} + {b} 的结果是 {result}') 25 | return result 26 | 27 | 28 | if __name__ == '__main__': 29 | print(f2.__name__) 30 | f2.clear() 31 | for i in range(200): 32 | f2.push(i, i * 2) 33 | 34 | -------------------------------------------------------------------------------- /test_frame/best_simple_example/test_publish.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | 6 | from test_frame.best_simple_example.test_consume import f2 7 | 8 | f2.clear() 9 | 10 | for i in range(10000): 11 | time.sleep(0.1) 12 | f2.pub({'a': i, 'b': 2 * i}) # pub这是发布字典，另外还能设置函数控制参数 13 | f2.push(i*10, i * 20) # push这是发布函数参数形式，不是发布一个字典的形式 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /test_frame/best_simple_example/test_qps_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | import threading 6 | from function_scheduling_distributed_framework import task_deco, BrokerEnum,ConcurrentModeEnum 7 | 8 | t_start = time.time() 9 | 10 | @task_deco('queue_test2_qps', qps=2, broker_kind=BrokerEnum.PERSISTQUEUE,concurrent_mode=ConcurrentModeEnum.THREADING,concurrent_num=600 ) 11 | def f2(a, b): 12 | """ 13 | concurrent_num = 600 不用怕，因为这是智能线程池，如果函数耗时短，不会真开那么多线程。 14 | 这个例子是测试函数耗时是动态变化的，这样就不可能通过提前设置参数预估函数固定耗时和搞鬼了。看看能不能实现qps稳定和线程池自动扩大自动缩小 15 | 要说明的是打印的线程数量也包含了框架启动时候几个其他的线程，所以数量不是刚好和所需的线程计算一样的。 16 | 17 | ## 可以在运行控制台搜索 新启动线程 这个关键字，看看是不是何时适合扩大线程数量。 18 | ## 可以在运行控制台搜索 停止线程 这个关键字，看看是不是何时适合缩小线程数量。 19 | """ 20 | result = a + b 21 | sleep_time = 0.01 22 | if time.time() - t_start > 60: # 先测试函数耗时慢慢变大了，框架能不能按需自动增大线程数量 23 | sleep_time = 7 24 | if time.time() - t_start > 120: 25 | sleep_time = 31 26 | if time.time() - t_start > 200: 27 | sleep_time = 79 28 | if time.time() - t_start > 400: # 最后把函数耗时又减小，看看框架能不能自动缩小线程数量。 29 | sleep_time = 0.8 30 | if time.time() - t_start > 500: 31 | sleep_time = None 32 | print(f'{time.strftime("%H:%M:%S")} ，当前线程数量是 {threading.active_count()}, {a} + {b} 的结果是 {result}， sleep {sleep_time} 秒') 33 | if sleep_time is not None: 34 | time.sleep(sleep_time) # 模拟做某事需要阻塞n秒种，必须用并发绕过此阻塞。 35 | return result 36 | 37 | if __name__ == '__main__': 38 | f2.clear() 39 | for i in range(1400): 40 | f2.push(i, i * 2) 41 | f2.consume() -------------------------------------------------------------------------------- /test_frame/car_home_crawler_sample/car_home_publisher.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import fsdf_background_scheduler 2 | from test_frame.car_home_crawler_sample.car_home_consumer import crawl_list_page, crawl_detail_page 3 | 4 | crawl_list_page.clear() # 清空列表页 5 | crawl_detail_page.clear() # 清空详情页 6 | 7 | # # 推送列表页首页，同时设置翻页为True 8 | crawl_list_page.push('news', 1, do_page_turning=True) # 新闻 9 | crawl_list_page.push('advice', page=1, do_page_turning=True) # 导购 10 | crawl_list_page.push(news_type='drive', page=1, do_page_turning=True) # 驾驶评测 11 | 12 | 13 | for news_typex in ['news', 'advice', 'drive']: # 定时任务，语法入参是apscheduler包相同。每隔120秒查询一次首页更新,这个可以不要。 14 | fsdf_background_scheduler.add_timing_publish_job(crawl_list_page, 'interval', seconds=120, kwargs={"news_type": news_typex, "page": 1, "do_page_turning": False}) 15 | fsdf_background_scheduler.start() # 启动首页查询有没有新的新闻的定时发布任务 16 | -------------------------------------------------------------------------------- /test_frame/car_home_crawler_sample/carhome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/car_home_crawler_sample/carhome.png -------------------------------------------------------------------------------- /test_frame/car_home_crawler_sample/carhome2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/car_home_crawler_sample/carhome2.png -------------------------------------------------------------------------------- /test_frame/car_home_crawler_sample/carhome3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/car_home_crawler_sample/carhome3.png -------------------------------------------------------------------------------- /test_frame/car_home_crawler_sample/kuaisu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/car_home_crawler_sample/kuaisu.png -------------------------------------------------------------------------------- /test_frame/complex_example/README.md: -------------------------------------------------------------------------------- 1 | 演示复杂用法。 2 | 3 | 一次性启动多个消费者。 4 | 5 | 模拟单核cpu已达到100%，需要开启多进程的用法，演示使用多进程加多线程（协程）。 6 | 7 | 这里使用redis作为中间件演示，需要安装redis。正式墙裂推荐安装mq。 8 | -------------------------------------------------------------------------------- /test_frame/complex_example/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | from multiprocessing import Process 5 | import time 6 | from function_scheduling_distributed_framework import get_consumer, get_publisher, AbstractConsumer 7 | from function_scheduling_distributed_framework.consumers.redis_consumer import RedisConsumer 8 | from function_scheduling_distributed_framework.utils import LogManager 9 | 10 | 11 | 12 | logger = LogManager('complex_example').get_logger_and_add_handlers() 13 | 14 | pb2 = get_publisher('task2_queue', broker_kind=2) 15 | 16 | 17 | def task1(x, y): 18 | logger.info(f'消费此消息 {x} - {y} ,结果是 {x - y}') 19 | for i in range(10): 20 | pb2.publish({'n': x * 100 + i}) # 消费时候发布任务到别的队列或自己的队列。可以边消费边推送。 21 | time.sleep(10) # 模拟做某事需要阻塞10秒种，必须用并发绕过此阻塞。 22 | 23 | 24 | def task2(n): 25 | logger.info(n) 26 | time.sleep(3) 27 | 28 | 29 | def multi_processing_consume(): 30 | get_consumer('task1_queue', consuming_function=task1, broker_kind=2).start_consuming_message() 31 | RedisConsumer('task2_queue', consuming_function=task2, threads_num=100).start_consuming_message() 32 | AbstractConsumer.join_shedual_task_thread() # linux多进程启动时候一定要加这一句,否则即使是while 1 的线程如果不join，子进程也会迅速退出。windows下可以不需要这一句。 33 | 34 | 35 | if __name__ == '__main__': 36 | [Process(target=multi_processing_consume).start() for _ in range(4)] 37 | -------------------------------------------------------------------------------- /test_frame/complex_example/test_consume2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | from multiprocessing import Process 5 | import time 6 | from function_scheduling_distributed_framework import get_consumer, get_publisher, AbstractConsumer 7 | from function_scheduling_distributed_framework.consumers.redis_consumer import RedisConsumer 8 | from function_scheduling_distributed_framework.utils import LogManager 9 | 10 | logger = LogManager('complex_example').get_logger_and_add_handlers() 11 | pb2 = get_publisher('task2_queue', broker_kind=2) 12 | 13 | 14 | def task1(x, y): 15 | logger.info(f'消费此消息 {x} - {y} ,结果是 {x - y}') 16 | for i in range(10): 17 | pb2.publish({'n': x * 100 + i}) # 消费时候发布任务到别的队列或自己的队列。可以边消费边推送。 18 | time.sleep(10) # 模拟做某事需要阻塞10秒种，必须用并发绕过此阻塞。 19 | 20 | 21 | def task2(n): 22 | logger.info(n) 23 | time.sleep(3) 24 | 25 | 26 | def multi_processing_consume(): 27 | get_consumer('task1_queue', consuming_function=task1, broker_kind=2).start_consuming_message() 28 | RedisConsumer('task2_queue', consuming_function=task2, threads_num=100).start_consuming_message() 29 | AbstractConsumer.join_shedual_task_thread() # linux多进程启动时候一定要加这一句,否则即使是while 1 的线程如果不join，子进程也会迅速退出。windows下可以不需要这一句。 30 | 31 | 32 | if __name__ == '__main__': 33 | [Process(target=multi_processing_consume).start() for _ in range(4)] 34 | -------------------------------------------------------------------------------- /test_frame/complex_example/test_publish.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | 6 | from function_scheduling_distributed_framework import get_publisher 7 | 8 | pb = get_publisher('task1_queue', broker_kind=2) 9 | 10 | for i in range(100): 11 | pb.publish({'x':i,'y':i * 2}) -------------------------------------------------------------------------------- /test_frame/d3/t6.py: -------------------------------------------------------------------------------- 1 | 2 | print('导入nb_log之前的print是普通的') 3 | import time 4 | import sys 5 | print(sys.path) 6 | 7 | from nb_log import get_logger 8 | 9 | # logger = get_logger('lalala',log_filename='jinzhifengzhuang.log',formatter_template=5) 10 | 11 | # logger.debug(f'debug是绿色，说明是调试的，代码ok ') 12 | # logger.info('info是天蓝色，日志正常 ') 13 | # logger.warning('黄色yello，有警告了 ') 14 | # logger.error('粉红色说明代码有错误 ') 15 | # logger.critical('血红色，说明发生了严重错误 ') 16 | 17 | # print('导入nb_log之后的print是强化版的可点击跳转的') 18 | 19 | 20 | 21 | #raise Exception("dsadsd") 22 | 23 | while 1: 24 | time.sleep(5) -------------------------------------------------------------------------------- /test_frame/git_fsdf.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import subprocess 3 | import os 4 | import time 5 | 6 | # os.environ["path"] = r'C:\Program Files\Git\mingw64\libexec\git-core' 7 | def getstatusoutput(cmd): 8 | try: 9 | data = subprocess.check_output(cmd, shell=True, universal_newlines=True, 10 | stderr=subprocess.STDOUT, encoding='utf8') 11 | exitcode = 0 12 | except subprocess.CalledProcessError as ex: 13 | data = ex.output 14 | exitcode = ex.returncode 15 | if data[-1:] == '\n': 16 | data = data[:-1] 17 | return exitcode, data 18 | 19 | 20 | def do_cmd(cmd_strx): 21 | print(f'执行 {cmd_strx}') 22 | retx = getstatusoutput(cmd_strx) 23 | print(retx[0]) 24 | # if retx[0] !=0: 25 | # raise ValueError('要检查git提交') 26 | print(retx[1], '\n') 27 | return retx 28 | 29 | 30 | t0 = time.time() 31 | 32 | do_cmd('git pull') 33 | 34 | do_cmd('git diff') 35 | 36 | do_cmd('git add ../.') 37 | 38 | do_cmd('git commit -m commit') 39 | 40 | do_cmd('git push origin') 41 | 42 | # print(subprocess.getstatusoutput('git push github')) 43 | print(f'{time.strftime("%H:%M:%S")} spend_time {time.time() - t0}') 44 | time.sleep(100000) 45 | -------------------------------------------------------------------------------- /test_frame/git_fsdf_github.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import subprocess 3 | import os 4 | import time 5 | 6 | # os.environ["path"] = r'C:\Program Files\Git\mingw64\libexec\git-core' 7 | def getstatusoutput(cmd): 8 | try: 9 | data = subprocess.check_output(cmd, shell=True, universal_newlines=True, 10 | stderr=subprocess.STDOUT, encoding='utf8') 11 | exitcode = 0 12 | except subprocess.CalledProcessError as ex: 13 | data = ex.output 14 | exitcode = ex.returncode 15 | if data[-1:] == '\n': 16 | data = data[:-1] 17 | return exitcode, data 18 | 19 | 20 | def do_cmd(cmd_strx): 21 | print(f'执行 {cmd_strx}') 22 | retx = getstatusoutput(cmd_strx) 23 | print(retx[0]) 24 | # if retx[0] !=0: 25 | # raise ValueError('要检查git提交') 26 | print(retx[1], '\n') 27 | return retx 28 | 29 | 30 | t0 = time.time() 31 | 32 | do_cmd('git pull') 33 | 34 | do_cmd('git diff') 35 | 36 | do_cmd('git add ../.') 37 | 38 | do_cmd('git commit -m commit') 39 | 40 | do_cmd('git push origin') 41 | 42 | do_cmd('git push github') 43 | 44 | # print(subprocess.getstatusoutput('git push github')) 45 | print(f'{time.strftime("%H:%M:%S")} spend_time {time.time() - t0}') 46 | time.sleep(100000) 47 | -------------------------------------------------------------------------------- /test_frame/jietu/20210428-211001.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/20210428-211001.mp4 -------------------------------------------------------------------------------- /test_frame/jietu/20210428-2_clip.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/20210428-2_clip.gif -------------------------------------------------------------------------------- /test_frame/jietu/20210428-2_clip1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/20210428-2_clip1.gif -------------------------------------------------------------------------------- /test_frame/jietu/5种数据库模拟消息队列.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/5种数据库模拟消息队列.png -------------------------------------------------------------------------------- /test_frame/jietu/QQ图片20190923130527.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/QQ图片20190923130527.png -------------------------------------------------------------------------------- /test_frame/jietu/celery_dir2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/celery_dir2.png -------------------------------------------------------------------------------- /test_frame/jietu/celery_proj_dir.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/celery_proj_dir.png -------------------------------------------------------------------------------- /test_frame/jietu/linuxgevent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/linuxgevent.png -------------------------------------------------------------------------------- /test_frame/jietu/linux上运行使用gevent模式的截图2a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/linux上运行使用gevent模式的截图2a.png -------------------------------------------------------------------------------- /test_frame/jietu/pysnooper改版测试截图.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/pysnooper改版测试截图.png -------------------------------------------------------------------------------- /test_frame/jietu/win运行.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/win运行.png -------------------------------------------------------------------------------- /test_frame/jietu/~$流程图.docx: -------------------------------------------------------------------------------- 1 | Administrator Administrator -------------------------------------------------------------------------------- /test_frame/jietu/五彩日志.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/五彩日志.png -------------------------------------------------------------------------------- /test_frame/jietu/任务消费统计曲线.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/任务消费统计曲线.png -------------------------------------------------------------------------------- /test_frame/jietu/函数状态持久化.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/函数状态持久化.png -------------------------------------------------------------------------------- /test_frame/jietu/函数精确控频运行完成100次每秒.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/函数精确控频运行完成100次每秒.png -------------------------------------------------------------------------------- /test_frame/jietu/函数结果和运行次数和错误异常查看.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/函数结果和运行次数和错误异常查看.png -------------------------------------------------------------------------------- /test_frame/jietu/彩色和可跳转演示.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/彩色和可跳转演示.png -------------------------------------------------------------------------------- /test_frame/jietu/我开发时候的工具和方式.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/我开发时候的工具和方式.png -------------------------------------------------------------------------------- /test_frame/jietu/操作文档图片.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/操作文档图片.png -------------------------------------------------------------------------------- /test_frame/jietu/流程图.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/流程图.docx -------------------------------------------------------------------------------- /test_frame/jietu/运行截图.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/jietu/运行截图.png -------------------------------------------------------------------------------- /test_frame/log_example.py: -------------------------------------------------------------------------------- 1 | print('导入nb_log之前的print是普通的') 2 | 3 | from nb_log import get_logger 4 | 5 | print('导入nb_log之后的print是强化版的可点击跳转的') 6 | 7 | logger = get_logger('lalala', log_filename='lalala.log') 8 | 9 | for i in range(100): 10 | logger.debug(f'debug是绿色，说明是调试的，代码ok。 ' * 4) 11 | logger.info('info是天蓝色，日志正常。 ' * 4) 12 | logger.warning('黄色yello，有警告了。 ' * 4) 13 | logger.error('粉红色说明代码有错误。 ' * 4) 14 | logger.critical('血红色，说明发生了严重错误。 ' * 4) 15 | print('只要导入nb_log一次之后的项目任意文件的print是强化版的可点击跳转的，在输出控制台点击行号能自动打开文件跳转到精确行号。') 16 | 17 | -------------------------------------------------------------------------------- /test_frame/multi_steps_conusme/README.md: -------------------------------------------------------------------------------- 1 | # 演示多个步骤的连续消费 -------------------------------------------------------------------------------- /test_frame/multi_steps_conusme/multi_steps_example1.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum,ConcurrentModeEnum 4 | import gevent.monkey;gevent.monkey.patch_all() 5 | 6 | @task_deco('queue_test_step1', qps=0.5, broker_kind=BrokerEnum.REDIS_ACK_ABLE,concurrent_mode=ConcurrentModeEnum.GEVENT) 7 | def step1(x): 8 | print(f'x 的值是 {x}') 9 | if x == 0: 10 | for i in range(1, 3): 11 | step1.pub(dict(x=x + i)) 12 | for j in range(10): 13 | step2.push(y=x * 100 + j) # push是直接发送多个参数，pub是发布一个字典 14 | time.sleep(10) 15 | 16 | 17 | @task_deco('queue_test_step2', qps=3, broker_kind=BrokerEnum.REDIS_ACK_ABLE) 18 | def step2(y): 19 | print(f'y 的值是 {y}') 20 | time.sleep(10) 21 | 22 | 23 | if __name__ == '__main__': 24 | # step1.clear() 25 | step1.push(0) 26 | 27 | step1.consume() # 可以连续启动两个消费者，sonusme是启动独立线程里面while 1调度的，所以可以连续运行多个启动消费。 28 | step2.consume() 29 | -------------------------------------------------------------------------------- /test_frame/multi_steps_conusme/multi_steps_example2.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import get_consumer, task_deco 2 | 3 | 4 | def step1(x): 5 | print(f'x 的值是 {x}') 6 | if x == 0: 7 | for i in range(1, 10): 8 | consumer1.publisher_of_same_queue.publish(dict(x=x + i)) 9 | for j in range(10): 10 | consumer2.publisher_of_same_queue.publish(dict(y=x * 100 + j)) 11 | 12 | 13 | def step2(y): 14 | print(f'y 的值是 {y}') 15 | 16 | 17 | consumer1 = get_consumer('queue_test_step1', consuming_function=step1) 18 | consumer2 = get_consumer('queue_test_step2', consuming_function=step2) 19 | consumer1.publisher_of_same_queue.clear() 20 | 21 | consumer1.publisher_of_same_queue.publish({'x': 0}) 22 | 23 | consumer1.start_consuming_message() 24 | consumer2.start_consuming_message() 25 | 26 | -------------------------------------------------------------------------------- /test_frame/other_tests/test2.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from copyreg import pickle 3 | 4 | # print(datetime.datetime.now()) 5 | import time 6 | 7 | # import nb_log 8 | # import decorator_libs 9 | # import pysnooper_click_able 10 | t1 = time.time() 11 | 12 | # 13 | # # @pysnooper_click_able.snoop(depth=10000) 14 | # def f(): 15 | # import function_scheduling_distributed_framework 16 | # 17 | # 18 | # f() 19 | # print(time.time() - t1) 20 | # import pickle 21 | # import threading 22 | # import redis 23 | # 24 | # class CannotPickleObject: 25 | # def __init__(self): 26 | # self._lock = threading.Lock() 27 | # 28 | # 29 | # class CannotPickleObject2: 30 | # def __init__(self): 31 | # self._redis = redis.Redis() 32 | # 33 | # print(pickle.dumps(CannotPickleObject())) 34 | # print(pickle.dumps(CannotPickleObject2())) 35 | 36 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, IdeAutoCompleteHelper, PriorityConsumingControlConfig, run_consumer_with_multi_process 37 | 38 | 39 | @task_deco('test_queue', broker_kind=BrokerEnum.REDIS) 40 | def ff(x, y): 41 | import os 42 | time.sleep(10) 43 | print(os.getpid(), x, y) 44 | 45 | 46 | if __name__ == '__main__': 47 | # ff.publish() 48 | ff.clear() # 清除 49 | for i in range(1000): 50 | ff.push(i, y=i * 2) 51 | 52 | # 这个与push相比是复杂的发布，第一个参数是函数本身的入参字典，后面的参数为任务控制参数，例如可以设置task_id，设置延时任务，设置是否使用rpc模式等。 53 | ff.publish({'x': i * 10, 'y': i * 2}, priority_control_config=PriorityConsumingControlConfig(countdown=1, misfire_grace_time=1)) 54 | 55 | ff(666, 888) # 直接运行函数 56 | ff.start() # 和 conusme()等效 57 | ff.consume() # 和 start()等效 58 | run_consumer_with_multi_process(ff, 2) # 启动两个进程 59 | ff.multi_process_start(2) # 启动两个进程，和上面的run_consumer_with_multi_process等效,现在新增这个multi_process_start方法。 60 | # IdeAutoCompleteHelper(ff).multi_process_start(3) # IdeAutoCompleteHelper 可以补全提示，但现在装饰器加了类型注释，ff. 已近可以在pycharm下补全了。 61 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_apscheduler.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework.utils.apscheduler_monkey import patch_run_job 2 | patch_run_job() 3 | 4 | from apscheduler.events import EVENT_JOB_MISSED 5 | from apscheduler.executors.asyncio import AsyncIOExecutor 6 | from apscheduler.executors.pool import ThreadPoolExecutor 7 | from apscheduler.jobstores.redis import RedisJobStore 8 | from apscheduler.schedulers.background import BackgroundScheduler 9 | import datetime 10 | import time 11 | from apscheduler.schedulers . blocking import BlockingScheduler 12 | import threading 13 | import decorator_libs 14 | 15 | def func(x,y,d): 16 | time.sleep(5) 17 | print(f'{d} {x} + {y} = {x + y}') 18 | 19 | scheduler = BackgroundScheduler() 20 | # scheduler.add_job(func,'date', run_date=datetime.datetime(2021, 5, 19, 10, 53, 20), args=(5, 6,)) 21 | # print(scheduler._executors) 22 | scheduler.add_executor(ThreadPoolExecutor(1)) 23 | # scheduler.add_jobstore(RedisJobStore(jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',db=6)) 24 | # scheduler.add_jobstore() 25 | 26 | 27 | def job_miss(event): 28 | # print(event) 29 | # print(event.__dict__) 30 | # print(scheduler.get_job(event.job_id)) 31 | print(event.function,event.function_kwargs,event.function_args) 32 | 33 | 34 | scheduler.add_listener(job_miss, EVENT_JOB_MISSED) 35 | scheduler.start() 36 | 37 | with decorator_libs.TimerContextManager(): 38 | for i in range(10): 39 | run_date = datetime.datetime(2021, 5, 19, 18, 56, 30) + datetime.timedelta(seconds=i) 40 | scheduler.add_job(func,'date',run_date=run_date , args=(i, i,run_date),misfire_grace_time=None) 41 | 42 | 43 | print(datetime.datetime.now()) 44 | print('over') 45 | 46 | 47 | while 1: 48 | # print(threading.active_count()) 49 | # print(scheduler._executors) 50 | time.sleep(2) -------------------------------------------------------------------------------- /test_frame/other_tests/test_color.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2020/1/13 0013 9:15 4 | print('测试没有没有导入框架之前，print应该是白颜色,并且在控制台不知道是从哪个文件哪一行print出来的') 5 | 6 | # 只要任意地方导入一次框架后，项目任意模块里面的print自动发生变化，只要是在此后发生的print一律变色＆可跳转。print变色和日志变色是独立的,可以按需选择。 7 | from function_scheduling_distributed_framework import LogManager 8 | 9 | print('导入utils_ydf后,自动打了print猴子补丁，此行颜色i你该发生变化，和pycharm下可点击跳转') 10 | 11 | logger = LogManager('lalala').get_logger_and_add_handlers() 12 | 13 | for i in range(3): 14 | logger.debug(f'在console里面设置为monokai主题下测试debug颜色{i}') 15 | logger.info(f'在console里面设置为monokai主题下测试info颜色{i}') 16 | logger.warning(f'在console里面设置为monokai主题下测试warning颜色{i}') 17 | logger.error(f'在console里面设置为monokai主题下测试error颜色{i}') 18 | logger.critical(f'在console里面设置为monokai主题下测试 严重错误 颜色{i}') 19 | print(f'直接print也会变色和可自动跳转，因为打了猴子补丁{i}') 20 | 21 | 22 | 23 | # print('''File "D:/coding2/distributed_framework/test_frame/other_tests/test_color.py", line ''') 24 | # 25 | # print('''"D:/coding2/distributed_framework/test_frame/other_tests/test_color.py"''') 26 | # 27 | # raise Exception('aaaaaaaa') -------------------------------------------------------------------------------- /test_frame/other_tests/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | import random 6 | from function_scheduling_distributed_framework import task_deco, LogManager,BrokerEnum 7 | 8 | 9 | 10 | 11 | @task_deco('queue_test2', qps=30, broker_kind=BrokerEnum.REDIS,) # 通过设置broker_kind，一键切换中间件为mq或redis等13种中间件或包。 12 | def f2(a, b): 13 | print(f'消费此消息 {a} + {b} 中。。。。。') 14 | time.sleep(random.randint(1,1000)/100.0) # 模拟做某事需要随机消耗一段时间的，精确控频 15 | print(f'计算 {a} + {b} 得到的结果是 {a + b}') 16 | 17 | 18 | if __name__ == '__main__': 19 | f2.clear() 20 | for i in range(200000): 21 | f2.push(i, i * 2) 22 | f2.consume() -------------------------------------------------------------------------------- /test_frame/other_tests/test_gevent_process.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from gevent import monkey 4 | 5 | monkey.patch_all() 6 | from multiprocessing import Process 7 | 8 | print(6666) 9 | def f(x): 10 | print(x) 11 | time.sleep(10000) 12 | 13 | if __name__ == '__main__': 14 | [Process(target=f,args=(2,)).start() for i in range(2)] -------------------------------------------------------------------------------- /test_frame/other_tests/test_import_time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | print(1,datetime.datetime.now()) 3 | import apscheduler 4 | 5 | print(2,datetime.datetime.now()) 6 | import gevent 7 | print(3,datetime.datetime.now()) 8 | import eventlet 9 | print(4,datetime.datetime.now()) 10 | import asyncio 11 | 12 | 13 | print(5,datetime.datetime.now()) 14 | 15 | import threading 16 | 17 | print(6,datetime.datetime.now()) 18 | 19 | import pymongo 20 | 21 | print(7,datetime.datetime.now()) 22 | import redis 23 | 24 | print(8,datetime.datetime.now()) 25 | 26 | import pysnooper 27 | 28 | print(9,datetime.datetime.now()) -------------------------------------------------------------------------------- /test_frame/other_tests/test_monitor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/9/18 0018 12:15 4 | import time 5 | 6 | from function_scheduling_distributed_framework import nb_print 7 | from function_scheduling_distributed_framework.utils.resource_monitoring import ResourceMonitor 8 | 9 | 10 | 11 | def test_monitor(): 12 | while 1: 13 | monitor = ResourceMonitor().set_log_level(10) 14 | nb_print(monitor.cpu_count) 15 | nb_print(monitor.get_current_process_memory()) 16 | nb_print(monitor.get_current_process_cpu()) 17 | nb_print(monitor.get_os_cpu_percpu()) 18 | nb_print(monitor.get_os_cpu_avaragecpu()) 19 | nb_print(monitor.get_os_cpu_totalcpu()) 20 | nb_print(monitor.get_os_virtual_memory()) 21 | nb_print(monitor.get_os_net_info()) 22 | time.sleep(1) 23 | nb_print(monitor.get_os_net_info()) 24 | nb_print(monitor.get_all_info()) 25 | 26 | 27 | if __name__ == '__main__': 28 | # test_monitor() 29 | monitorx = ResourceMonitor(is_save_info_to_mongo=True).set_log_level(10) 30 | # monitorx.start_build_info_loop(5) 31 | 32 | monitorx.start_build_info_loop_on_daemon_thread(5) 33 | while True: 34 | time.sleep(10) 35 | 36 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_nats.py: -------------------------------------------------------------------------------- 1 | from pynats import NATSClient 2 | import nb_log 3 | # #pip install nats-python 4 | # with NATSClient('nats://192.168.6.134:4222') as client: 5 | # # Connect 6 | # client.connect() 7 | # 8 | # # Subscribe 9 | # def callback(msg): 10 | # print(type(msg)) 11 | # print(f"Received a message with subject {msg.subject}: {msg}") 12 | # 13 | # client.subscribe(subject="test-subject", callback=callback) 14 | # 15 | # # Publish a message 16 | # for i in range(10): 17 | # client.publish(subject="test-subject", payload=f"test-payload_{i}".encode()) 18 | # 19 | # # wait for 1 message 20 | # client.wait() 21 | 22 | 23 | client = NATSClient('nats://192.168.6.134:4222') 24 | # Connect 25 | client.connect() 26 | 27 | # Subscribe 28 | def callback(msg): 29 | print(type(msg)) 30 | print(f"Received a message with subject {msg.subject}: {msg}") 31 | 32 | client.subscribe(subject="test_queue66c", callback=callback) 33 | 34 | # Publish a message 35 | for i in range(10): 36 | client.publish(subject="test-subject2", payload=f"test-payload_{i}".encode()) 37 | 38 | # wait for 1 message 39 | client.wait() -------------------------------------------------------------------------------- /test_frame/other_tests/test_pulsar.py: -------------------------------------------------------------------------------- 1 | """" 2 | 3 | https://blog.csdn.net/pchwenwenti/article/details/83143345 4 | 5 | pulsar manager 账号密码 "name": "admin", "password": "apachepulsar", 6 | 7 | pip install pulsar-cleint 8 | 9 | http://192.168.6.130:9527/#/login?redirect=%2Fmanagement%2Ftenants 10 | """ 11 | from auto_run_on_remote import run_current_script_on_remote 12 | run_current_script_on_remote() 13 | 14 | import pulsar 15 | 16 | client = pulsar.Client('pulsar://localhost:6650') 17 | 18 | producer = client.create_producer('my-topic36') 19 | 20 | for i in range(10): 21 | producer.send(('Hello-%d' % i).encode('utf-8')) 22 | 23 | client.close() 24 | 25 | 26 | """ 27 | import pulsar 28 | 29 | client = pulsar.Client('pulsar://localhost:6650') 30 | 31 | consumer = client.subscribe('my-topic', 'my-subscription') 32 | 33 | while True: 34 | msg = consumer.receive() 35 | try: 36 | print("Received message '{}' id='{}'".format(msg.data(), msg.message_id())) 37 | # Acknowledge successful processing of the message 38 | consumer.acknowledge(msg) 39 | except: 40 | # Message failed to be processed 41 | consumer.negative_acknowledge(msg) 42 | 43 | client.close() 44 | """ 45 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_pysnooper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/11/6 0006 16:49 4 | 5 | ''' 6 | 测试改版的pysnooper装饰器。 7 | ''' 8 | import requests 9 | from function_scheduling_distributed_framework.utils import RedisMixin, pysnooper_ydf, LogManager 10 | 11 | 12 | 13 | 14 | def foo(): 15 | raise TypeError('bad') 16 | 17 | 18 | def bar(): 19 | try: 20 | foo() 21 | except Exception: 22 | str(1) 23 | raise 24 | 25 | 26 | logger = LogManager('test_pysnoop').get_logger_and_add_handlers(log_filename='test_pysnoop.log') 27 | 28 | 29 | @pysnooper_ydf.snoop(depth=300) 30 | def main(): 31 | try: 32 | # logger.info('测试pysnoop') 33 | for i in range(5): 34 | print(i) 35 | j = 333 36 | resp = requests.get('https://www.baidu.com') # 测试深层次跳转下的代码轨迹自动跟踪效果。 37 | logger.debug(resp.text) 38 | print(RedisMixin().redis_db_frame.set('key_test', '1')) 39 | bar() 40 | 41 | except: 42 | pass 43 | 44 | 45 | expected_output = ''' 46 | Source path:... Whatever 47 | 12:18:08.017782 call 17 def main(): 48 | 12:18:08.018142 line 18 try: 49 | 12:18:08.018181 line 19 bar() 50 | 12:18:08.018223 call 8 def bar(): 51 | 12:18:08.018260 line 9 try: 52 | 12:18:08.018293 line 10 foo() 53 | 12:18:08.018329 call 4 def foo(): 54 | 12:18:08.018364 line 5 raise TypeError('bad') 55 | 12:18:08.018396 exception 5 raise TypeError('bad') 56 | TypeError: bad 57 | Call ended by exception 58 | 12:18:08.018494 exception 10 foo() 59 | TypeError: bad 60 | 12:26:33.942623 line 11 except Exception: 61 | 12:26:33.942674 line 12 str(1) 62 | 12:18:08.018655 line 13 raise 63 | Call ended by exception 64 | 12:18:08.018718 exception 19 bar() 65 | TypeError: bad 66 | 12:18:08.018761 line 20 except: 67 | 12:18:08.018787 line 21 pass 68 | 12:18:08.018813 return 21 pass 69 | Return value:.. None 70 | ''' 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_redis_stream.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework.utils.redis_manager import RedisMixin 2 | 3 | print(RedisMixin().redis_db_frame_version3.xinfo_groups('queue_test_f01')) 4 | 5 | print(RedisMixin().redis_db_frame_version3.xinfo_stream('queue_test_f01')) 6 | 7 | print(RedisMixin().redis_db_frame_version3.xinfo_consumers('queue_test_f01', 'distributed_frame_group')) 8 | 9 | print(RedisMixin().redis_db_frame_version3.xpending('queue_test_f01', 'distributed_frame_group')) 10 | 11 | 12 | RedisMixin().redis_db_frame_version3 -------------------------------------------------------------------------------- /test_frame/other_tests/test_zeromq/test_zeromq_broker.py: -------------------------------------------------------------------------------- 1 | import zmq,time 2 | 3 | # Prepare our context and sockets 4 | context = zmq.Context() 5 | frontend = context.socket(zmq.ROUTER) 6 | backend = context.socket(zmq.DEALER) 7 | frontend.bind("tcp://*:5559") 8 | backend.bind("tcp://*:5560") 9 | 10 | # Initialize poll set 11 | poller = zmq.Poller() 12 | poller.register(frontend, zmq.POLLIN) 13 | poller.register(backend, zmq.POLLIN) 14 | 15 | # Switch messages between sockets 16 | while True: 17 | socks = dict(poller.poll()) #轮询器 循环接收 18 | 19 | if socks.get(frontend) == zmq.POLLIN: 20 | message = frontend.recv_multipart() 21 | backend.send_multipart(message) 22 | 23 | if socks.get(backend) == zmq.POLLIN: 24 | message = backend.recv_multipart() 25 | frontend.send_multipart(message) 26 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_zeromq/test_zeromq_client.py: -------------------------------------------------------------------------------- 1 | import zmq,time 2 | 3 | # Prepare our context and sockets 4 | context = zmq.Context() 5 | socket = context.socket(zmq.REQ) 6 | socket.connect("tcp://localhost:5559") 7 | 8 | # Do 10 requests, waiting each time for a response 9 | for request in range(1,11): 10 | time.sleep(2) 11 | socket.send(f"Hello {request}".encode()) 12 | message = socket.recv() 13 | print("Received reply %s [%s]" % (request, message)) 14 | 15 | -------------------------------------------------------------------------------- /test_frame/other_tests/test_zeromq/test_zeromq_server.py: -------------------------------------------------------------------------------- 1 | import zmq 2 | 3 | context = zmq.Context() 4 | socket = context.socket(zmq.REP) 5 | socket.connect("tcp://localhost:5560") 6 | 7 | while True: 8 | message = socket.recv() 9 | print("Received request: %s" % message) 10 | socket.send(b"World") 11 | -------------------------------------------------------------------------------- /test_frame/other_tests/tests-redis_performance.py: -------------------------------------------------------------------------------- 1 | """ 2 | 测试redis无其他逻辑时候的性能 3 | """ 4 | from redis3 import Redis 5 | import decorator_libs 6 | import nb_log 7 | 8 | r = Redis(decode_responses=True) 9 | 10 | # with decorator_libs.TimerContextManager(): 11 | # for i in range(1000000): 12 | # print(i) 13 | # r.lpush('test_performance',i) 14 | 15 | 16 | # while 1: 17 | # result = r.brpop('test_performance') 18 | # print(result) 19 | 20 | # for i in range(100000): 21 | # print(i) 22 | # r.xadd('testp2',{"":i}) 23 | 24 | 25 | -------------------------------------------------------------------------------- /test_frame/test_apschedual/README.md: -------------------------------------------------------------------------------- 1 | # 演示定时的使用 -------------------------------------------------------------------------------- /test_frame/test_apschedual/test_timer.py: -------------------------------------------------------------------------------- 1 | # 定时运行消费演示之1，最好是看另外一个演示版本test_timer2.py 2 | import datetime 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, fsdf_background_scheduler 4 | 5 | 6 | @task_deco('queue_test_666', broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE) 7 | def consume_func(x, y): 8 | print(f'{x} + {y} = {x + y}') 9 | 10 | 11 | # 写一个推送的函数 12 | def pubilsh_task(): 13 | consume_func.push(1, 2) 14 | 15 | 16 | if __name__ == '__main__': 17 | # aps_background_scheduler.add_job(pubilsh_task, 'interval', id='3_second_job', seconds=3) # 每隔3秒发布一次任务，自然就能每隔3秒消费一次任务了。 18 | fsdf_background_scheduler.add_job(pubilsh_task, 'date', run_date=datetime.datetime(2020, 7, 24, 13, 53, 6)) # 定时，只执行一次 19 | fsdf_background_scheduler.add_job(pubilsh_task, 'cron', day_of_week='*', hour=13, minute=53, second=20) # 定时，每天的11点32分20秒都执行一次。 20 | # 启动定时 21 | fsdf_background_scheduler.start() 22 | 23 | # 启动消费 24 | consume_func.consume() -------------------------------------------------------------------------------- /test_frame/test_apschedual/test_timer2.py: -------------------------------------------------------------------------------- 1 | # 定时运行消费演示 2 | import datetime 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, fsdf_background_scheduler, timing_publish_deco 4 | 5 | 6 | @task_deco('queue_test_666', broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE) 7 | def consume_func(x, y): 8 | print(f'{x} + {y} = {x + y}') 9 | 10 | 11 | if __name__ == '__main__': 12 | fsdf_background_scheduler.add_job(timing_publish_deco(consume_func), 'interval', id='3_second_job', seconds=3, kwargs={"x": 5, "y": 6}) # 每隔3秒发布一次任务，自然就能每隔3秒消费一次任务了。 13 | fsdf_background_scheduler.add_job(timing_publish_deco(consume_func), 'date', run_date=datetime.datetime(2020, 7, 24, 13, 53, 6), args=(5, 6,)) # 定时，只执行一次 14 | fsdf_background_scheduler.add_timing_publish_job(consume_func, 'cron', day_of_week='*', hour=14, minute=51, second=20, args=(5, 6,)) # 定时，每天的11点32分20秒都执行一次。 15 | # 启动定时 16 | fsdf_background_scheduler.start() 17 | 18 | # 启动消费 19 | consume_func.consume() 20 | -------------------------------------------------------------------------------- /test_frame/test_async_consumer/readme_async_test.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum, ExceptionForRequeue 2 | import asyncio 3 | import random 4 | 5 | @task_deco('test_async_queue2', concurrent_mode=ConcurrentModeEnum.ASYNC,qps=3, 6 | broker_kind=BrokerEnum.REDIS_ACK_ABLE, log_level=10, concurrent_num=500, is_using_rpc_mode=True,do_task_filtering=0) 7 | async def async_f(x): 8 | print(id(asyncio.get_event_loop())) # 可以看到每个并发用的是同一个loop 9 | await asyncio.sleep(1, ) 10 | if x % 10 == 0 and random.randint(1,10) < 5: 11 | raise ValueError(x) 12 | # raise ExceptionForRequeue(x) 13 | print(x) 14 | return x % 10 15 | 16 | 17 | if __name__ == '__main__': 18 | async_f.clear() 19 | for i in range(40): 20 | async_f.push(i, ) 21 | async_f.consume() 22 | -------------------------------------------------------------------------------- /test_frame/test_async_consumer/test_async_consume.py: -------------------------------------------------------------------------------- 1 | import time 2 | import asyncio 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum 4 | 5 | 6 | @task_deco('test_async_queue', concurrent_mode=ConcurrentModeEnum.ASYNC, qps=3, broker_kind=BrokerEnum.PERSISTQUEUE, concurrent_num=60) 7 | async def async_f(x): # 调度异步消费函数 8 | print(x) 9 | # time.sleep(2) # 不能搞同步time.sleep 2秒的代码，否则不管设置多大并发实际qps最大只能达到0.5 10 | await asyncio.sleep(2) 11 | return x * 3 12 | 13 | 14 | @task_deco('test_f_queue', concurrent_mode=ConcurrentModeEnum.THREADING, qps=5, broker_kind=BrokerEnum.PERSISTQUEUE) 15 | def f(y): # 调度同步消费函数 16 | print(y) 17 | time.sleep(7) 18 | 19 | 20 | if __name__ == '__main__': 21 | async_f.clear() 22 | f.clear() 23 | 24 | for i in range(2000): 25 | async_f.push(i) 26 | f.push(i * 10) 27 | 28 | async_f.consume() 29 | f.consume() 30 | -------------------------------------------------------------------------------- /test_frame/test_async_consumer/test_async_consume2.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | 这个脚本可以得出协程asyncio + aiohttp 比 threading + requests 多线程快70% 4 | """ 5 | from function_scheduling_distributed_framework import task_deco, BrokerEnum,ConcurrentModeEnum 6 | import asyncio 7 | import time 8 | import aiohttp 9 | import requests 10 | 11 | url = 'http://mini.eastday.com/assets/v1/js/search_word.js' 12 | # url = 'https://www.baidu.com/content-search.xml' 13 | # rl = 'https://www.google-analytics.com/analytics.js' 14 | 15 | loop = asyncio.new_event_loop() 16 | ss = aiohttp.ClientSession(loop=loop, ) 17 | 18 | 19 | @task_deco('test_async_queue2', concurrent_mode=ConcurrentModeEnum.ASYNC, broker_kind=BrokerEnum.REDIS, log_level=10, 20 | concurrent_num=500,specify_async_loop=loop) 21 | async def async_f(x): 22 | # 如果使使用了同一个session，async with ss.request，必须指定specify_async_loop的值和ClientSession的loop相同。 23 | # 否则 如果使使用 async with aiohttp.request ，则无需指定specify_async_loop参数。 24 | # async with aiohttp.request('get', url=url) as resp: 25 | # print(x,55555555555) 26 | # await asyncio.sleep(1,) 27 | # print(x,66666666666) 28 | async with ss.request('get', url=url) as resp: 29 | text = await resp.text() 30 | # print(x, resp.url, text[:10]) 31 | print(x) 32 | return x 33 | 34 | 35 | rss = requests.session() 36 | 37 | 38 | @task_deco('test_f_queue2', concurrent_mode=ConcurrentModeEnum.THREADING, broker_kind=BrokerEnum.REDIS, concurrent_num=500, log_level=10) 39 | def f(y): 40 | # resp = requests.request('get', url) 41 | resp = rss.request('get', url) 42 | print(y, resp.text[:10]) 43 | 44 | 45 | if __name__ == '__main__': 46 | 47 | async_f.clear() 48 | f.clear() 49 | 50 | for i in range(10000): 51 | async_f.push(i) 52 | f.push(i * 10) 53 | 54 | # async_f.consume() 55 | f.consume() 56 | -------------------------------------------------------------------------------- /test_frame/test_async_consumer/test_async_rpc.py: -------------------------------------------------------------------------------- 1 | 2 | from test_frame.test_async_consumer.readme_async_test import async_f 3 | 4 | 5 | async_result = async_f.push(64,) 6 | print(async_result.result) -------------------------------------------------------------------------------- /test_frame/test_async_consumer/演示指定async_loop.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import aiohttp 4 | import aiomysql 5 | 6 | from function_scheduling_distributed_framework import task_deco,ConcurrentModeEnum,BrokerEnum 7 | 8 | aiomysql.connect() 9 | 10 | ''' 11 | 演示 specify_async_loop 用法，根据自己代码需要按情况使用specify_async_loop 12 | 这里举得是aiohttp例子，例如还有aiomysql例子，aioredis例子 13 | ''' 14 | 15 | 16 | loop = asyncio.get_event_loop() 17 | session = aiohttp.ClientSession(loop=loop) 18 | 19 | # 如果使用全局变量session，必须指定loop，并且这个loop同时传给ClientSession类和task_deco装饰器，否则报错。 20 | @task_deco('test_specify_loop',concurrent_mode=ConcurrentModeEnum.ASYNC,specify_async_loop=loop) 21 | async def f(url): 22 | async with session.get(url) as response: 23 | print("Status:", response.status) 24 | print("Content-type:", response.headers['content-type']) 25 | 26 | # 如果没使用全局变量session，task_deco装饰器无需指定specify_async_loop入参 27 | @task_deco('test_specify_loop',concurrent_mode=ConcurrentModeEnum.ASYNC) 28 | async def f2(url): 29 | async with aiohttp.ClientSession() as session: 30 | async with session.get(url) as response: 31 | print("Status:", response.status) 32 | print("Content-type:", response.headers['content-type']) 33 | 34 | if __name__ == '__main__': 35 | for p in range(1,100): 36 | urlx = f'https://www.cnblogs.com/#p{p}' 37 | f.push(urlx) 38 | f2.push(urlx) 39 | f.consume() 40 | f2.consume() -------------------------------------------------------------------------------- /test_frame/test_async_consumer/演示错误调用async函数的方式.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | 这是一个错误的调用 async 协程的方式，协程成了废物。包括celery，如果为了调用async def的函数，只要这么玩，异步就成了废物。 4 | """ 5 | import asyncio 6 | from function_scheduling_distributed_framework import task_deco,BrokerEnum,ConcurrentModeEnum 7 | 8 | async def f(x,): 9 | print(id(asyncio.get_event_loop())) # 从这里可以看到，打印出来的loop每次都是不同的，不是同一个循环 10 | await asyncio.sleep(4) 11 | return x + 5 12 | 13 | ##为了兼容调用asyncio， 需要多加一个函数， 真的多此一举。 正确方法是直接把装饰器加在上面那个async def f上面，然后设置concurrent_mode=ConcurrentModeEnum.ASYNC 14 | @task_deco('test_asyncio_error_queue',broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE,) 15 | def my_task(x): 16 | print(asyncio.new_event_loop().run_until_complete(f(x,))) 17 | 18 | 19 | if __name__ == '__main__': 20 | for i in range(5): 21 | my_task.push(i) 22 | my_task.consume() -------------------------------------------------------------------------------- /test_frame/test_auto_run_on_remote/show.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def show(x): 5 | print(f'显示python解释器： {sys.executable} 文件： {__file__} x: {x}') 6 | -------------------------------------------------------------------------------- /test_frame/test_auto_run_on_remote/test_run.py: -------------------------------------------------------------------------------- 1 | import os 2 | from auto_run_on_remote import run_current_script_on_remote 3 | from test_frame.test_auto_run_on_remote.show import show 4 | 5 | print(os.name) 6 | run_current_script_on_remote() 7 | for i in range(10): 8 | show(i) -------------------------------------------------------------------------------- /test_frame/test_broker/celery_sqlite3.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_broker/celery_sqlite3.sqlite -------------------------------------------------------------------------------- /test_frame/test_broker/celery_sqlite3x.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_broker/celery_sqlite3x.sqlite -------------------------------------------------------------------------------- /test_frame/test_broker/dssf_kombu_sqlite2.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_broker/dssf_kombu_sqlite2.sqlite -------------------------------------------------------------------------------- /test_frame/test_broker/test_consume.py: -------------------------------------------------------------------------------- 1 | # from auto_run_on_remote import run_current_script_on_remote 2 | # run_current_script_on_remote() 3 | import json 4 | import os 5 | import time 6 | import random 7 | # from distributed_frame_config import REDIS_HOST 8 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum,FunctionResultStatusPersistanceConfig 9 | from function_scheduling_distributed_framework.utils import RedisMixin 10 | 11 | 12 | # @task_deco('test_queue66', broker_kind=BrokerEnum.RABBITMQ_AMQPSTORM, qps=5, log_level=10, is_print_detail_exception=False, is_show_message_get_from_broker=False, 13 | # is_using_distributed_frequency_control=True) 14 | @task_deco('test_queue70c',broker_kind=BrokerEnum.REDIS,log_level=20,concurrent_mode=ConcurrentModeEnum.SINGLE_THREAD) 15 | def f(x,y): 16 | return x+y 17 | 18 | 19 | 20 | if __name__ == '__main__': 21 | # pass 22 | # f.clear() 23 | # for i in range(1000): 24 | # f.push(i, i * 2) 25 | 26 | f.consume() 27 | -------------------------------------------------------------------------------- /test_frame/test_broker/test_publish.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from test_frame.test_broker.test_consume import f 4 | 5 | f.clear() 6 | for i in range(1,1000000): 7 | # time.sleep(0.2) 8 | if i == 0: 9 | print(time.strftime("%H:%M:%S"), '发布第一条') 10 | if i % 10000 == 0: 11 | print(time.strftime("%H:%M:%S"), f'发布第{i}条') 12 | f.push(i, i * 2) 13 | 14 | if __name__ == '__main__': 15 | pass 16 | # f.multi_process_pub_params_list([{'a':i,'b':2*i} for i in range(100000)],process_num=5) -------------------------------------------------------------------------------- /test_frame/test_broker/testtt.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | asyncio.run() -------------------------------------------------------------------------------- /test_frame/test_broker_kafka/kafka_cosumer_test.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, FunctionResultStatusPersistanceConfig 4 | 5 | 6 | @task_deco('test_kafka12', broker_kind=BrokerEnum.KAFKA_CONFLUENT, qps=3, 7 | function_result_status_persistance_conf=FunctionResultStatusPersistanceConfig(True, True, 3600, is_use_bulk_insert=False)) 8 | def f(x): 9 | print(f'开始 {x}') 10 | time.sleep(random.randint(1, 50)) 11 | print(f'结束 {x}') 12 | return x*10 13 | 14 | 15 | if __name__ == '__main__': 16 | # for i in range(200): 17 | # f.push(i) 18 | # f.publisher.get_message_count() 19 | f.consume() 20 | -------------------------------------------------------------------------------- /test_frame/test_broker_kafka/kafka_publisher_test.py: -------------------------------------------------------------------------------- 1 | from kafka_cosumer_test import f 2 | 3 | # i = 203 4 | # f.push(i) 5 | 6 | 7 | for i in range(200): 8 | f.push(i) 9 | -------------------------------------------------------------------------------- /test_frame/test_celery/README.md: -------------------------------------------------------------------------------- 1 | celery测试。 2 | 主要是和此框架调度同一个函数。使用同一种redis内网中间件、运行在同一个机器、都使用gevent模式运行、都不要消费结果回调、相同的并发数量所有硬件和软件环境保持一致的情况下，测试两个框架的推动和消费性能的区别。 -------------------------------------------------------------------------------- /test_frame/test_celery/celery_start.bat: -------------------------------------------------------------------------------- 1 | :: 也可以用这个启动，没必要，直接启动 python test_test_celery_app.py 香多了 2 | celery -A test_celery_app worker --pool=threads --concurrency=50 -n worker1@%h --loglevel=DEBUG --queues=queue_f1,queue_add2,queue_sub2 -------------------------------------------------------------------------------- /test_frame/test_celery/celerybeat-schedule.bak: -------------------------------------------------------------------------------- 1 | 'entries', (0, 421) 2 | '__version__', (512, 15) 3 | 'tz', (1024, 4) 4 | 'utc_enabled', (1536, 4) 5 | -------------------------------------------------------------------------------- /test_frame/test_celery/celerybeat-schedule.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_celery/celerybeat-schedule.dat -------------------------------------------------------------------------------- /test_frame/test_celery/celerybeat-schedule.dir: -------------------------------------------------------------------------------- 1 | 'entries', (0, 421) 2 | '__version__', (512, 15) 3 | 'tz', (1024, 4) 4 | 'utc_enabled', (1536, 4) 5 | -------------------------------------------------------------------------------- /test_frame/test_celery/celerybeat.pid: -------------------------------------------------------------------------------- 1 | 93776 2 | -------------------------------------------------------------------------------- /test_frame/test_celery/test_add_task.py: -------------------------------------------------------------------------------- 1 | import time 2 | import nb_log 3 | 4 | from test_frame.test_celery.test_celery_app import add, sub 5 | 6 | 7 | 8 | t1 = time.time() 9 | for i in range(1,20): 10 | # print('生产者添加任务') 11 | # print(i) 12 | # result = add.delay(i, i * 2) 13 | # time.sleep(0.01) 14 | result = add.apply_async(args=(i, i * 2),countdown=0) 15 | # result = add.apply_async(args=(i, i * 2), ) 16 | print(result) 17 | # print(result.get()) 18 | # print(type(result)) 19 | # sub.delay(i * 10, i * 20) 20 | 21 | 22 | print(time.time() - t1) 23 | print('任务添加完成') 24 | 25 | """ 26 | import celery 27 | celery.result.AsyncResult 28 | """ 29 | -------------------------------------------------------------------------------- /test_frame/test_decorator_run_example/README.md: -------------------------------------------------------------------------------- 1 | ## 使用装饰器方式运行的例子 2 | 3 | test_decorator_task_example.py 4 | 5 | 6 | 7 | 8 | ## 和对比常规运行方式 9 | 10 | test_common_no_decorator_example.py -------------------------------------------------------------------------------- /test_frame/test_decorator_run_example/test_common_no_decorator_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | 测试非装饰器版本方式，注意对比装饰器版本test_decorator_task_example.py 3 | """ 4 | 5 | from function_scheduling_distributed_framework import get_consumer 6 | 7 | 8 | def f(a, b): 9 | print(a + b) 10 | 11 | 12 | consumer = get_consumer('queue_test_f01', consuming_function=f, qps=0.2, broker_kind=0) 13 | 14 | for i in range(10, 20): 15 | consumer.publisher_of_same_queue.publish(dict(a=i, b=i * 2)) 16 | 17 | consumer.start_consuming_message() 18 | -------------------------------------------------------------------------------- /test_frame/test_decorator_run_example/test_decorator_task_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | 测试装饰器版本方式,注意对比非装饰器版本 test_common_no_decorator_example.py 3 | """ 4 | import time 5 | 6 | from function_scheduling_distributed_framework import task_deco, IdeAutoCompleteHelper, BrokerEnum 7 | 8 | 9 | @task_deco('queue_test_f01', broker_kind=BrokerEnum.REDIS_STREAM,log_level=10) 10 | def f(a, b): 11 | print(f'{a} + {b} = {a + b}') 12 | 13 | 14 | if __name__ == '__main__': 15 | # f(1000, 2000) 16 | # f.clear() 17 | for i in range(0, 1): 18 | # f.pub(dict(a=i, b=i * 2)) 19 | # f.push(i * 10, i * 20, ) 20 | f.delay(i , b=i * 2,) 21 | # # IdeAutoCompleteHelper(f).pub({'a': i * 1000, 'b': i * 2000}) # 和上面等效，但可以自动补全方法名字和入参。 22 | 23 | IdeAutoCompleteHelper(f).start_consuming_message() # f.consume() 等效 24 | -------------------------------------------------------------------------------- /test_frame/test_delay_task/test_delay_consume.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import task_deco,BrokerEnum,ConcurrentModeEnum 2 | import time 3 | 4 | @task_deco('test_delay',broker_kind=BrokerEnum.REDIS_ACK_ABLE,qps=0.5,concurrent_mode=ConcurrentModeEnum.SINGLE_THREAD) 5 | def f(x): 6 | print(x) 7 | 8 | if __name__ == '__main__': 9 | f.consume() -------------------------------------------------------------------------------- /test_frame/test_delay_task/test_delay_push.py: -------------------------------------------------------------------------------- 1 | # 需要用publish，而不是push，这个前面已经说明了，如果要传函数入参本身以外的参数到中间件，需要用publish。 2 | # 不然框架分不清哪些是函数入参，哪些是控制参数。如果无法理解就，就好好想想琢磨下celery的 apply_async 和 delay的关系。 3 | from test_frame.test_delay_task.test_delay_consume import f 4 | import datetime 5 | import time 6 | from function_scheduling_distributed_framework import PriorityConsumingControlConfig 7 | 8 | """ 9 | 测试发布延时任务，不是发布后马上就执行函数。 10 | 11 | countdown 和 eta 只能设置一个。 12 | countdown 指的是 离发布多少秒后执行， 13 | eta是指定的精确时间运行一次。 14 | 15 | misfire_grace_time 是指定消息轮到被消费时候，如果已经超过了应该运行的时间多少秒之内，仍然执行。 16 | misfire_grace_time 如果设置为None，则消息一定会被运行，不会由于大连消息积压导致消费时候已近太晚了而取消运行。 17 | misfire_grace_time 如果不为None，必须是大于等于1的整数，此值表示消息轮到消费时候超过本应该运行的时间的多少秒内仍然执行。 18 | 此值的数字设置越小，如果由于消费慢的原因，就有越大概率导致消息被丢弃不运行。如果此值设置为1亿，则几乎不会导致放弃运行(1亿的作用接近于None了) 19 | 如果还是不懂这个值的作用，可以百度 apscheduler 包的 misfire_grace_time 概念 20 | 21 | """ 22 | for i in range(1, 20): 23 | time.sleep(1) 24 | 25 | # 消息发布10秒后再执行。如果消费慢导致任务积压，misfire_grace_time为None，即使轮到消息消费时候离发布超过10秒了仍然执行。 26 | f.publish({'x': i}, priority_control_config=PriorityConsumingControlConfig(countdown=10)) 27 | 28 | # 规定消息在17点56分30秒运行，如果消费慢导致任务积压，misfire_grace_time为None，即使轮到消息消费时候已经过了17点56分30秒仍然执行。 29 | f.publish({'x': i * 10}, priority_control_config=PriorityConsumingControlConfig( 30 | eta=datetime.datetime(2021, 5, 19, 17, 56, 30) + datetime.timedelta(seconds=i))) 31 | 32 | # 消息发布10秒后再执行。如果消费慢导致任务积压，misfire_grace_time为30，如果轮到消息消费时候离发布超过40 (10+30) 秒了则放弃执行， 33 | # 如果轮到消息消费时候离发布时间是20秒，由于 20 < (10 + 30)，则仍然执行 34 | f.publish({'x': i * 100}, priority_control_config=PriorityConsumingControlConfig( 35 | countdown=10, misfire_grace_time=30)) 36 | 37 | # 规定消息在17点56分30秒运行，如果消费慢导致任务积压，如果轮到消息消费时候已经过了17点57分00秒， 38 | # misfire_grace_time为30，如果轮到消息消费时候超过了17点57分0秒 则放弃执行， 39 | # 如果如果轮到消息消费时候是17点56分50秒则执行。 40 | f.publish({'x': i * 1000}, priority_control_config=PriorityConsumingControlConfig( 41 | eta=datetime.datetime(2021, 5, 19, 17, 56, 30) + datetime.timedelta(seconds=i), 42 | misfire_grace_time=30)) # 按指定的时间运行一次。 43 | 44 | # 这个设置了消息由于消息堆积导致运行的时候比本应该运行的时间如果小于1亿秒，就仍然会被执行，所以几乎肯定不会被放弃运行 45 | f.publish({'x': i * 10000}, priority_control_config=PriorityConsumingControlConfig( 46 | eta=datetime.datetime(2021, 5, 19, 17, 56, 30) + datetime.timedelta(seconds=i), 47 | misfire_grace_time=100000000)) # 按指定的时间运行一次。 48 | -------------------------------------------------------------------------------- /test_frame/test_fabric_deploy/test_deploy1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | import socket 6 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum, fabric_deploy 7 | 8 | import os 9 | 10 | 11 | def get_host_ip(): 12 | ip = '' 13 | host_name = '' 14 | # noinspection PyBroadException 15 | try: 16 | sc = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 17 | sc.connect(('8.8.8.8', 80)) 18 | ip = sc.getsockname()[0] 19 | host_name = socket.gethostname() 20 | sc.close() 21 | except Exception: 22 | pass 23 | return ip, host_name 24 | 25 | 26 | computer_ip, computer_name = get_host_ip() 27 | 28 | 29 | # 通过设置broker_kind，一键切换中间件为mq或redis等20种中间件或包。 30 | # 通过设置concurrent_mode，来设置并发方式，改这一个参数能自动支持threading eventlet gevent asyncio并发 31 | # 通过设置qps，能够精确的指定每秒运行多少次函数，无视函数需要耗时多久。 32 | # 通过设置concurrent_num，设置并发大小。此例，设置300的线程池大小，但qps为6，函数需要耗时5秒钟， 33 | # 框架的智能线程池会自动只开30个线程，不会造成启动线程太多切换影响效率性能，所以可以自信大胆的设置线程数量。 34 | # 智能线程池能自动扩大也能自动缩小线程数量，例如某段时间函数耗时大，会增大线程数量来达到qps，如果函数耗时变小了，会自动缩小线程数量，框架不需要提前知道函数的确定耗时，会自动调节并发数量的。 35 | # 还有其他30种函数运行控制参数，看代码里面的函数入参说明，说的非常详细了。 36 | 37 | # @task_deco('queue_test2', ) # @task_deco必须参数只有一个。 38 | @task_deco('queue_test30', qps=0.2, broker_kind=BrokerEnum.REDIS) 39 | def f2(a, b): 40 | sleep_time = 7 41 | result = a + b 42 | # print(f'机器：{get_host_ip()} 进程：{os.getpid()}， 消费此消息 {a} + {b} 中。。。。。,此次需要消耗 {sleep_time} 秒') 43 | time.sleep(sleep_time) # 模拟做某事需要阻塞n秒种，必须用并发绕过此阻塞。 44 | print(f'机器：{get_host_ip()} 进程：{os.getpid()}，{a} + {b} 的结果是 {result}') 45 | return result 46 | 47 | 48 | @task_deco('queue_test31', qps=0.2, broker_kind=BrokerEnum.REDIS) 49 | def f3(a, b): 50 | print(f'机器：{get_host_ip()} 进程：{os.getpid()}，{a} - {b} 的结果是 {a - b}') 51 | return a - b 52 | 53 | 54 | if __name__ == '__main__': 55 | print(f2.__name__) 56 | f2.clear() 57 | for i in range(20000): 58 | f2.push(i, i * 2) 59 | f3.push(i, i * 2) 60 | f2.consume() 61 | f3.multi_process_consume(2) 62 | # # 192.168.114.135 192.168.6.133 63 | f2.fabric_deploy('192.168.6.133', 22, 'ydf', '372148', process_num=2) 64 | f3.fabric_deploy('106.44.xxx.110', 22, 'root', 'fdsfdsfdsad', 65 | only_upload_within_the_last_modify_time=1 * 24 * 60 * 60, 66 | file_volume_limit=100 * 1000, process_num=2) 67 | -------------------------------------------------------------------------------- /test_frame/test_frame_using_thread/README.md: -------------------------------------------------------------------------------- 1 | 测试框架的使用，可以通过修改broker_kind来测试不同种类的中间件。 2 | 3 | 即使没有安装任何中间件，也可以使用此框架的基于sqlite persist queue方式的基于本机的分布式（需要设置broker_kind为6）（比基于python 内置 Queue对象比，兼容了不在同一解释器启动的脚本共享消息队列）。 4 | 5 | 正式使用墙裂推荐安装rabbitmq中间件。 -------------------------------------------------------------------------------- /test_frame/test_frame_using_thread/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | # import gevent.monkey;gevent.monkey.patch_all() 5 | import time 6 | import random 7 | 8 | from function_scheduling_distributed_framework import get_consumer, AbstractConsumer 9 | from function_scheduling_distributed_framework.consumers.base_consumer import ConsumersManager, FunctionResultStatusPersistanceConfig 10 | from function_scheduling_distributed_framework.utils import LogManager 11 | 12 | 13 | 14 | logger = LogManager('test_consume').get_logger_and_add_handlers() 15 | 16 | 17 | class RandomError(Exception): 18 | pass 19 | 20 | 21 | def add(a, b): 22 | logger.info(f'消费此消息 {a} + {b} 中。。。。。') 23 | # time.sleep(random.randint(1, 3)) # 模拟做某事需要阻塞10秒种，必须用并发绕过此阻塞。 24 | if random.randint(4, 6) == 5: 25 | raise RandomError('演示随机出错') 26 | logger.info(f'计算 {a} + {b} 得到的结果是 {a + b}') 27 | return a + b 28 | 29 | 30 | def sub(x, y): 31 | logger.info(f'消费此消息 {x} - {y} 中。。。。。') 32 | time.sleep(4) # 模拟做某事需要阻塞10秒种，必须用并发绕过此阻塞。 33 | if random.randint(1, 10) == 4: 34 | raise ValueError('4444') 35 | result = x - y 36 | logger.info(f'计算 {x} - {y} 得到的结果是 {result}') 37 | return result 38 | 39 | 40 | # 把消费的函数名传给consuming_function，就这么简单。 41 | consumer_add = get_consumer('queue_test569', consuming_function=add, concurrent_num=50, max_retry_times=3, 42 | qps=2000, log_level=10, logger_prefix='zz平台消费', 43 | function_timeout=0, is_print_detail_exception=False, 44 | msg_expire_senconds=3600, 45 | is_using_rpc_mode=True, 46 | function_result_status_persistance_conf=FunctionResultStatusPersistanceConfig(True, False, 7 * 24 * 3600), 47 | broker_kind=9, concurrent_mode=1, ) # 通过设置broker_kind，一键切换中间件为rabbitmq或redis等9种中间件或包。 48 | 49 | consumer_sub = get_consumer('queue_test86', consuming_function=sub, concurrent_num=200, qps=108, log_level=10, logger_prefix='xxxxx平台消费', 50 | is_print_detail_exception=True, 51 | broker_kind=9, concurrent_mode=1) # 通过设置 52 | 53 | if __name__ == '__main__': 54 | ConsumersManager.show_all_consumer_info() 55 | consumer_add.start_consuming_message() 56 | # consumer_sub.start_consuming_message() 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /test_frame/test_frame_using_thread/test_publish.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | 5 | from test_frame.test_frame_using_thread.test_consume import consumer_add, consumer_sub 6 | from function_scheduling_distributed_framework.publishers.base_publisher import PriorityConsumingControlConfig 7 | 8 | pb_add = consumer_add.bulid_a_new_publisher_of_same_queue() 9 | pb_add.clear() 10 | 11 | pb_sub = consumer_sub.bulid_a_new_publisher_of_same_queue() 12 | pb_sub.clear() 13 | 14 | for i in range(1000000): 15 | # time.sleep(0.2) 16 | async_result = pb_add.publish(dict(a=i, b=2 * i)) 17 | # print(async_result.status_and_result) 18 | # print('结果', async_result.result) 19 | # async_result = consumer_sub.publisher_of_same_queue.publish({'x': i, 'y': i * 6}, 20 | # priority_control_config=PriorityConsumingControlConfig(is_using_rpc_mode=False)) 21 | # print('同步结果', async_result.result) 22 | 23 | """ 24 | mtfy 25 | export PYTHONPATH=./ 26 | python test_frame/test_frame_using_thread/test_publish.py 27 | """ 28 | 29 | -------------------------------------------------------------------------------- /test_frame/test_instead_thread_asyncio/test_instead_thread.py: -------------------------------------------------------------------------------- 1 | import time 2 | from function_scheduling_distributed_framework import task_deco, BrokerEnum 3 | 4 | 5 | @task_deco("test_insteda_thread_queue", broker_kind=BrokerEnum.MEMORY_QUEUE, concurrent_num=10) 6 | def f(x): 7 | time.sleep(3) 8 | print(x) 9 | 10 | 11 | for i in range(100): 12 | f.push(i) 13 | 14 | f.consume() 15 | 16 | -------------------------------------------------------------------------------- /test_frame/test_instead_thread_asyncio/test_use_thread.py: -------------------------------------------------------------------------------- 1 | import time 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | 5 | def f(x): 6 | time.sleep(3) 7 | print(x) 8 | 9 | 10 | pool = ThreadPoolExecutor(10) 11 | 12 | for i in range(100): 13 | pool.submit(f,i) -------------------------------------------------------------------------------- /test_frame/test_multiprocess_diffrent_linux_and_win/test_start_multiprocess.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process 2 | import threading 3 | 4 | 5 | def f(xx): 6 | print(xx) 7 | 8 | x = 2 9 | x = threading.Lock() 10 | x = (i for i in range(5)) 11 | 12 | 13 | 14 | 15 | if __name__ == '__main__': 16 | Process(target=f, args=(x,)).start() 17 | 18 | # Process(target=f, args=(x,)).start() -------------------------------------------------------------------------------- /test_frame/test_nb_log/log_example.py: -------------------------------------------------------------------------------- 1 | print('导入nb_log之前的print是普通的') 2 | 3 | from nb_log import get_logger 4 | 5 | print('导入nb_log之后的print是强化版的可点击跳转的') 6 | 7 | logger = get_logger('lalala', log_filename='lalala.log',is_add_elastic_handler=True) 8 | 9 | for i in range(3): 10 | logger.debug(f'debug是绿色，说明是调试的，代码ok。 ' * 4) 11 | logger.info('info是天蓝色，日志正常。 ' * 4) 12 | logger.warning('黄色yello，有警告了。 ' * 4) 13 | logger.error('粉红色说明代码有错误。 ' * 4) 14 | logger.critical('血红色，说明发生了严重错误。 ' * 4) 15 | print('只要导入nb_log一次之后的项目任意文件的print是强化版的可点击跳转的，在输出控制台点击行号能自动打开文件跳转到精确行号。') 16 | 17 | -------------------------------------------------------------------------------- /test_frame/test_qps/qps_demo_use_frame.py: -------------------------------------------------------------------------------- 1 |  2 | """ 3 | 这个代码是模拟常规并发手段无法达到每秒持续精确运行8次函数(请求flask接口8次)的目的。 4 | 但是使用分布式函数调度框架能轻松达到这个目的。 5 | 6 | 下面的代码使用分部署函数调度框架来调度运行 request_flask_api 函数， 7 | 8 | flask_veiw_mock 接口耗时0.1秒时候，控制台每秒打印8次 hello world，非常精确的符合控频目标 9 | flask_veiw_mock 接口耗时1秒时候，控制台每秒打印8次 hello world，非常精确的符合控频目标 10 | flask_veiw_mock 接口耗时10秒时候控，控制台每秒打印8次 hello world，非常精确的符合控频目标 11 | flask_veiw_mock 接口耗时0.001秒时候，控制台每秒打印8次 hello world，非常精确的符合控频目标 12 | flask_veiw_mock 接口耗时50 秒时候，控制台每秒打印8次 hello world，非常精确的符合控频目标 13 | 可以发现分布式函数调度框架无视函数耗时大小，都能做到精确控频，常规的线程池 asyncio什么的，面对这种不确定的接口耗时，简直毫无办法。 14 | 15 | 有些人到现在还没明白并发数量和qps(每秒执行多少次)之间的区别，并发数量只有在函数耗时刚好精确等于1秒时候才等于qps。 16 | """ 17 | import time 18 | from function_scheduling_distributed_framework import task_deco,BrokerEnum 19 | 20 | 21 | def flask_veiw_mock(x): 22 | time.sleep(0.1) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 23 | # time.sleep(1) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 24 | # time.sleep(10) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 25 | return f"hello world {x}" 26 | 27 | @task_deco("test_qps",broker_kind=BrokerEnum.MEMORY_QUEUE,qps=8) 28 | def request_flask_api(x): 29 | response = flask_veiw_mock(x) 30 | print(time.strftime("%H:%M:%S"), ' ', response) 31 | 32 | 33 | if __name__ == '__main__': 34 | for i in range(800): 35 | request_flask_api.push(i) 36 | request_flask_api.consume() 37 | -------------------------------------------------------------------------------- /test_frame/test_qps/qps_demo_use_threadpool.py: -------------------------------------------------------------------------------- 1 |  2 | """ 3 | 这个代码是模拟常规并发手段无法达到每秒持续精确运行8次函数(请求flask接口8次)的目的。 4 | 5 | 下面的代码使用8个线程并发运行 request_flask_api 函数， 6 | 当flask_veiw_mock 接口耗时0.1秒时候,在python输出控制台可以看到，10秒钟就运行结束了，控制台每秒打印了80次hello world，严重超频10倍了不符合需求 7 | 当flask_veiw_mock 接口耗时刚好精确等于1秒时候,在python输出控制台可以看到，100秒钟运行结束了，控制台每秒打印了8次hello world，只有当接口耗时刚好精确等于1秒时候，并发数量才符合qps需求 8 | 当flask_veiw_mock 接口耗时10秒时候,在python输出控制台可以看到，需要1000秒钟运行结束，控制台是每隔10秒才打印8次hello world，严重不符合持续每1秒都打印8次的目的。 9 | 由此可见，用设置并发数量来达到每秒请求8次flask的目的非常困难，99。99%的情况下服务端没那么巧刚好耗时1秒。 10 | 11 | 有些人到现在还没明白并发数量和qps(每秒执行多少次)之间的区别，并发数量只有在函数耗时刚好精确等于1秒时候才等于qps。 12 | """ 13 | import time 14 | from concurrent.futures import ThreadPoolExecutor 15 | 16 | 17 | def flask_veiw_mock(x): 18 | # time.sleep(0.1) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 19 | # time.sleep(1) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 20 | time.sleep(10) # 通过不同的sleep大小来模拟服务端响应需要消耗的时间 21 | return f"hello world {x}" 22 | 23 | 24 | def request_flask_api(x): 25 | response = flask_veiw_mock(x) 26 | print(time.strftime("%H:%M:%S"), ' ', response) 27 | 28 | 29 | if __name__ == '__main__': 30 | with ThreadPoolExecutor(8) as pool: 31 | for i in range(800): 32 | pool.submit(request_flask_api,i) 33 | -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/333.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_rabbitmq/333.png -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_rabbitmq/img.png -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_rabbitmq/img_1.png -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_rabbitmq/img_2.png -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/test_librabbitmq/test_librabbitmq_conusme.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from amqp import Message, Connection, ConnectionError, ChannelError 3 | import nb_log 4 | TEST_QUEUE = 'pyrabbit.testq' 5 | 6 | connection = Connection(host='localhost:5672', userid='guest', 7 | password='guest', virtual_host='/') 8 | channel = connection.channel() 9 | channel.queue_delete(TEST_QUEUE) 10 | 11 | channel.exchange_declare(TEST_QUEUE, 'direct') 12 | x = channel.queue_declare(TEST_QUEUE) 13 | # self.assertEqual(x.message_count, x[1]) 14 | # self.assertEqual(x.consumer_count, x[2]) 15 | # self.assertEqual(x.queue, TEST_QUEUE) 16 | channel.queue_bind(TEST_QUEUE, TEST_QUEUE, TEST_QUEUE) -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/test_rabbitmq_consume.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | from function_scheduling_distributed_framework import task_deco,BrokerEnum 4 | 5 | @task_deco('test_rabbit_queue7',broker_kind=BrokerEnum.RABBITMQ_AMQPSTORM,is_show_message_get_from_broker=True) 6 | def test_fun(x): 7 | # time.sleep(2.9) 8 | # sleep时间随机从0.1毫秒到5秒任意徘徊,最小耗时和最大耗时差距达到了5万倍。 9 | # 传统的恒定并发数量的线程池对未知的耗时任务且波动达到了5万倍，持续100次每秒的精确控频无能为力， 10 | # 但此框架只要简单设置一个qps就自动达到了这个目的。 11 | # random_sleep = random.randrange(1,50000) / 10000 12 | # time.sleep(random_sleep) 13 | # print(x,random_sleep) 14 | time.sleep(20000) 15 | print(x * 2) 16 | return x*2 17 | 18 | 19 | if __name__ == '__main__': 20 | test_fun.consume() -------------------------------------------------------------------------------- /test_frame/test_rabbitmq/test_rabbitmq_pubilish.py: -------------------------------------------------------------------------------- 1 | from test_frame.test_rabbitmq.test_rabbitmq_consume import test_fun 2 | 3 | test_fun.clear() 4 | for i in range(20): 5 | test_fun.push(i) 6 | 7 | -------------------------------------------------------------------------------- /test_frame/test_redis_ack_able/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_redis_ack_able/__init__.py -------------------------------------------------------------------------------- /test_frame/test_redis_ack_able/test_redis_ack_able_consumer.py: -------------------------------------------------------------------------------- 1 | """ 2 | 这个是用来测试，以redis为中间件，随意关闭代码会不会造成任务丢失的。 3 | """ 4 | import time 5 | 6 | from function_scheduling_distributed_framework import task_deco,BrokerEnum 7 | 8 | @task_deco('test_cost_long_time_fun_queue2',broker_kind=BrokerEnum.REDIS_ACK_ABLE,concurrent_num=5) 9 | def cost_long_time_fun(x): 10 | print(f'正在消费 {x} 中 。。。。') 11 | time.sleep(3) 12 | print(f'消费完成 {x} ') 13 | 14 | if __name__ == '__main__': 15 | cost_long_time_fun.consume() -------------------------------------------------------------------------------- /test_frame/test_redis_ack_able/test_redis_ack_able_publisher.py: -------------------------------------------------------------------------------- 1 | 2 | from test_frame.test_redis_ack_able.test_redis_ack_able_consumer import cost_long_time_fun 3 | 4 | for i in range(500): 5 | cost_long_time_fun.push(i) 6 | -------------------------------------------------------------------------------- /test_frame/test_redis_conn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 13:53 4 | 5 | 6 | 7 | 8 | 9 | from function_scheduling_distributed_framework.utils import RedisMixin 10 | 11 | 12 | 13 | print(RedisMixin().redis_db_frame.keys()) 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /test_frame/test_request_baidu/baidu_consumer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | 6 | from function_scheduling_distributed_framework import task_deco, BrokerEnum 7 | from nb_http_client import HttpOperator, ObjectPool 8 | 9 | http_pool = ObjectPool(object_type=HttpOperator, object_pool_size=100, object_init_kwargs=dict(host='mini.eastday.com', port=80), 10 | max_idle_seconds=30) 11 | 12 | 13 | @task_deco('test_baidu', broker_kind=BrokerEnum.REDIS, log_level=20,is_print_detail_exception=False,concurrent_num=200) 14 | def request_url(url): 15 | with http_pool.get() as conn: 16 | r1 = conn.request_and_getresponse('GET', url) 17 | # print(r1.text[:10], ) 18 | 19 | 20 | if __name__ == '__main__': 21 | request_url.clear() 22 | for i in range(100000): 23 | request_url.push('http://mini.eastday.com/assets/v1/js/search_word.js') 24 | 25 | request_url.multi_process_consume(2) 26 | -------------------------------------------------------------------------------- /test_frame/test_request_baidu/multi_aio_server.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import socket 4 | import time 5 | from aiohttp import web 6 | 7 | 8 | def mk_socket(host="127.0.0.1", port=8000, reuseport=False): 9 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 10 | if reuseport: 11 | SO_REUSEPORT = 15 12 | sock.setsockopt(socket.SOL_SOCKET, SO_REUSEPORT, 1) 13 | sock.bind((host, port)) 14 | return sock 15 | 16 | async def handle(request): 17 | name = request.match_info.get('name', "Anonymous") 18 | pid = os.getpid() 19 | text = "{:.2f}: Hello {}! Process {} is treating you\n".format( 20 | time.time(), name, pid) 21 | time.sleep(0.5) # intentionally blocking sleep to simulate CPU load 22 | return web.Response(text=text) 23 | 24 | if __name__ == '__main__': 25 | host = "127.0.0.1" 26 | port=8000 27 | reuseport = True 28 | app = web.Application() 29 | sock = mk_socket(host, port, reuseport=reuseport) 30 | app.add_routes([web.get('/', handle), 31 | web.get('/{name}', handle)]) 32 | loop = asyncio.get_event_loop() 33 | coro = loop.create_server( 34 | protocol_factory=app.make_handler(), 35 | sock=sock, 36 | ) 37 | srv = loop.run_until_complete(coro) 38 | loop.run_forever() -------------------------------------------------------------------------------- /test_frame/test_rpc/README.md: -------------------------------------------------------------------------------- 1 | 演示框架的使用，演示rpc功能。 2 | 3 | rpc功能在跨进程 跨python解释器 跨服务器 跨不同语言之间很有用。 4 | 5 | 只要不是直接面对前端的接口，都不是必须的。可以改成rpc，减少接口代码和增强性能.。 -------------------------------------------------------------------------------- /test_frame/test_rpc/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | import time 5 | 6 | from function_scheduling_distributed_framework import task_deco, BrokerEnum 7 | 8 | 9 | @task_deco('test_rpc_queue', is_using_rpc_mode=True, broker_kind=BrokerEnum.REDIS_ACK_ABLE,qps=100) 10 | def add(a, b): 11 | time.sleep(2) 12 | return a / b 13 | 14 | 15 | if __name__ == '__main__': 16 | add.consume() 17 | 18 | -------------------------------------------------------------------------------- /test_frame/test_rpc/test_publish.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | from function_scheduling_distributed_framework import PriorityConsumingControlConfig 5 | from test_frame.test_rpc.test_consume import add 6 | 7 | 8 | def show_result(status_and_result): 9 | print(status_and_result) 10 | 11 | add.clear() 12 | for i in range(1,50): 13 | # async_result = add.push(i, i * 2) 14 | # print(async_result.result) 15 | async_result = add.publish(dict(a=i*10, b=i * 20),priority_control_config = PriorityConsumingControlConfig(is_using_rpc_mode=True)) 16 | async_result.set_callback(show_result) 17 | # print(async_result.status_and_result) 18 | -------------------------------------------------------------------------------- /test_frame/test_socket/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_socket/__init__.py -------------------------------------------------------------------------------- /test_frame/test_socket/test_socket_consumer.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, ConcurrentModeEnum 4 | 5 | 6 | @task_deco('10.0.126.147:5691', broker_kind=BrokerEnum.TCP, qps=0.5, is_print_detail_exception=True,max_retry_times=3) 7 | def f(x): 8 | # time.sleep(7) 9 | if x % 10 == 0 and random.random() < 0.2: 10 | raise ValueError('测试函数出错') 11 | print(x) 12 | return x * 10 13 | 14 | 15 | if __name__ == '__main__': 16 | f.consume() 17 | for i in range(10): 18 | f.push(i) 19 | -------------------------------------------------------------------------------- /test_frame/test_socket/test_socket_publisher.py: -------------------------------------------------------------------------------- 1 | import time 2 | from test_socket_consumer import f 3 | 4 | for i in range(1000000): 5 | time.sleep(1) 6 | # print(i) 7 | f.push(i) 8 | -------------------------------------------------------------------------------- /test_frame/test_socket/test_tcp_client.py: -------------------------------------------------------------------------------- 1 | import time 2 | from socket import * 3 | 4 | HOST = '127.0.0.1' # or 'localhost' 5 | PORT = 21567 6 | BUFSIZ = 1024 7 | ADDR = (HOST, PORT) 8 | 9 | tcpCliSock = socket(AF_INET, SOCK_STREAM) 10 | tcpCliSock.connect(ADDR) 11 | # while True: 12 | # data1 = input('>') 13 | # # data = str(data) 14 | # if not data1: 15 | # break 16 | # tcpCliSock.send(data1.encode()) 17 | # data1 = tcpCliSock.recv(BUFSIZ) 18 | # if not data1: 19 | # break 20 | # print(data1.decode('utf-8')) 21 | # tcpCliSock.close() 22 | 23 | data1 ='heloo' 24 | tcpCliSock.send(data1.encode()) 25 | data1 = tcpCliSock.recv(BUFSIZ) 26 | print(data1.decode('utf-8')) 27 | tcpCliSock.close() 28 | 29 | time.sleep(1000) -------------------------------------------------------------------------------- /test_frame/test_socket/test_tcp_server.py: -------------------------------------------------------------------------------- 1 | from socket import * 2 | from time import ctime 3 | 4 | HOST = '' 5 | PORT = 21567 6 | BUFSIZ = 1024 7 | ADDR = (HOST, PORT) 8 | 9 | tcpSerSock = socket(AF_INET, SOCK_STREAM) 10 | tcpSerSock.bind(ADDR) 11 | tcpSerSock.listen(5) 12 | 13 | while True: 14 | print('waiting for connection...') 15 | tcpCliSock, addr = tcpSerSock.accept() 16 | print('...connnecting from:', addr) 17 | 18 | while True: 19 | data = tcpCliSock.recv(BUFSIZ) 20 | if not data: 21 | break 22 | # tcpCliSock.send('[%s] %s' %(bytes(ctime(),'utf-8'),data)) 23 | tcpCliSock.send(('[%s] %s' % (ctime(), data)).encode()) 24 | tcpCliSock.close() 25 | tcpSerSock.close() -------------------------------------------------------------------------------- /test_frame/test_socket/test_udp_server.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | BUFSIZE = 2 4 | ip_port = ('127.0.0.1', 9999) 5 | server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # udp协议 6 | server.bind(ip_port) 7 | while True: 8 | data, client_addr = server.recvfrom(BUFSIZE) 9 | print('server收到的数据', data) 10 | 11 | server.sendto(data.upper(), client_addr) 12 | 13 | server.close() -------------------------------------------------------------------------------- /test_frame/test_socket/test_uod_client.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | BUFSIZE = 2 4 | client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 5 | while True: 6 | msg = input(">> ").strip() 7 | ip_port = ('127.0.0.1', 9999) 8 | client.sendto(msg.encode('utf-8'), ip_port) 9 | 10 | data, server_addr = client.recvfrom(BUFSIZE) 11 | print('客户端recvfrom ', data, server_addr) 12 | 13 | client.close() -------------------------------------------------------------------------------- /test_frame/test_speed/speed_test_consume.py: -------------------------------------------------------------------------------- 1 | # import gevent.monkey;gevent.monkey.patch_all() 2 | import time 3 | 4 | from function_scheduling_distributed_framework import task_deco, BrokerEnum,run_consumer_with_multi_process,ConcurrentModeEnum 5 | import nb_log 6 | 7 | logger = nb_log.get_logger('sdsda',is_add_stream_handler=False,log_filename='xxx.log') 8 | 9 | 10 | @task_deco('20000', broker_kind=BrokerEnum.REDIS,concurrent_num=2, log_level=20, qps=0,concurrent_mode=ConcurrentModeEnum.SINGLE_THREAD,) 11 | def f_test_speed(x): 12 | pass 13 | # logger.debug(x) 14 | # f_test_speed2.push(x * 10) 15 | print(x) 16 | # time.sleep(20) 17 | 18 | 19 | # @task_deco('speed_test_queue2', broker_kind=BrokerEnum.REDIS, log_level=20, qps=2) 20 | # def f_test_speed2(y): 21 | # pass 22 | # print(y) 23 | 24 | 25 | if __name__ == '__main__': 26 | # f_test_speed.clear() 27 | 28 | # for i in range(1000000): 29 | # f_test_speed.push(i) 30 | 31 | # f_test_speed.consume() 32 | run_consumer_with_multi_process(f_test_speed,1) 33 | # # f_test_speed2.consume() 34 | -------------------------------------------------------------------------------- /test_frame/test_speed/speed_test_push.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import redis 4 | from function_scheduling_distributed_framework import frame_config 5 | 6 | from test_frame.test_speed.speed_test_consume import f_test_speed 7 | 8 | redis_db_frame = redis.Redis(host=frame_config.REDIS_HOST, password=frame_config.REDIS_PASSWORD, port=frame_config.REDIS_PORT, db=frame_config.REDIS_DB) 9 | 10 | # f_test_speed.clear() 11 | for i in range(500000): 12 | 13 | f_test_speed.push(i) 14 | # redis_db_frame.lpush('no_frame_queue',f'{{"x":{i}}}') 15 | 16 | # from function_scheduling_distributed_framework.utils.redis_manager import RedisMixin 17 | # 18 | # 19 | # 20 | # r = RedisMixin().redis_db_frame_version3 21 | # 22 | # for _ in range(30): 23 | # # r.lpush('speed_test_queue',*[f'{{"x":{i}}}' for i in range (200000)]) 24 | # pass 25 | # 26 | # t1 = time.time() 27 | # # with r.pipeline() as p: 28 | # # for i in range(10000): 29 | # # p.lpushx('test567',i) 30 | # # p.execute() 31 | # 32 | # 33 | # # for i in range(10000): 34 | # # r.lpushx('test568',i) 35 | # 36 | # 37 | # r.lpush('stest569',*[i for i in range (10000)]) 38 | # print(time.time()-t1) -------------------------------------------------------------------------------- /test_frame/test_speed/speed_test_use_bigger_frame.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework.beggar_version_implementation.beggar_redis_consumer import start_consuming_message 2 | 3 | 4 | def f(x): 5 | print(x) 6 | 7 | start_consuming_message('no_frame_queue',consume_function=f) -------------------------------------------------------------------------------- /test_frame/test_speed/test_baidu_consume.py: -------------------------------------------------------------------------------- 1 | # import gevent.monkey;gevent.monkey.patch_all() 2 | import time 3 | 4 | from function_scheduling_distributed_framework import task_deco, BrokerEnum,run_consumer_with_multi_process,ConcurrentModeEnum 5 | import urllib3 6 | import requests 7 | http = urllib3.PoolManager() 8 | 9 | 10 | 11 | 12 | @task_deco('speed_baidu', broker_kind=BrokerEnum.REDIS, 13 | log_level=20, concurrent_num=60,concurrent_mode=ConcurrentModeEnum.THREADING, 14 | is_using_distributed_frequency_control=True,is_print_detail_exception=False) 15 | def baidu_speed(x,): 16 | # print(x) 17 | try: 18 | resp = requests.request('get','http://www.baidu.com/content-search.xml') 19 | except: 20 | pass 21 | 22 | 23 | 24 | 25 | if __name__ == '__main__': 26 | 27 | run_consumer_with_multi_process(baidu_speed,10) 28 | # # f_test_speed2.consume() 29 | -------------------------------------------------------------------------------- /test_frame/test_speed/test_baidu_push.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework.utils.redis_manager import RedisMixin 2 | 3 | 4 | 5 | r = RedisMixin().redis_db_frame_version3 6 | 7 | for i in range(100): 8 | r.lpush('speed_baidu',*[f'{{"x":{i}}}' for i in range(10000)]) -------------------------------------------------------------------------------- /test_frame/test_timing/test_timming.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, fsdf_background_scheduler, timing_publish_deco 3 | 4 | """ 5 | 定时的语法和入参与本框架无关系，不是本框架发明的定时语法，具体的需要学习 apscheduler包。 6 | """ 7 | 8 | @task_deco('queue_test_666', broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE) 9 | def consume_func(x, y): 10 | print(f'{x} + {y} = {x + y}') 11 | 12 | 13 | if __name__ == '__main__': 14 | # fsdf_background_scheduler.add_job(timing_publish_deco(consume_func), 'interval', id='3_second_job', seconds=3, kwargs={"x": 5, "y": 6}) # 每隔3秒发布一次任务，自然就能每隔3秒消费一次任务了。 15 | # fsdf_background_scheduler.add_job(timing_publish_deco(consume_func), 'date', run_date=datetime.datetime(2020, 7, 24, 13, 53, 6), args=(5, 6,)) # 定时，只执行一次 16 | # fsdf_background_scheduler.add_timing_publish_job(consume_func, 'cron', day_of_week='*', hour=14, minute=51, second=20, args=(5, 6,)) # 定时，每天的11点32分20秒都执行一次。 17 | fsdf_background_scheduler.add_timing_publish_job(consume_func, 'cron', day_of_week='*', hour='*', minute='*', second=20, args=(5, 6,)) # 定时，每分钟的第20秒执行一次 18 | # 启动定时 19 | fsdf_background_scheduler.start() 20 | # 启动消费 21 | consume_func.consume() -------------------------------------------------------------------------------- /test_frame/test_two_deco/test_2_deco.py: -------------------------------------------------------------------------------- 1 | import time 2 | from functools import wraps 3 | from function_scheduling_distributed_framework import task_deco 4 | 5 | def my_deco(f): 6 | wraps(f) 7 | def _inner(y): 8 | print('开始执行',y) 9 | f(y) 10 | print('结束执行',y) 11 | return _inner 12 | 13 | 14 | @task_deco('test2deco') 15 | @my_deco 16 | def fun(x): 17 | time.sleep(10) 18 | print(x) 19 | 20 | if __name__ == '__main__': 21 | fun.push(5) 22 | fun.consume() 23 | 24 | 25 | 26 | # for i in range(10): 27 | # fun.push(i) 28 | # fun.consume() 29 | # fun.multi_process_consume(2) -------------------------------------------------------------------------------- /test_frame/test_use_gevent/README.md: -------------------------------------------------------------------------------- 1 | 测试使用gevent并发模式。设置concurent_mode为2 2 | 默认是使用了线程。 -------------------------------------------------------------------------------- /test_frame/test_use_gevent/test_consume.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | # import gevent.monkey;gevent.monkey.patch_all() # 需要打猴子补丁。 5 | import eventlet;eventlet.monkey_patch(all=True) 6 | import time 7 | 8 | from function_scheduling_distributed_framework import get_consumer,ConcurrentModeEnum 9 | from function_scheduling_distributed_framework.utils import LogManager 10 | 11 | 12 | 13 | logger = LogManager('f2').get_logger_and_add_handlers() 14 | 15 | 16 | def f2(a, b): 17 | logger.info(f'消费此消息 {a} + {b} 中。。。。。') 18 | time.sleep(10) # 模拟做某事需要阻塞10秒种，必须用并发绕过此阻塞。 19 | logger.info(f'计算 {a} + {b} 得到的结果是 {a + b}') 20 | 21 | 22 | # 把消费的函数名传给consuming_function，就这么简单。 23 | consumer = get_consumer('queue_test62', consuming_function=f2, concurrent_num=200, msg_schedule_time_intercal=0.1, log_level=10, logger_prefix='zz平台消费', 24 | function_timeout=20, is_print_detail_exception=True, 25 | msg_expire_senconds=500, broker_kind=6, concurrent_mode=ConcurrentModeEnum.EVENTLET) # 通过设置broker_kind，一键切换中间件为mq或redis等7种中间件或包。 26 | 27 | if __name__ == '__main__': 28 | consumer.start_consuming_message() 29 | -------------------------------------------------------------------------------- /test_frame/test_use_gevent/test_publish.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author : ydf 3 | # @Time : 2019/8/8 0008 14:57 4 | from test_frame.test_use_gevent.test_consume import consumer 5 | 6 | consumer.publisher_of_same_queue.clear() 7 | [consumer.publisher_of_same_queue.publish({'a': i, 'b': 2 * i}) for i in range(10000)] 8 | -------------------------------------------------------------------------------- /test_frame/test_use_gevent/运行截图.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydf0509/distributed_framework/497f670fe77a9fa757c0b1a47e04b8c4af99339c/test_frame/test_use_gevent/运行截图.png -------------------------------------------------------------------------------- /test_frame/test_with_multi_process/test_consume.py: -------------------------------------------------------------------------------- 1 | import time 2 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, IdeAutoCompleteHelper, PriorityConsumingControlConfig, run_consumer_with_multi_process 3 | 4 | """ 5 | 演示多进程启动消费，多进程和 asyncio/threading/gevnt/evntlet是叠加关系，不是平行的关系。 6 | """ 7 | 8 | 9 | # qps=5，is_using_distributed_frequency_control=True 分布式控频每秒执行5次。 10 | # 如果is_using_distributed_frequency_control不设置为True,默认每个进程都会每秒执行5次。 11 | @task_deco('test_queue', qps=5, is_using_distributed_frequency_control=True) 12 | def ff(x, y): 13 | import os 14 | time.sleep(2) 15 | print(os.getpid(), x, y) 16 | 17 | 18 | if __name__ == '__main__': 19 | # ff.publish() 20 | ff.clear() 21 | for i in range(1000): 22 | ff.push(i, y=i * 2) 23 | 24 | # 这个与push相比是复杂的发布，第一个参数是函数本身的入参字典，后面的参数为任务控制参数，例如可以设置task_id，设置延时任务，设置是否使用rpc模式等。 25 | ff.publish({'x': i * 10, 'y': i * 20}, ) 26 | 27 | ff(666, 888) # 直接运行函数 28 | # ff.start() # 和 conusme()等效 29 | # ff.consume() # 和 start()等效 30 | # run_consumer_with_multi_process(ff, 2) # 启动两个进程 31 | ff.multi_process_start(3) # 启动两个进程，和上面的run_consumer_with_multi_process等效,现在新增这个multi_process_start方法。 32 | # IdeAutoCompleteHelper(ff).multi_process_start(3) # IdeAutoCompleteHelper 可以补全提示，但现在装饰器加了类型注释，ff. 已近可以在pycharm下补全了。 -------------------------------------------------------------------------------- /test_frame/test_with_multi_process/test_push.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import PriorityConsumingControlConfig,IdeAutoCompleteHelper 2 | from test_frame.test_with_multi_process.test_consume import ff 3 | 4 | for i in range(1000, 10000): 5 | ff.push(i, y=i * 2) 6 | 7 | # 这个与push相比是复杂的发布，第一个参数是函数本身的入参字典，后面的参数为任务控制参数，例如可以设置task_id，设置延时任务，设置是否使用rpc模式等。 8 | ff.publish({'x': i * 10, 'y': i * 20}, priority_control_config=PriorityConsumingControlConfig(countdown=10, misfire_grace_time=30)) 9 | ff(6,7) 10 | IdeAutoCompleteHelper(ff)(8,9) -------------------------------------------------------------------------------- /test_frame/test_xiaoxianrou/download_xiaoxianrou_pictures.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import task_deco 2 | import re 3 | import requests 4 | from parsel import Selector 5 | from pathlib import Path 6 | 7 | """ 8 | http://www.5442tu.com/mingxing/list_2_1.html 下载所有明星图片 9 | """ 10 | 11 | 12 | @task_deco('xiaoxianrou_list_page', qps=0.05) 13 | def cralw_list_page(page_index): 14 | url = f'http://www.5442tu.com/mingxing/list_2_{page_index}.html' 15 | resp = requests.get(url) 16 | sel = Selector(resp.content.decode('gbk')) 17 | detail_sels = sel.xpath('//div[@class="imgList2"]/ul/li/a[1]') 18 | for detail_sel in detail_sels: 19 | crawl_detail_page.push(detail_sel.xpath('./@href').extract_first(), detail_sel.xpath('./@title').extract_first(), 1, is_first_picture=True) 20 | 21 | 22 | @task_deco('xiaoxianrou_detail_page', qps=2,do_task_filtering=True) 23 | def crawl_detail_page(url, title, picture_index, is_first_picture=False,): 24 | resp = requests.get(url) 25 | sel = Selector(resp.content.decode('gbk')) 26 | if is_first_picture: # 详情页图册也需要翻页。 27 | total_page_str = sel.xpath('//div[@class="page"]/ul/li/a/text()').extract_first() 28 | total_page = int(re.search(r'共(\d+)页', total_page_str).group(1)) 29 | for p in range(2, total_page + 1): 30 | next_pic_page_url = url[:-5] + f'_{p}.html' 31 | crawl_detail_page.push(next_pic_page_url, title, picture_index=p) 32 | pic_url = sel.xpath('//p[@align="center"]/a/img/@src').extract_first() 33 | resp_pic = requests.get(pic_url) 34 | Path(f'./pictures/{title}/').mkdir(parents=True, exist_ok=True) 35 | (Path(f'./pictures/{title}/') / Path(f'./{title}_{picture_index}.jpg')).write_bytes(resp_pic.content) # 保存图片。 36 | print(f'''保存图片成功：\n {(Path(f'./pictures/{title}/') / Path(f'./{title}_{picture_index}.jpg')).absolute()} ''') 37 | 38 | 39 | if __name__ == '__main__': 40 | # cralw_list_page(1) 41 | # crawl_detail_page('https://www.5442tu.com/mingxing/20181105/78924.html','范冰冰弟弟范丞丞阳光帅气明星壁纸图片高清',1,True) 42 | cralw_list_page.clear() 43 | crawl_detail_page.clear() 44 | 45 | for p in range(1, 10): 46 | cralw_list_page.push(p) 47 | 48 | cralw_list_page.consume() 49 | crawl_detail_page.consume() 50 | -------------------------------------------------------------------------------- /test_frame/test_xiaoxianrou/download_xiaoxianrou_pictures_三函数版本.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework import task_deco 2 | import re 3 | import requests 4 | from parsel import Selector 5 | from pathlib import Path 6 | 7 | """ 8 | http://www.5442tu.com/mingxing/list_2_1.html 下载所有明星图片 9 | """ 10 | 11 | 12 | @task_deco('xiaoxianrou_list_page', qps=0.5) 13 | def cralw_list_page(page_index): 14 | url = f'http://www.5442tu.com/mingxing/list_2_{page_index}.html' 15 | resp = requests.get(url) 16 | sel = Selector(resp.content.decode('gbk')) 17 | detail_sels = sel.xpath('//div[@class="imgList2"]/ul/li/a') 18 | for detail_sel in detail_sels: 19 | crawl_detail_page.push(detail_sel.xpath('./@href').extract_first(), detail_sel.xpath('./@title').extract_first(), 1, is_first_picture=True) 20 | 21 | 22 | @task_deco('xiaoxianrou_detail_page', qps=3, ) 23 | def crawl_detail_page(url, title, picture_index, is_first_picture=False): 24 | resp = requests.get(url) 25 | sel = Selector(resp.content.decode('gbk')) 26 | if is_first_picture: # 详情页图册也需要翻页。 27 | total_page_str = sel.xpath('//div[@class="page"]/ul/li/a/text()').extract_first() 28 | total_page = int(re.search('共(\d+)页', total_page_str).group(1)) 29 | for p in range(2, total_page + 1): 30 | next_pic_page_url = url[:-5] + f'_{p}.html' 31 | crawl_detail_page.push(next_pic_page_url, title, picture_index=p) 32 | pic_url = sel.xpath('//p[@align="center"]/a/img/@src').extract_first() 33 | downlaod_picture.push(pic_url, title, picture_index) 34 | 35 | 36 | @task_deco('xiaoxianrou_download_pictiure', qps=1, is_print_detail_exception=False) 37 | def downlaod_picture(pic_url, title, picture_index): 38 | print(pic_url) 39 | resp_pic = requests.get(pic_url) 40 | Path(f'./pictures/{title}/').mkdir(parents=True, exist_ok=True) 41 | (Path(f'./pictures/{title}/') / Path(f'./{title}_{picture_index}.jpg')).write_bytes(resp_pic.content) # 保存图片。 42 | 43 | 44 | if __name__ == '__main__': 45 | # cralw_list_page(1) 46 | # crawl_detail_page('https://www.5442tu.com/mingxing/20181105/78924.html','范冰冰弟弟范丞丞阳光帅气明星壁纸图片高清',1,True) 47 | cralw_list_page.clear() 48 | crawl_detail_page.clear() 49 | downlaod_picture.clear() 50 | 51 | for p in range(1, 353): 52 | cralw_list_page.push(p) 53 | 54 | cralw_list_page.consume() 55 | crawl_detail_page.consume() 56 | downlaod_picture.consume() 57 | -------------------------------------------------------------------------------- /test_frame/tests_concurrent_pool/test_concurrent_pool1.py: -------------------------------------------------------------------------------- 1 | import time 2 | import asyncio 3 | from function_scheduling_distributed_framework.concurrent_pool import * 4 | from function_scheduling_distributed_framework.utils.decorators import TimerContextManager 5 | import aiohttp 6 | import requests 7 | import sys 8 | import threading 9 | 10 | url = 'http://mini.eastday.com/assets/v1/js/search_word.js' 11 | url = 'https://www.baidu.com/content-search.xml' 12 | 13 | total_succ_async =0 14 | total_succ_sync =0 15 | sync_count_lock = threading.Lock() 16 | 17 | async def async_request(i): 18 | try: 19 | async with aiohttp.request('get',url,timeout=aiohttp.ClientTimeout(20)) as resp: 20 | text = await resp.text() 21 | # print(text[:10]) 22 | print(f'异步{i}') 23 | await asyncio.sleep(0.1) 24 | global total_succ_async 25 | total_succ_async += 1 26 | except Exception as e: 27 | pass 28 | print(e) 29 | 30 | 31 | 32 | def sync_request(i): 33 | try: 34 | resp = requests.get(url,timeout=10) 35 | text = resp.text 36 | print(f'同步{i}') 37 | time.sleep(0.1) 38 | 39 | # print(text[:10]) 40 | global total_succ_sync 41 | with sync_count_lock: 42 | total_succ_sync += 1 43 | except Exception as e: 44 | pass 45 | print(e) 46 | 47 | pool_works = 600 48 | test_times = 10000 49 | pool1 = AsyncPoolExecutor(pool_works) 50 | pool2 = CustomThreadPoolExecutor(pool_works) 51 | 52 | t1 = time.time() 53 | for j in range(test_times): 54 | # t_submit =time.time() 55 | pool1.submit(async_request,j) 56 | # print(time.time()-t_submit) 57 | 58 | pool1.shutdown() 59 | spend_time_async = time.time() -t1 60 | 61 | t2 = time.time() 62 | with TimerContextManager(): 63 | for j in range(test_times): 64 | # t_submit = time.time() 65 | pool2.submit(sync_request, j) 66 | # print(time.time() - t_submit) 67 | pool2.shutdown() 68 | spend_time_sync = time.time() -t2 69 | 70 | print(total_succ_async) 71 | print(total_succ_sync) 72 | print(spend_time_async) 73 | print(spend_time_sync) 74 | 75 | -------------------------------------------------------------------------------- /test_frame/tests_concurrent_pool/test_concurrent_pool2.py: -------------------------------------------------------------------------------- 1 | import time 2 | import asyncio 3 | from function_scheduling_distributed_framework.concurrent_pool import * 4 | from function_scheduling_distributed_framework.utils.decorators import TimerContextManager 5 | import aiohttp 6 | import requests 7 | import sys 8 | import threading 9 | 10 | url = 'http://mini.eastday.com/assets/v1/js/search_word.js' 11 | # url = 'https://www.baidu.com/content-search.xml' 12 | 13 | total_succ_async = 0 14 | total_succ_sync = 0 15 | total_async_run = 0 16 | total_sync_run = 0 17 | spend_time_95_async = None 18 | spend_time_95_sync = None 19 | sync_count_lock = threading.Lock() 20 | 21 | loopx = asyncio.new_event_loop() 22 | ss = aiohttp.ClientSession(loop=loopx) 23 | 24 | 25 | async def async_request(i): 26 | global total_async_run, spend_time_95_async 27 | total_async_run += 1 28 | if total_async_run / test_times > 0.95 and spend_time_95_async is None: 29 | spend_time_95_async = time.time() - t1 30 | try: 31 | t0 = time.time() 32 | async with ss.request('get', url, timeout=aiohttp.ClientTimeout(30), ) as resp: 33 | text = await resp.text() 34 | # print(text[:10]) 35 | print(f'异步{i} {time.time() - t0}') 36 | # await asyncio.sleep(0.1) 37 | global total_succ_async 38 | total_succ_async += 1 39 | except Exception as e: 40 | pass 41 | # print(e) 42 | 43 | 44 | ss2 = requests.Session() 45 | 46 | 47 | def sync_request(i): 48 | global total_sync_run, spend_time_95_sync 49 | with sync_count_lock: 50 | total_sync_run += 1 51 | if total_sync_run / test_times > 0.9 and spend_time_95_sync is None: 52 | spend_time_95_sync = time.time() - t2 53 | try: 54 | t0 = time.time() 55 | resp = ss2.get(url, timeout=30) 56 | text = resp.text 57 | print(f'同步{i} {time.time() - t0}') 58 | # time.sleep(0.1) 59 | # print(text[:10]) 60 | global total_succ_sync 61 | with sync_count_lock: 62 | total_succ_sync += 1 63 | except: 64 | pass 65 | 66 | 67 | pool_works = 500 68 | test_times = 10 69 | pool1 = AsyncPoolExecutor(500, loop=loopx) 70 | pool2 = CustomThreadPoolExecutor(500) 71 | 72 | t1 = time.time() 73 | for j in range(test_times): 74 | print(j) 75 | # t_submit =time.time() 76 | pool1.submit(async_request, j) 77 | # print(time.time()-t_submit) 78 | pool1.shutdown() 79 | spend_time_async = time.time() - t1 80 | 81 | t2 = time.time() 82 | for j in range(test_times): 83 | # t_submit = time.time() 84 | pool2.submit(sync_request, j) 85 | # print(time.time() - t_submit) 86 | pool2.shutdown() 87 | spend_time_sync = time.time() - t2 88 | 89 | print(total_succ_async) 90 | print(total_succ_sync) 91 | print(spend_time_async) 92 | print(spend_time_sync) 93 | 94 | print(spend_time_95_async) 95 | print(spend_time_95_sync) 96 | -------------------------------------------------------------------------------- /test_frame/tests_concurrent_pool/test_cost_time.py: -------------------------------------------------------------------------------- 1 | from function_scheduling_distributed_framework.concurrent_pool.custom_threadpool_executor import CustomThreadpoolExecutor 2 | 3 | import time 4 | 5 | import threading 6 | from concurrent.futures import ThreadPoolExecutor 7 | 8 | 9 | pool = CustomThreadpoolExecutor(10) 10 | pool2 = ThreadPoolExecutor(10) 11 | t1 = time.time() 12 | 13 | lock = threading.Lock() 14 | def f(x): 15 | with lock: 16 | print(x) 17 | 18 | for i in range(10000): 19 | # pool.submit(f,i) 20 | pool2.submit(f,i) 21 | # threading.Thread(target=f,args=(i,)).start() 22 | # f(i) 23 | 24 | print("&&&",time.time() -t1) -------------------------------------------------------------------------------- /test_frame/tttt: -------------------------------------------------------------------------------- 1 | { 2 | "body": "W1szLCA2XSwge30sIHsiY2FsbGJhY2tzIjogbnVsbCwgImVycmJhY2tzIjogbnVsbCwgImNoYWluIjogbnVsbCwgImNob3JkIjogbnVsbH1d", 3 | "content-encoding": "utf-8", 4 | "content-type": "application/json", 5 | "headers": { 6 | "lang": "py", 7 | "task": "test_task\u554a", 8 | "id": "39198371-8e6a-4994-9f6b-0335fe2e9b92", 9 | "shadow": null, 10 | "eta": null, 11 | "expires": null, 12 | "group": null, 13 | "retries": 0, 14 | "timelimit": [ 15 | null, 16 | null 17 | ], 18 | "root_id": "39198371-8e6a-4994-9f6b-0335fe2e9b92", 19 | "parent_id": null, 20 | "argsrepr": "(3, 6)", 21 | "kwargsrepr": "{}", 22 | "origin": "gen22848@FQ9H7TVDZLJ4RBT" 23 | }, 24 | "properties": { 25 | "correlation_id": "39198371-8e6a-4994-9f6b-0335fe2e9b92", 26 | "reply_to": "3ef38b98-1417-3f3d-995b-89e8e15849fa", 27 | "delivery_mode": 2, 28 | "delivery_info": { 29 | "exchange": "", 30 | "routing_key": "test_a" 31 | }, 32 | "priority": 0, 33 | "body_encoding": "base64", 34 | "delivery_tag": "59e39055-2086-4be8-a801-993061fee443" 35 | } 36 | } -------------------------------------------------------------------------------- /test_frame/use_in_flask_tonardo_fastapi/flask_api.py: -------------------------------------------------------------------------------- 1 | import flask 2 | from flask_mail import Mail, Message 3 | from function_scheduling_distributed_framework import task_deco, BrokerEnum, PriorityConsumingControlConfig 4 | from function_scheduling_distributed_framework.publishers.base_publisher import RedisAsyncResult 5 | 6 | app = flask.Flask(__name__) 7 | 8 | app.config.update( 9 | MAIL_SERVER='MAIL_SERVER', 10 | MAIL_PORT=678, 11 | MAIL_USE_TLS=True, 12 | MAIL_USEERNAME='MAIL_USERNAME', 13 | MAIL_PASSWORD='MAIL_PASSWORD', 14 | MAIL_DEFAULT_SENDER=('Grey Li', 'MAIL_USERNAME') 15 | ) 16 | 17 | mail = Mail(app) 18 | 19 | 20 | @task_deco(queue_name='flask_test_queue', broker_kind=BrokerEnum.REDIS) 21 | def send_email(msg): 22 | """ 23 | 演示一般性大部分任务，如果函数不需要使用app上下文 24 | :param msg: 25 | :return: 26 | """ 27 | print(f'发送邮件 {msg}') 28 | 29 | 30 | @task_deco(queue_name='flask_test_queue', broker_kind=BrokerEnum.REDIS) 31 | def send_main_with_app_context(msg): 32 | """ 33 | 演示使用 flask_mail ，此包需要用到app上下文 34 | :param msg: 35 | :return: 36 | """ 37 | with app.app_context(): 38 | # 这个Message类需要读取flask的配置里面邮件发件人等信息，由于发布和消费不是同一太机器或者进程，必须使用上下文才嫩知道邮件配置 39 | message = Message(subject='title', recipients=['367224698@qq.com'], body=msg) 40 | mail.send(message) 41 | 42 | 43 | 44 | ############################# 如果你想再封装一点，就加个通用上下文装饰器开始 ################################ 45 | 46 | def your_app_context_deco(flask_appx: flask.Flask): 47 | def _deco(fun): 48 | def __deco(*args, **kwargs): 49 | with flask_appx.app_context(): 50 | return fun(*args, **kwargs) 51 | 52 | return _deco 53 | 54 | return _deco 55 | 56 | 57 | @task_deco(queue_name='flask_test_queue', broker_kind=BrokerEnum.REDIS) 58 | @your_app_context_deco(app) 59 | def send_main_with_app_context2(msg): 60 | """ 61 | 演示使用 flask_mail ，此包需要用到app上下文 62 | :param msg: 63 | :return: 64 | """ 65 | message = Message(subject='title', recipients=['367224698@qq.com'], body=msg) 66 | mail.send(message) 67 | 68 | ############################# 如果你想再封装一点，就加个通用上下文装饰器结束 ################################ 69 | 70 | 71 | # 这是falsk接口，在接口中发布任务到消息队列 72 | @app.route('/') 73 | def send_email_api(): 74 | """ 75 | :return: 76 | """ 77 | # 如果前端不关注结果，只是把任务发到中间件，那就使用 send_email.push(msg='邮件内容') 就可以了。 78 | async_result = send_email.publish(dict(msg='邮件内容'), priority_control_config=PriorityConsumingControlConfig(is_using_rpc_mode=True)) # type: RedisAsyncResult 79 | return async_result.task_id 80 | # return async_result.result # 不推荐，这种会阻塞接口，一般是采用另外写一个ajax接口带着task_id去获取结果，后端实现方式为 RedisAsyncResult(task_id,timeout=30).result 81 | 82 | # 如果要结果，最好的方式并不是轮询ajax，使用mqtt是最好的方案，前端订阅 mqtt topic，消费函数里面把结果发布到mqtt的topic。 83 | # 这种方式实现难度暴击使用后端导入一大推websocket相关的模块，性能也很好单个mqtt支持几十万长连接暴击python+websocket，实现上页更简单，并且实时性暴击ajax轮询。 84 | 85 | 86 | if __name__ == '__main__': 87 | app.run() 88 | -------------------------------------------------------------------------------- /test_frame/use_in_flask_tonardo_fastapi/readme.md: -------------------------------------------------------------------------------- 1 | ## 演示和web搭配使用，web接口中发布任务到中间件，另外单独启动后台消费。 2 | 3 | 4 | ## 如果前端要关注消费结果，最好的方式是mqtt，比ajax轮询和websocket都好。 -------------------------------------------------------------------------------- /test_frame/use_in_flask_tonardo_fastapi/run_backedn_tasks.py: -------------------------------------------------------------------------------- 1 | from test_frame.use_in_flask_tonardo_fastapi.flask_api import send_email,send_main_with_app_context 2 | 3 | if __name__ == '__main__': 4 | # send_main_with_app_context.cosume() 5 | send_email.consume() 6 | --------------------------------------------------------------------------------