├── .gitignore ├── corelib ├── __init__.py ├── api_auth │ ├── __init__.py │ ├── api_auth.py │ ├── bin │ │ └── token_manager │ ├── defaults.py │ └── token_api │ │ ├── __init__.py │ │ ├── api.py │ │ ├── handlers.py │ │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_authtoken.py │ │ └── __init__.py │ │ ├── models.py │ │ └── urls.py ├── api_base │ ├── __init__.py │ ├── api_field_types.py │ ├── api_handler_base.py │ ├── api_ingress_base.py │ ├── decorators.py │ ├── defaults.py │ └── file_upload_handler.py ├── api_serializing_mixins │ ├── __init__.py │ ├── add_data_mixin.py │ ├── defaults.py │ ├── delete_data_mixin.py │ ├── get_data_common.py │ ├── get_detail_data_mixin.py │ ├── get_list_data_mixin.py │ └── modify_data_mixin.py ├── asynctask │ ├── __init__.py │ ├── async_api │ │ ├── __init__.py │ │ ├── api.py │ │ ├── handlers.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_asynctask.py │ │ │ └── __init__.py │ │ ├── models.py │ │ └── urls.py │ ├── bin │ │ └── asynctask_server │ └── lib │ │ ├── __init__.py │ │ ├── async_task_client.py │ │ ├── async_task_server.py │ │ ├── decorators.py │ │ └── defaults.py ├── permission │ ├── __init__.py │ ├── api.py │ ├── decorators.py │ ├── defaults.py │ ├── handlers.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_apipermission.py │ │ └── __init__.py │ ├── models.py │ ├── tools.py │ └── urls.py ├── recorder │ ├── __init__.py │ ├── api.py │ ├── decorators.py │ ├── handlers.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_apicallingrecord.py │ │ └── __init__.py │ ├── models.py │ └── urls.py ├── timer │ ├── __init__.py │ ├── bin │ │ └── timer_server │ ├── lib │ │ ├── __init__.py │ │ ├── decorator.py │ │ ├── defaults.py │ │ └── service.py │ └── timer_api │ │ ├── __init__.py │ │ ├── api.py │ │ ├── choices.py │ │ ├── handlers.py │ │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_availabletasks_cronjob.py │ │ └── __init__.py │ │ ├── models.py │ │ └── urls.py └── tools │ ├── __init__.py │ ├── ansible_runner.py │ ├── coroutine.py │ ├── func_tools.py │ ├── gitlab_client.py │ ├── k8s_clients │ ├── __init__.py │ ├── client_base.py │ ├── configmap_client.py │ ├── daemonset_client.py │ ├── deployment_client.py │ ├── ingress_client.py │ ├── node_client.py │ └── service_client.py │ ├── logger.py │ ├── mysql_client.py │ └── saltstack_runner.py └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /corelib/__init__.py: -------------------------------------------------------------------------------- 1 | from .api_auth.api_auth import APIAuth 2 | from .api_base.api_ingress_base import APIIngressBase 3 | from .api_base.api_handler_base import APIHandlerBase 4 | from .api_base.file_upload_handler import FileUploader 5 | from .api_base.decorators import pre_handler 6 | from .api_base.api_field_types import BoolType, StrType, ChoiceType, ObjectType, ListType, DictType, IntType, DatetimeType, DateType, IPType, ScriptType 7 | 8 | __all__ = ( 9 | 'APIAuth', 10 | 'APIIngressBase', 11 | 'APIHandlerBase', 12 | 'FileUploader', 13 | 'pre_handler', 14 | 'BoolType', 15 | 'StrType', 16 | 'ChoiceType', 17 | 'ObjectType', 18 | 'ListType', 19 | 'DictType', 20 | 'IntType', 21 | 'DatetimeType', 22 | 'DateType', 23 | 'IPType', 24 | 'ScriptType', 25 | ) 26 | -------------------------------------------------------------------------------- /corelib/api_auth/__init__.py: -------------------------------------------------------------------------------- 1 | from .api_auth import APIAuth 2 | 3 | 4 | __all__ = ('APIAuth', ) 5 | -------------------------------------------------------------------------------- /corelib/api_auth/api_auth.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from .defaults import ACTION_STATIC_TOKENS 3 | import re 4 | 5 | 6 | class APIAuth(object): 7 | def auth_by_token(self, auth_token): 8 | # To check configuration for token_manager. 9 | if 'corelib.token_manager' not in settings.INSTALLED_APPS: 10 | # If `token_manager` is not used, user can use STATIC_TOKEN to simply authenticating API requests. 11 | if auth_token in ACTION_STATIC_TOKENS: 12 | return True 13 | return False 14 | # Plugably import. 15 | from corelib.token_manager.models import AuthToken 16 | from datetime import datetime, timedelta 17 | 18 | # To do token authentication. 19 | if re.search(r'^\w+\.\w+$', auth_token): 20 | username = auth_token.split('.')[0] 21 | token = auth_token.split('.')[1] 22 | queryset = AuthToken.objects.filter(username=username, token=token) 23 | if queryset.exists(): 24 | obj = queryset.get(username=username, token=token) 25 | if obj.expired_time == 0: 26 | return True 27 | if obj.sign_date + timedelta(seconds=obj.expired_time) > datetime.now(): 28 | return True 29 | return False 30 | 31 | def auth_by_session(self, user_obj): 32 | if user_obj.is_authenticated: 33 | return True 34 | return False 35 | -------------------------------------------------------------------------------- /corelib/api_auth/bin/token_manager: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | import os 3 | import sys 4 | import django 5 | import random 6 | import string 7 | import argparse 8 | import re 9 | 10 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 11 | sys.path.append(BASE_DIR) 12 | 13 | manage_file = os.path.join(BASE_DIR, 'manage.py') 14 | if not os.path.isfile(manage_file): 15 | raise SystemExit(f'FATAL: It seems not a django project in directory: {BASE_DIR}') 16 | 17 | DJANGO_SETTING = None 18 | with open(manage_file) as f: 19 | for line in f: 20 | if re.search(r"os.environ.setdefault\('DJANGO_SETTINGS_MODULE'", line): 21 | DJANGO_SETTING = line.split("'")[3] 22 | break 23 | if not DJANGO_SETTING: 24 | raise SystemExit(f"FATAL: It seems not a django `manage.py`: {manage_file}") 25 | 26 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", DJANGO_SETTING) 27 | django.setup() 28 | 29 | from corelib.api_auth.token_api.models import AuthToken 30 | from datetime import timedelta 31 | from django.utils import timezone 32 | 33 | 34 | class TokenManager(object): 35 | model = AuthToken 36 | username_format = r'^[\w\.\-\_]+$' 37 | 38 | def __init__(self, username_len=8, token_len=64): 39 | self.username_len = username_len 40 | self.token_len = token_len 41 | self.args = None 42 | 43 | def _token_dup_check(self, filter_field, func, *func_args): 44 | value = func(*func_args) 45 | while self.model.objects.filter(**{filter_field: value}).exists(): 46 | value = func(*func_args) 47 | return value 48 | 49 | def _gen_random_str(self, seeds, length): 50 | return ''.join(seeds[random.randint(0, len(seeds) - 1)] for i in range(length)) 51 | 52 | def _make_expired_message(self, token_obj): 53 | expired_message = 'Expired!' 54 | if token_obj.expired_time == 0: 55 | expired_message = "Never expired." 56 | elif(timezone.now() - token_obj.sign_date < timedelta(seconds=token_obj.expired_time)): 57 | expired_date = token_obj.sign_date + timedelta(seconds=token_obj.expired_time) 58 | expired_message = 'Will be expired at {}'.format(expired_date.strftime('%F %T')) 59 | return expired_message 60 | 61 | def _delete_obj(self, obj): 62 | username, token = obj.username, obj.token 63 | obj.delete() 64 | return '{}.{}\tDeleted!'.format(username, token) 65 | 66 | def make_token(self): 67 | if not self.args.username: 68 | self.args.username = self._token_dup_check('username', self._gen_random_str, string.ascii_letters, self.username_len) 69 | if not re.search(self.username_format, self.args.username) or len(self.args.username) > 64: 70 | raise SystemExit('Invalid username: {}'.format(self.args.username)) 71 | 72 | token = self._token_dup_check('token', self._gen_random_str, string.ascii_letters + string.digits, self.token_len) 73 | obj = self.model(username=self.args.username, token=token, expired_time=self.args.expired_time) 74 | obj.save() 75 | 76 | print('{}.{}\t{}'.format(self.args.username, token, self._make_expired_message(obj))) 77 | 78 | def list_token(self): 79 | for obj in self.model.objects.all(): 80 | print('{}.{}\t{}'.format(obj.username, obj.token, self._make_expired_message(obj))) 81 | 82 | def clean_token(self): 83 | for obj in self.model.objects.all(): 84 | if obj.expired_time == 0 or (timezone.now() - obj.sign_date > timedelta(seconds=obj.expired_time)): 85 | print(self._delete_obj(obj)) 86 | 87 | def delete_token(self): 88 | token = self.args.token_string.split('.') 89 | token_user = '.'.join(token[:-1]) 90 | token_key = token[-1] 91 | if len(token_key) != 64: 92 | raise SystemExit('Invalid token string: {}'.format(self.args.token_string)) 93 | if not re.search(self.username_format, token_user) or len(token_user) > 64: 94 | raise SystemExit('Invalid token string: {}'.format(self.args.token_string)) 95 | 96 | for obj in self.model.objects.filter(username=token_user, token=token_key): 97 | print(self._delete_obj(obj)) 98 | 99 | def run(self): 100 | parser = argparse.ArgumentParser(prog='token_manager', description="A command line tool to manage tokens.") 101 | 102 | subparsers = parser.add_subparsers(title='subcommands', dest='subcommand', metavar='') 103 | 104 | # Subcommand: add 105 | add_parser = subparsers.add_parser('add', help="To add a token") 106 | add_parser.add_argument('-u', '--user', metavar='username', required=False, dest='username', 107 | help='To specify a username. If not provided, A random username will be auto-generated.') 108 | add_parser.add_argument('-e', '--expired-time', metavar='n', type=int, required=False, default=86400, 109 | help="To specify a digit expired time in seconds for this token. Default is '86400', means one day. If '0' provided, this token \ 110 | will never be expired.") 111 | add_parser.set_defaults(func=self.make_token) 112 | 113 | # Subcommand: list 114 | list_parser = subparsers.add_parser('list', help="To list all tokens") 115 | list_parser.set_defaults(func=self.list_token) 116 | 117 | # Subcommand: delete 118 | delete_parser = subparsers.add_parser('delete', aliases=['del'], help="To delete a specified token") 119 | delete_parser.add_argument('token_string', metavar='', 120 | help="A token string in format: '.<64-length token>'") 121 | delete_parser.set_defaults(func=self.delete_token) 122 | 123 | # Subcommand: clean 124 | clean_parser = subparsers.add_parser('clean', help="To clean expired tokens") 125 | clean_parser.set_defaults(func=self.clean_token) 126 | 127 | # Dispatching 128 | self.args = parser.parse_args() 129 | if self.args.subcommand is None: 130 | parser.print_help() 131 | else: 132 | self.args.func() 133 | 134 | 135 | if __name__ == '__main__': 136 | token_manager = TokenManager() 137 | token_manager.run() 138 | -------------------------------------------------------------------------------- /corelib/api_auth/defaults.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | # Defaults. 5 | _ACTION_STATIC_TOKENS = [] # Static tokens are never expired. 6 | 7 | 8 | # Var that really works. 9 | ACTION_STATIC_TOKENS = getattr(settings, 'ACTION_STATIC_TOKENS', _ACTION_STATIC_TOKENS) 10 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/api_auth/token_api/__init__.py -------------------------------------------------------------------------------- /corelib/api_auth/token_api/api.py: -------------------------------------------------------------------------------- 1 | from corelib import APIIngressBase 2 | from .handlers import AuthTokenHandler 3 | 4 | 5 | class APIIngress(APIIngressBase): 6 | """ 7 | actions定义说明： 8 | 9 | getTokenList 获取token列表 10 | 11 | addToken 添加一个token 12 | 13 | deleteToken 删除一个token 14 | 15 | setTokenExpiredTime 设置token的失效时间。 16 | """ 17 | 18 | actions = { 19 | 'getTokenList': AuthTokenHandler, 20 | 'addToken': AuthTokenHandler, 21 | 'deleteToken': AuthTokenHandler, 22 | 'setTokenExpiredTime': AuthTokenHandler, 23 | } 24 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/handlers.py: -------------------------------------------------------------------------------- 1 | from corelib import APIHandlerBase, pre_handler, StrType, IntType, ObjectType 2 | from corelib.api_serializing_mixins import ListDataMixin, AddDataMixin, DeleteDataMixin, ModifyDataMixin 3 | from .models import AuthToken 4 | from corelib.tools.func_tools import genUUID 5 | 6 | 7 | class AuthTokenHandler(APIHandlerBase, ListDataMixin, AddDataMixin, DeleteDataMixin, ModifyDataMixin): 8 | post_fields = { 9 | # for list 10 | "search": StrType(), 11 | 'page_index': IntType(min=1), 12 | 'page_length': IntType(min=0), 13 | 14 | # for add 15 | 'username': StrType(regex=r'^[\w\.\-\_]+$', max_length=64), 16 | 17 | # for delete, set 18 | 'id': ObjectType(AuthToken), 19 | 'expired_time': IntType(min=0), 20 | } 21 | 22 | @pre_handler(opt=["search", "page_index", "page_length"], perm="admin") 23 | def getTokenList(self): 24 | self.getList(model=AuthToken) 25 | 26 | @pre_handler(opt=['username', 'expired_time'], perm="admin") 27 | def addToken(self): 28 | if not self.checked_params.get('username'): 29 | self.checked_params['username'] = genUUID(8) 30 | self.checked_params['token'] = genUUID(64) 31 | self.addData(AuthToken) 32 | 33 | @pre_handler(req=['id'], perm="admin") 34 | def deleteToken(self): 35 | self.deleteData() 36 | 37 | @pre_handler(req=['id', 'expired_time'], perm="admin") 38 | def setTokenExpiredTime(self): 39 | self.modifyData() 40 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 07:52 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ] 10 | 11 | operations = [ 12 | ] 13 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/migrations/0002_authtoken.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ('token_api', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='AuthToken', 17 | fields=[ 18 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 19 | ('username', models.CharField(default='', max_length=64, verbose_name='用户名称')), 20 | ('token', models.CharField(default='', max_length=64, verbose_name='用户TOKEN')), 21 | ('sign_date', models.DateTimeField(auto_now_add=True, verbose_name='注册日期')), 22 | ('expired_time', models.IntegerField(default=86400, verbose_name='有效期限')), 23 | ], 24 | options={ 25 | 'unique_together': {('username', 'token')}, 26 | }, 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/api_auth/token_api/migrations/__init__.py -------------------------------------------------------------------------------- /corelib/api_auth/token_api/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils import timezone 3 | from datetime import timedelta 4 | 5 | 6 | class AuthToken(models.Model): 7 | id = models.AutoField('ID', primary_key=True) 8 | username = models.CharField('用户名称', max_length=64, default='') 9 | token = models.CharField('用户TOKEN', max_length=64, default='') 10 | sign_date = models.DateTimeField('注册日期', auto_now_add=True) 11 | expired_time = models.IntegerField('有效期限', default=86400) # Default is One day. '0' means never expired. 12 | 13 | class Meta: 14 | unique_together = ['username', 'token'] 15 | 16 | @property 17 | def is_expired(self): 18 | if self.expired_time <= 0: 19 | return False 20 | return timezone.now() >= self.sign_date + timedelta(seconds=self.expired_time) 21 | 22 | # serializing settings 23 | list_fields = ['id', 'username', 'token', 'sign_date', 'expired_time', 'is_expired'] 24 | detail_fields = list_fields 25 | search_fields = ['username', 'token'] 26 | -------------------------------------------------------------------------------- /corelib/api_auth/token_api/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from .api import APIIngress 3 | 4 | urlpatterns = [ 5 | path('api/v1', APIIngress.as_view(), name="token_manager_api"), 6 | ] 7 | -------------------------------------------------------------------------------- /corelib/api_base/__init__.py: -------------------------------------------------------------------------------- 1 | from .api_field_types import ( 2 | BoolType, IntType, StrType, IPType, ScriptType, ChoiceType, DatetimeType, 3 | DateType, ObjectType, ListType, DictType 4 | ) 5 | 6 | from .api_handler_base import APIHandlerBase 7 | from .api_ingress_base import APIIngressBase 8 | from .file_upload_handler import FileUploader 9 | from .decorators import pre_handler 10 | 11 | __all__ = ('BoolType', 'IntType', 'StrType', 'IPType', 'ScriptType', 'ChoiceType', 'DatetimeType', 12 | 'DateType', 'ObjectType', 'ListType', 'DictType', 13 | 'APIHandlerBase', 'APIIngressBase', 'FileUploader', 'pre_handler') 14 | -------------------------------------------------------------------------------- /corelib/api_base/api_field_types.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import re 3 | from datetime import datetime 4 | 5 | 6 | # To define all supported field_types for operaters. 7 | class FieldType(object): 8 | """ 9 | This is the Top class of all field types class. 10 | All sub-classes must define a `check(self, field_value, ...)` method to validate parameters of api action handers. 11 | `check` method always returns a tuple: `checked_value, None` if checked successfully or `None, err_message`. 12 | """ 13 | def __init__(self, **kwargs): 14 | self.field_name = kwargs.get('name', None) 15 | self.err_prefix = f"Illegal value of field '{self.field_name}': " if self.field_name else "Illegal params: " 16 | 17 | def failed(self, error, ingore_prefix=False): 18 | return None, error if ingore_prefix else self.err_prefix + error 19 | 20 | def check(self): 21 | """ Need to be rewrite by sub-classs. """ 22 | return self.failed("ERROR: `check` method is not implemented.", ingore_prefix=True) 23 | 24 | 25 | class BoolType(FieldType): 26 | """ 27 | True or False. Default False. 28 | """ 29 | 30 | def __init__(self, default=False, **kwargs): 31 | super().__init__(**kwargs) 32 | self.default = default 33 | 34 | def __str__(self): 35 | return '' 36 | 37 | def check(self, field_value=None): 38 | if field_value is None: 39 | return self.default, None 40 | return bool(field_value), None # Means BoolType never check failed. 41 | 42 | 43 | class StrType(FieldType): 44 | """ 45 | This field type requires that field value must be a string, and can be matched with 'self.regex'. 46 | 47 | Initiallizing params: 48 | regex: can be provided at defining-time. Default '.*' to match any strings. 49 | """ 50 | 51 | def __init__(self, regex='.*', min_length=None, max_length=None, allow_none=False, **kwargs): 52 | super().__init__(**kwargs) 53 | self.regex = re.compile(regex) 54 | self.allow_none = allow_none 55 | self.min_length = min_length 56 | self.max_length = max_length 57 | 58 | def __str__(self): 59 | return f"" 60 | 61 | def check(self, field_value): 62 | if self.allow_none and field_value is None: 63 | return field_value, None 64 | _field_value = str(field_value) 65 | _len = len(_field_value) 66 | 67 | if self.min_length is not None and _len < self.min_length: 68 | return self.failed(f'Length of StrType value less than min_length {self.min_length}.') 69 | 70 | if self.max_length is not None and _len > self.max_length: 71 | return self.failed(f'Length of StrType value greater than max_length {self.max_length}.') 72 | 73 | if self.regex.match(_field_value): 74 | return _field_value, None 75 | return self.failed(f"Not match with regex '{self.regex}': '{field_value}'.") 76 | 77 | 78 | class ScriptType(FieldType): 79 | """ 80 | To check if field value contained dangerous command in each script line. 81 | """ 82 | 83 | def __init__(self, **kwargs): 84 | super().__init__(**kwargs) 85 | _pre, _suf = r'^(.*\s+|\&\&|\|\||)', r'(\s+.*|\\|)$' 86 | self.rules = { 87 | "alias": re.compile(f"{_pre}alias{_suf}"), 88 | "rm": re.compile(f"{_pre}rm{_suf}"), 89 | "mv": re.compile(f"{_pre}mv{_suf}"), 90 | "mysql": re.compile(f"{_pre}mysql{_suf}"), 91 | "redis": re.compile(rf"{_pre}redis-\w+{_suf}"), 92 | "mongo": re.compile(rf"{_pre}mongo-\w+{_suf}"), 93 | } 94 | 95 | def __str__(self): 96 | return '' 97 | 98 | def check(self, field_value): 99 | commands = str(field_value).split('\n') 100 | for cmd in commands: 101 | for k, v in self.rules.items(): 102 | if v.match(cmd): 103 | return self.failed(f"Dangerous command found: '{cmd}'.") 104 | return field_value, None 105 | 106 | 107 | class IntType(FieldType): 108 | """ 109 | Checking value should be integer type. Or can be transform to integer type as a string number. 110 | 111 | Initiallizing params: 112 | min: Number min value. 113 | max: Number max value. 114 | string_num: If True , checking value can be string number. 115 | allow_empty: Only usefull when is True. Allow checking value to be empty string ''. 116 | empty_return: Only usefull when both and are True. Return when checking value is empty string ''. 117 | """ 118 | 119 | def __init__(self, min=-9999999999, max=9999999999, string_num=False, allow_empty=False, empty_return=0, **kwargs): 120 | super().__init__(**kwargs) 121 | self.min = min 122 | self.max = max 123 | self.string_num = string_num 124 | self.string_num_type = StrType(r'^\d+$') 125 | self.allow_empty = allow_empty 126 | self.empty_return = empty_return 127 | 128 | def __str__(self): 129 | return '' 130 | 131 | def _check_value(self, value): 132 | if self.min <= value <= self.max: 133 | return value, None 134 | return self.failed(f"Number not in [{self.min}, {self.max}]: '{value}'.") 135 | 136 | def check(self, field_value=None): 137 | if self.string_num: 138 | if self.allow_empty and field_value == '': 139 | return self.empty_return, None 140 | _v, err = self.string_num_type.check(field_value) 141 | if err is not None: 142 | return self.failed(f"Illegal string number: '{field_value}'.") 143 | return self._check_value(int(_v)) 144 | elif isinstance(field_value, int): 145 | return self._check_value(int(field_value)) 146 | return self.failed(f"Not a number: '{field_value}'.") 147 | 148 | 149 | class ChoiceType(FieldType): 150 | """ 151 | This field type requires that field value must be one of self.choices. 152 | 153 | Initiallizing params: 154 | choices: collection of choice value. 155 | allow_empty: If True, allow field value to be , which is not in . 156 | empty_value: Only usefull when is True. Specify which value not in is extra allowed. 157 | empty_return: Only usefull when is True. Specify which value to be returned when check successfully. 158 | """ 159 | 160 | def __init__(self, *choices, allow_empty=False, empty_value=None, empty_return=None, **kwargs): 161 | super().__init__(**kwargs) 162 | self.choices = list(choices) 163 | self.allow_empty = allow_empty 164 | self.empty_value = empty_value 165 | self.empty_return = empty_return 166 | 167 | def __str__(self): 168 | return f"" 169 | 170 | def check(self, field_value): 171 | if self.allow_empty and field_value == self.empty_value: 172 | return self.empty_return, None 173 | if field_value in self.choices: 174 | return field_value, None 175 | return self.failed(f"Not in choices '{str(self.choices)}': '{field_value}'.") 176 | 177 | 178 | class IPType(FieldType): 179 | """ 180 | This field type requires that field value must be IP address or a IP subnet. 181 | 182 | Initiallizing params: 183 | check_subnet: If True, Treat field value as IP subnet to check. 184 | """ 185 | 186 | def __init__(self, check_subnet=False, **kwargs): 187 | super().__init__(**kwargs) 188 | self.check_subnet = check_subnet 189 | 190 | def __str__(self): 191 | return '' 192 | 193 | def _check_ip(self, value): 194 | try: 195 | socket.inet_aton(value) 196 | except Exception: 197 | return self.failed(f"Not an IP address: '{value}'.") 198 | return value, None 199 | 200 | def _check_subnet(self, value): 201 | err = self.failed(f"Not an IP subnet: '{value}'.") 202 | tmp_l = value.split('/') 203 | if len(tmp_l) != 2: 204 | return None, err 205 | _, _err = self._check_ip(tmp_l[0]) 206 | if _err is not None: 207 | return None, err 208 | 209 | try: 210 | subnet_mask_len = int(tmp_l[1]) 211 | except Exception: 212 | return err 213 | if not 0 <= subnet_mask_len <= 32: 214 | return None, err 215 | 216 | return value, None 217 | 218 | def check(self, field_value): 219 | _field_value = str(field_value) 220 | if self.check_subnet: 221 | return self._check_subnet(_field_value) 222 | return self._check_ip(_field_value) 223 | 224 | 225 | class DatetimeType(FieldType): 226 | """ 227 | This field type requires that field value must be a datetime string. 228 | 229 | Initiallizing params: 230 | format: datetime string format passed to function datetime.strptime(). 231 | 232 | extra_allowed_values: A list. Contains extra allowed values not match the `format`. 233 | extra_allowed_trans: A Dict. Using it to translate the extra_allowed_values to target values. 234 | """ 235 | 236 | def __init__(self, format='%Y-%m-%d %H:%M:%S', extra_allowed_values=None, extra_allowed_trans=None, **kwargs): 237 | super().__init__(**kwargs) 238 | self.format = format 239 | self.extra_allowed_values = [] if extra_allowed_values is None else extra_allowed_values 240 | self.extra_allowed_trans = {} if extra_allowed_trans is None else extra_allowed_trans 241 | 242 | def __str__(self): 243 | return f"" 244 | 245 | def check(self, field_value): 246 | if self.extra_allowed_values and field_value in self.extra_allowed_values: 247 | _value = self.extra_allowed_trans[field_value] if field_value in self.extra_allowed_trans else field_value 248 | return _value, None 249 | 250 | _field_value = str(field_value) 251 | try: 252 | date_time = datetime.strptime(_field_value, self.format) 253 | except ValueError: 254 | return self.failed(f"Not matched with datetime format '{self.format}': '{field_value}'.") 255 | return date_time, None 256 | 257 | 258 | class DateType(FieldType): 259 | """ 260 | This field type requires that field value must be a date string. 261 | 262 | Initiallizing params: 263 | format: date string format passed to function datetime.strptime(). 264 | """ 265 | 266 | def __init__(self, format='%Y-%m-%d', **kwargs): 267 | super().__init__(**kwargs) 268 | self.format = format 269 | 270 | def __str__(self): 271 | return f"" 272 | 273 | def check(self, field_value): 274 | _field_value = str(field_value) 275 | try: 276 | val = datetime.strptime(_field_value, self.format).date() 277 | except ValueError: 278 | return self.failed(f"Not matched with date format '{self.format}': '{field_value}'.") 279 | return val, None 280 | 281 | 282 | class ObjectType(FieldType): 283 | """ 284 | This field type requires that field value could be used to query out a unique data object from the specified db model. 285 | 286 | Initiallizing params: 287 | model: Must be a well defined in django app's models. 288 | identified_by: a field name defined in , which usually has a 'unique=True' defined. 289 | real_query: If True, query object data from by , return a db data object. 290 | If False, only check data exists or not, return the origin value. 291 | """ 292 | def __init__(self, model, identified_by='id', real_query=True, **kwargs): 293 | super().__init__(**kwargs) 294 | self.model = model 295 | self.identified_by = identified_by 296 | self.real_query = real_query 297 | 298 | def __str__(self): 299 | return f"" 300 | 301 | def check(self, field_value): 302 | object_filter = {self.identified_by: field_value} 303 | count = self.model.objects.filter(**object_filter).count() 304 | if count == 0: 305 | return self.failed(f"No data object matched by filter: '{self.identified_by}={field_value}'.") 306 | elif count >= 2: 307 | return self.failed(f"Multi data objects found by filter: '{self.identified_by}={field_value}'.") 308 | value = self.model.objects.get(**object_filter) if self.real_query else field_value 309 | return value, None 310 | 311 | 312 | class ListType(FieldType): 313 | """ 314 | This field type requires field value must be a list, which has items matched self.item_type. 315 | 316 | Initiallizing params: 317 | item: 'None' means no restriction on item. Or can be instance of any FieldTypes, to make a recursive checking. 318 | """ 319 | 320 | def __init__(self, item=None, **kwargs): 321 | super().__init__(**kwargs) 322 | self.item_type = item 323 | 324 | def __str__(self): 325 | return "" if self.item_type is None else f"" 326 | 327 | def check(self, field_value): 328 | if isinstance(field_value, list): 329 | if self.item_type is None: 330 | return field_value, None 331 | 332 | items = [] 333 | for item in field_value: 334 | _v, _err = self.item_type.check(item) 335 | if _err is not None: 336 | return self.failed(f"Item '{item}' not matched with item_type '{str(self.item_type)}'.") 337 | items.append(_v) 338 | return items, None 339 | return self.failed(f"Not a list: '{field_value}'.") 340 | 341 | 342 | class DictType(FieldType): 343 | """ 344 | This field type requires field value must be a dict. 345 | 346 | Initiallizing params: 347 | key: 'None' means no restriction. Or can be instance of any FieldTypes, to make a recursive checking. 348 | val: 'None' means no restriction. Or can be instance of any FieldTypes, to make a recursive checking. 349 | format: A dict with fixed key, and value must be instance of any FieldTypes, to make a recursive checking. 350 | Param , will be ignored when dict is provided. 351 | """ 352 | 353 | def __init__(self, key=None, val=None, format=None, **kwargs): 354 | super().__init__(**kwargs) 355 | self.key_type = key 356 | self.val_type = val 357 | self.format = format 358 | 359 | def __str__(self): 360 | if isinstance(format, dict): 361 | return f"" 362 | if self.key_type is not None: 363 | if self.val_type is not None: 364 | return f"" 365 | return f"" 366 | if self.val_type is not None: 367 | return f"" 368 | return f"" 369 | 370 | def check(self, field_value): 371 | if isinstance(field_value, dict): 372 | _dict = {} 373 | 374 | # if self.format is set, use the given format to check data. 375 | if isinstance(self.format, dict): 376 | err = self.failed(f"Not matched with fixed dict format '{str(self.format)}': '{field_value}'.") 377 | for key, fieldtype in self.format.items(): 378 | if key not in field_value: 379 | return None, err 380 | val_check, _err = fieldtype.check(field_value[key]) 381 | if _err is not None: 382 | return None, err 383 | _dict[key] = val_check 384 | return _dict, None 385 | 386 | # else use key/val type definations to check data. 387 | for key, val in field_value.items(): 388 | # To check keys. 389 | key_check, _err = key, None 390 | if self.key_type is not None: 391 | key_check, _err = self.key_type.check(key) 392 | if _err is not None: 393 | return self.failed(f"Dict key '{str(key)}' not matched with FieldType '{str(self.key_type)}'.") 394 | # To check values. 395 | val_check, _err = val, None 396 | if self.val_type is not None: 397 | val_check, _err = self.val_type.check(val) 398 | if _err is not None: 399 | return self.failed(f"Dict val '{str(val)}' not matched with FieldType '{str(self.val_type)}'.") 400 | _dict[key_check] = val_check 401 | return _dict, None 402 | return self.failed(f"Not a dict: '{field_value}'.") 403 | -------------------------------------------------------------------------------- /corelib/api_base/api_handler_base.py: -------------------------------------------------------------------------------- 1 | class APIHandlerBase(object): 2 | post_fields = {} 3 | 4 | def __init__(self, parameters=None, request=None, set_parameters_directly=False): 5 | # 由`apiIngress`传递的参数 6 | self.params = parameters if isinstance(parameters, dict) else {} 7 | self.request = request 8 | self.auth_token = self.params.pop("auth_token", None) 9 | self.action = self.params.pop("action", "") 10 | 11 | # 数据校验结果收集，在数据校验装饰器中设定 12 | self.checked_params = None 13 | 14 | # 将handler作为工具使用，而不是被`apiIngress`调用时，可以设置此参数，绕过数据校验逻辑 15 | self.set_parameters_directly = set_parameters_directly 16 | 17 | # 是否绕过pre_handler中的权限检查. bind_username用于手动指定接口调用人信息，是一个字符串。 18 | self.by_pass_perm_check = False 19 | self.by_pass_bind_username = '' 20 | 21 | # 处理结果 22 | self.user_perm = None 23 | self.result = True 24 | self.message = '' 25 | self.error_message = '' 26 | self.http_status = 200 27 | self.data = None 28 | 29 | def error(self, error_message, http_status=400, return_value=None, log=True): 30 | """ 31 | 当处理失败时，设置error的便捷方法 32 | """ 33 | self.result = False 34 | self.error_message += error_message 35 | self.http_status = http_status 36 | if log: 37 | print(self.error_message) 38 | err_log = { 39 | 'action': self.action, 40 | 'request_params': self.params, 41 | 'error_message': self.error_message, 42 | 'result': self.result, 43 | 'status_code': self.http_status, 44 | 'data': self.data 45 | } 46 | print(err_log) 47 | return return_value 48 | 49 | def setResult(self, *handlers, data=None): 50 | """ 51 | 用于此handler有直接调用其他handler时，合并多个handler的处理结果 52 | """ 53 | for handler in handlers: 54 | self.error_message += handler.error_message 55 | self.message += handler.message 56 | if not handler.result: 57 | self.result = False 58 | 59 | # 跟message，error_message, result不同，data只能设置一次 60 | if data: 61 | self.data = data 62 | -------------------------------------------------------------------------------- /corelib/api_base/api_ingress_base.py: -------------------------------------------------------------------------------- 1 | from django.views.generic import View 2 | from django.http import HttpResponse 3 | from django.views.decorators.csrf import csrf_exempt 4 | from django.utils.decorators import method_decorator 5 | from corelib import APIAuth 6 | from .defaults import ACTION_AUTH_REQUIRED, ACTIONS_AUTH_BY_PASS 7 | 8 | import json 9 | 10 | 11 | def get_error(msg, status_code=400): 12 | print(msg) 13 | return HttpResponse(msg, status=status_code) 14 | 15 | 16 | @method_decorator(csrf_exempt, name='dispatch') 17 | class APIIngressBase(View): 18 | actions = {} 19 | 20 | def post(self, request, *args, **kwargs): 21 | # To check post data in JSON. 22 | data = self.json_load(self.request) 23 | if isinstance(data, HttpResponse): 24 | return data 25 | action = data.get('action') 26 | auth_token = data.get('auth_token') 27 | 28 | # validate 'action' 29 | if not action: 30 | return get_error("ERROR: 'action' field is required.") 31 | if action not in self.actions: 32 | return get_error(f"ERROR: illegal action: '{action}'") 33 | 34 | # To get action func 35 | handler = self.actions[action](parameters=data, request=request) 36 | action_func = getattr(handler, action, None) 37 | if action_func is None: 38 | return get_error("ERROR: method not accomplished by handler.", 500) 39 | 40 | # authentication 41 | if action not in ACTIONS_AUTH_BY_PASS and ACTION_AUTH_REQUIRED: 42 | auth = APIAuth() 43 | if auth_token: 44 | if action_func._is_private: 45 | return get_error(f"ERROR: Private action '{action}' cannot authenticated by auth_token.") 46 | auth_result = auth.auth_by_token(auth_token) 47 | else: 48 | auth_result = auth.auth_by_session(request.user) 49 | if not auth_result: 50 | return get_error("ERROR: API authentication failed", 401) 51 | 52 | # To do the works. 53 | action_func() 54 | 55 | # make HttpResponse 56 | if handler.result: 57 | response_data = {"result": "SUCCESS", "message": str(handler.message)} 58 | if getattr(handler, 'data', None) is not None: 59 | response_data['data'] = handler.data 60 | if getattr(handler, 'data_total_length', None) is not None: 61 | response_data['data_total_length'] = handler.data_total_length 62 | else: 63 | response_data = {"result": "FAILED", "message": str(handler.error_message)} 64 | 65 | return HttpResponse(json.dumps(response_data), content_type='application/json', status=handler.http_status) 66 | 67 | def get(self, request, *args, **kwargs): 68 | return get_error("GET method is not allowed.", 403) 69 | 70 | def json_load(self, request, decode_type='utf-8'): 71 | """ 72 | To load json data from http request.body. 73 | If Not a JSON data, return ERROR. 74 | If data not a dict, return ERROR. 75 | """ 76 | 77 | try: 78 | post_data = json.loads(request.body.decode(decode_type)) 79 | except Exception: 80 | return get_error("ERROR: To load json data failed.") 81 | 82 | if isinstance(post_data, dict): 83 | return post_data 84 | else: 85 | return get_error("ERROR: Post data is not a dict.", 400) 86 | -------------------------------------------------------------------------------- /corelib/api_base/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from django.conf import settings 3 | from .api_field_types import BoolType, IntType 4 | 5 | 6 | def _dataValidate(self, req, opt): 7 | """ 8 | Not a decorator. 9 | Used in decorator `dataValidator`. 10 | """ 11 | if self.set_parameters_directly: 12 | return True 13 | 14 | _fd = self.post_fields 15 | # To check `req` fields first. 16 | for field in req: 17 | if field not in self.params: 18 | return self.error(f"ERROR: Field '{field}' is required.", return_value=False) 19 | 20 | # To set BoolType field in `opt` with default value. 21 | self.checked_params = {f: _fd[f].default for f in filter(lambda f: isinstance(_fd[f], BoolType), opt)} 22 | 23 | # To check value of all fields. Note: field not in `req` + `opt` will be dropped directly. 24 | for field in filter(lambda f: f in self.params, req + opt): 25 | field_type = _fd[field] 26 | checked_value, err = field_type.check(self.params[field]) 27 | if err is not None: 28 | return self.error(err, return_value=False) 29 | self.checked_params[field] = checked_value 30 | return True 31 | 32 | 33 | def dataValidator(req=None, opt=None): 34 | """ 35 | Can only be used for API action handlers. 36 | This decorator is used to validate 'self.params' for handlers. 37 | If no error, self.check_parameters will set with meaningful values. 38 | Params: 39 | req: A list, which contains field names must be provided. 40 | opt: A list, which contains field names can be provided optionally. 41 | """ 42 | def decorator(func): 43 | @wraps(func) 44 | def validate(self, *args, **kwargs): 45 | required = req if req is not None else [] 46 | optional = opt if opt is not None else [] 47 | 48 | # Default pagination 49 | if getattr(self, 'do_pagination', False): 50 | if self.post_fields.get('page_index', None) is None: 51 | self.post_fields['page_index'] = IntType(min=1) 52 | if self.post_fields.get('page_length', None) is None: 53 | self.post_fields['page_length'] = IntType(min=1) 54 | if 'page_index' not in optional and 'page_index' not in required: 55 | optional.append['page_index'] 56 | if 'page_length' not in optional and 'page_length' not in required: 57 | optional.append['page_length'] 58 | 59 | if not _dataValidate(self, required, optional): 60 | return None 61 | func_result = func(self, *args, **kwargs) 62 | return func_result 63 | return validate 64 | return decorator 65 | 66 | 67 | def pre_handler(req=None, opt=None, private=False, perm=None, record=False, record_label=None): 68 | """ 69 | Can only be used for API action handlers. 70 | To integrate other decorators together, with `permissionChecker` and `recorder` pluggable by django settings. 71 | Params: 72 | req Pass to decorator `dataValidator`. 73 | opt Pass to decorator `dataValidator`. 74 | private If 'True', authenticating by auth_token is not allowed for this action. 75 | perm Pass to decorator `permissionChecker`. If None, Means do not check user's permission for this handler. 76 | record Only useful when django app 'corelib.recorder' is installed. If True, handler calling will be recorded. 77 | record_label A readable name for action to record. 78 | """ 79 | def decorator(func): 80 | func = dataValidator(req, opt)(func) 81 | if 'corelib.recorder' in settings.INSTALLED_APPS and record: 82 | from corelib.recorder.decorators import recorder 83 | func = recorder(record_label)(func) 84 | if 'corelib.permission' in settings.INSTALLED_APPS and perm is not None: 85 | from corelib.permission.decorators import permissionChecker 86 | func = permissionChecker(perm)(func) 87 | 88 | func._is_private = private 89 | return func 90 | return decorator 91 | -------------------------------------------------------------------------------- /corelib/api_base/defaults.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | # Defaults. 5 | _ACTION_AUTH_REQUIRED = False # A global authencating switch. Set it to False for developing. 6 | _ACTIONS_AUTH_BY_PASS = ['login'] # Even though `AUTH_REQUIRED` is True, actions in this list can be by pass API authentication. 7 | 8 | 9 | # By pass API authentication settings. 10 | ACTION_AUTH_REQUIRED = getattr(settings, 'ACTION_AUTH_REQUIRED', _ACTION_AUTH_REQUIRED) 11 | ACTIONS_AUTH_BY_PASS = getattr(settings, 'ACTIONS_AUTH_BY_PASS', _ACTIONS_AUTH_BY_PASS) 12 | -------------------------------------------------------------------------------- /corelib/api_base/file_upload_handler.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django import forms 3 | from corelib.tools.func_tools import genUUID 4 | from django.utils import timezone 5 | from random import randint 6 | 7 | 8 | class UploadFileForm(forms.Form): 9 | file = forms.FileField() 10 | 11 | 12 | class FileUploader(object): 13 | """ 14 | 文件上传handler工具，需要在View请求处理函数中使用。 15 | 16 | 用法举例： 17 | ```python 18 | 19 | from django.http import HttpResponse 20 | from django.views.generic import View 21 | from django.views.decorators.csrf import csrf_exempt 22 | from django.utils.decorators import method_decorator 23 | from corelib import FileUploader 24 | import json 25 | 26 | @method_decorator(csrf_exempt, name='dispatch') 27 | class imageUploader(View): 28 | def post(self, request, *args, **kwargs): 29 | handler = FileUploader(request) 30 | handler.receiveFile() 31 | if not handler.result: 32 | return HttpResponse(handler.error_message, status=handler.http_status) 33 | return HttpResponse(json.dumps(handler.result_data), content_type='application/json') 34 | 35 | ``` 36 | 注意，以上示例没有做权限验证，任何人都可通过此接口上传文件，不可用作生产环境。 37 | """ 38 | 39 | def __init__(self, request, storage_base=None, storage_hash_type=None, filename_suffix_limit=None, keep_origin_filename=None, *args, **kwargs): 40 | """ 41 | 关于文件存储路径的设置属性： 42 | 43 | storage_base 本地存储根目录，要求绝对路径。默认值为'/tmp' 44 | 任何非法的设置，都会导致启用默认值。 45 | 46 | storage_hash_type 在storage_base存储根目录下，自动生成多级hash目录存储。 47 | 默认为None，表示不做多级目录存储，直接存储在storage_base目录下。 48 | 支持两种类型: 'number', 'date' 49 | - number: 表示随机自动生成1~255的三层存储目录。 50 | 例如：存储一个图片，'/tmp/2/255/10/test.png' 51 | - date: 表示根据当前日期生成三级存储目录。 52 | 例如：存储一个图片，'/tmp/2021/04/06/test.png' 53 | 54 | 关于文件类型限制的设置属性： 55 | （注意：目前只能从上传的文件名上做文件类型检查，无法从文件内容上做实际检查！） 56 | 57 | file_format_limits 是一个可以做in判断的数据对象，要求其元素为字符串，字母都需要小写。（列表，元组，集合都可）。 58 | 默认为None，表示不做限制。 59 | 例如：要限制只接受png，jpg格式的图片，可以设置为 ['png', 'jpg'] 60 | 61 | 若有无法满足需求的地方，请在继承此class时，重载这里的方法。 62 | 63 | 其他设置属性： 64 | 65 | keep_origin_filename 若为True则保留源文件的的文件名称。否则生成一个64位的随机文件名（默认）。 66 | """ 67 | self.request = request 68 | 69 | # 行为控制属性 70 | self.storage_base = storage_base 71 | self.storage_hash_type = storage_hash_type 72 | self.filename_suffix_limit = filename_suffix_limit 73 | self.keep_origin_filename = keep_origin_filename 74 | 75 | # 处理结果 76 | self.result = True 77 | self.message = '' 78 | self.error_message = '' 79 | self.http_status = 200 80 | self.result_data = None 81 | 82 | def error(self, error_message, http_status=400, return_value=None): 83 | """ 84 | 当处理失败时，设置error的便捷方法 85 | """ 86 | self.result = False 87 | self.error_message += error_message 88 | self.http_status = http_status 89 | return return_value 90 | 91 | def makeStoragePath(self): 92 | # 检查基础设置 93 | self.storage_base = str(self.storage_base) 94 | if not self.storage_base or not self.storage_base.startswith('/'): 95 | self.storage_base = '/tmp' 96 | 97 | # 创建存储跟目录 98 | if not os.path.isdir(self.storage_base): 99 | try: 100 | os.makedirs(self.storage_base) 101 | except Exception as e: 102 | self.error(f"ERROR: Failed create storage base dir: {self.storage_base}. {str(e)}", http_status=500) 103 | 104 | # 创建多级存储目录 105 | path = self.storage_base 106 | if self.storage_hash_type == 'number': 107 | for i in range(3): 108 | path = os.path.join(path, randint(1, 255)) 109 | elif self.storage_hash_type == 'date': 110 | _now = timezone.now() 111 | s_year = _now.strftime('%Y') 112 | s_month = _now.strftime('%m') 113 | s_day = _now.strftime('%d') 114 | path = os.path.join(path, s_year, s_month, s_day) 115 | if not os.path.isdir(path): 116 | try: 117 | os.makedirs(path) 118 | except Exception as e: 119 | self.error(f"ERROR: Failed create storage path: {path}. {str(e)}", http_status=500) 120 | 121 | return path 122 | 123 | def checkFileFormat(self, upload_file_name): 124 | """ 125 | 注意：只能从上传的文件名上做文件类型检查，无法从文件内容上做实际检查！ 126 | """ 127 | if self.filename_suffix_limit: 128 | tmp = upload_file_name.split('.') 129 | if len(tmp) < 2: 130 | return self.error(f"ERROR: Invalid file received: '{upload_file_name}'.", http_status=400) 131 | suffix = tmp[-1] 132 | if suffix.lower() not in self.filename_suffix_limit: 133 | return self.error(f"ERROR: Unsupported file format: '.{suffix}'.", http_status=400) 134 | return suffix 135 | 136 | def receiveFile(self): 137 | # 检查文件格式 138 | upload_file_name = str(self.request.FILES['file']) 139 | suffix = self.checkFileFormat(upload_file_name) 140 | if not self.result: 141 | return None 142 | 143 | # 创建存储目录 144 | storage_path = self.makeStoragePath() 145 | if not self.result: 146 | return None 147 | 148 | # make file storage full path 149 | if self.keep_origin_filename: 150 | file_path = os.path.join(storage_path, upload_file_name) 151 | else: 152 | uuid = genUUID(64) 153 | file_path = os.path.join(storage_path, f'{uuid}.{suffix}') 154 | while os.path.isfile(file_path): 155 | uuid = genUUID(64) 156 | file_path = os.path.join(storage_path, f'{uuid}.{suffix}') 157 | 158 | # Read file content and save it. 159 | form = UploadFileForm(self.request.POST, self.request.FILES) 160 | if form.is_valid(): 161 | with open(file_path, 'wb+') as f: 162 | for chunk in self.request.FILES['file'].chunks(): 163 | f.write(chunk) 164 | self.result_data = { 165 | 'result': 'SUCCESS', 166 | 'message': 'To upload file succeeded.', 167 | 'file_path': file_path 168 | } 169 | else: 170 | self.error('ERROR: Invalid file uploading post.', http_status=400) 171 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/__init__.py: -------------------------------------------------------------------------------- 1 | from .get_list_data_mixin import ListDataMixin 2 | from .get_detail_data_mixin import DetailDataMixin 3 | from .add_data_mixin import AddDataMixin 4 | from .delete_data_mixin import DeleteDataMixin 5 | from .modify_data_mixin import ModifyDataMixin 6 | 7 | 8 | __all__ = ('ListDataMixin', 'DetailDataMixin', 'AddDataMixin', 'DeleteDataMixin', 'ModifyDataMixin') 9 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/add_data_mixin.py: -------------------------------------------------------------------------------- 1 | from django.db.models import ManyToManyField 2 | 3 | 4 | class AddDataMixin(object): 5 | """ 6 | 作为核心功能的扩展，必须和核心类`APIHandlerBase`一起使用。 7 | """ 8 | 9 | def addData(self, model, row_data=None, success_msg=None, error_msg=None): 10 | if not row_data: 11 | row_data = self.checked_params 12 | 13 | # To peel off ManytoManyField data. 14 | m2m_fields = {} 15 | for field in filter(lambda f: isinstance(model._meta.get_field(f), ManyToManyField), row_data): 16 | m2m_fields[field] = row_data[field] 17 | for f in m2m_fields: 18 | row_data.pop(f) 19 | 20 | # To write normal fields first 21 | obj = model(**row_data) 22 | try: 23 | obj.save() 24 | except Exception as e: 25 | _msg = f"Failed to add data. {str(e)}" if error_msg is None else error_msg 26 | return self.error(_msg) 27 | 28 | # Then, to set m2m fields. 29 | for f in m2m_fields: 30 | getattr(obj, f).set(m2m_fields[f]) 31 | 32 | self.message = f"To add data succeeded." if success_msg is None else success_msg 33 | return obj 34 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/defaults.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | # Defaults. 5 | _DEFAULT_PAGE_LENGTH = 10 6 | 7 | 8 | # By pass API authentication settings. 9 | DEFAULT_PAGE_LENGTH = getattr(settings, 'DEFAULT_PAGE_LENGTH', _DEFAULT_PAGE_LENGTH) 10 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/delete_data_mixin.py: -------------------------------------------------------------------------------- 1 | class DeleteDataMixin(object): 2 | """ 3 | 作为核心功能的扩展，必须和核心类`APIHandlerBase`一起使用。 4 | """ 5 | 6 | def deleteData(self, identifier='id', success_msg=None, error_msg=None): 7 | obj = self.checked_params[identifier] 8 | try: 9 | obj.delete() 10 | except Exception as e: 11 | _msg = f"Failed to delete data with '{identifier}={self.params[identifier]}'. {str(e)}" if error_msg is None else error_msg 12 | return self.error(_msg) 13 | 14 | self.message = f"To delete data succeeded." if success_msg is None else success_msg 15 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/get_data_common.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, date, time 2 | from django.db.models import ForeignKey, ManyToManyField, OneToOneField, Model 3 | 4 | 5 | class BaseSerializingMixin(object): 6 | """ 7 | 集成了listDataMixin与DetailDataMixin的通用处理函数 8 | """ 9 | # 默认日期转换格式 10 | date_format = '%F' 11 | time_format = '%T' 12 | datetime_format = '%F %T' 13 | datetime_serializer = None # 自定义日期序列化函数，接受一个datetime对象，返回一个日期str 14 | 15 | def dateTimeSerializing(self, value): 16 | """ 17 | 用于将日期、时间对象的数据，转换成字符串 18 | """ 19 | if value is None: 20 | return '' 21 | 22 | if not (isinstance(value, datetime) or isinstance(value, date) or isinstance(value, time)): 23 | return value 24 | 25 | # 自定义日期序列化处理 26 | if self.datetime_serializer is not None: 27 | return self.datetime_serializer(value) 28 | 29 | # 默认日期序列化处理 30 | if isinstance(value, datetime): 31 | value = value.strftime(self.datetime_format) 32 | elif isinstance(value, date): 33 | value = value.strftime(self.date_format) 34 | elif isinstance(value, time): 35 | value = value.strftime(self.time_format) 36 | return value 37 | 38 | def getObjAttr(self, obj, field): 39 | """ 40 | 获取字段值; 41 | 关系型字段，按约定格式执行递归查找； 42 | 关系型字段，不管有没有指定具体属性，都会返回其id。 43 | """ 44 | if not isinstance(obj, Model): 45 | raise Exception("`obj` pass to this function must be an instance of a DB Model!") 46 | 47 | # 非关系型字段，直接取值。也作为递归的终止条件 48 | if isinstance(field, str): 49 | val = getattr(obj, field, None) 50 | if val is None: 51 | return field, None 52 | 53 | # 对未明确指定下级属性的关系型字段，仅返回下级obj的id 54 | if not isinstance(getattr(type(obj), field, None), property): 55 | if isinstance(obj._meta.get_field(field), OneToOneField) or isinstance(obj._meta.get_field(field), ForeignKey): 56 | val = {'id': val.id} 57 | elif isinstance(obj._meta.get_field(field), ManyToManyField): 58 | val = [{'id': obj.id} for obj in val.all()] 59 | 60 | # 处理时间类型的字段 61 | val = self.dateTimeSerializing(val) 62 | 63 | return field, val 64 | elif isinstance(field, dict): 65 | # ManyToManyField filters. 66 | m2m_excludes = field.get('__exclude__', None) 67 | m2m_filters = field.get('__filter__', None) 68 | 69 | # 获取关系字段名称，下级属性列表 70 | _tmp = [(k, v) for k, v in field.items() if k not in {'__exclude__', '__filter__'}] 71 | if len(_tmp) != 1: 72 | raise Exception(f"Relational field's serializing setting illegal: {field}") 73 | field_name, sub_fields = _tmp[0] 74 | if not isinstance(sub_fields, list) and not isinstance(sub_fields, tuple): 75 | raise Exception("Relational field's serializing setting illegal, sub fields must be a `list` or `tuple`!") 76 | 77 | sub_obj = getattr(obj, field_name, None) 78 | if sub_obj is None: 79 | return field_name, None 80 | 81 | sub_fields = list(sub_fields) 82 | if 'id' not in sub_fields: 83 | sub_fields.append('id') 84 | 85 | # ForeignKey或OneToOneField， 返回字典 86 | if isinstance(obj._meta.get_field(field_name), ForeignKey) or isinstance(obj._meta.get_field(field_name), OneToOneField): 87 | field_attrs = {} 88 | for sub_field in sub_fields: 89 | k, v = self.getObjAttr(obj=sub_obj, field=sub_field) 90 | field_attrs[k] = v 91 | return field_name, field_attrs 92 | 93 | # ManyToManyField返回列表（可做数据过滤），列表的元素是字典 94 | elif isinstance(obj._meta.get_field(field_name), ManyToManyField): 95 | attr_list = [] 96 | sub_queryset = sub_obj.all() 97 | if m2m_filters: 98 | for k, v in m2m_filters: 99 | sub_queryset = sub_queryset.filter(**{k: v}) 100 | if m2m_excludes: 101 | for k, v in m2m_excludes: 102 | sub_queryset = sub_queryset.exclude(**{k: v}) 103 | for real_sub_obj in sub_queryset: 104 | field_attrs = {} 105 | for sub_field in sub_fields: 106 | k, v = self.getObjAttr(obj=real_sub_obj, field=sub_field) 107 | field_attrs[k] = v 108 | attr_list.append(field_attrs) 109 | return field_name, attr_list 110 | 111 | # 关系属性值类型未知, 返回None，终止递归 112 | else: 113 | return field_name, None 114 | else: 115 | raise Exception("Serializing setting illegal, field must be `str` or `dict`.") 116 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/get_detail_data_mixin.py: -------------------------------------------------------------------------------- 1 | from .get_data_common import BaseSerializingMixin 2 | 3 | 4 | class DetailDataMixin(BaseSerializingMixin): 5 | """ 6 | 作为核心功能的扩展，必须和核心类`APIHandlerBase`一起使用，最终会将self.data设置为一个字典。 7 | 8 | 序列化约定： 9 | 1、model的定义中，需包定义一个`detail_fields: list`属性。 10 | 2、对于ForeignKey, MangToManyField, OneToOneField等关系型字段， 11 | 请在`detail_fields`中以字典定义其取值层级关系，字段长度为1，key为field_name, value为下级属性； 12 | 如没有以字典做明确指定，则默认仅返回下级obj的id； 13 | ForeignKey， OneToOneField字段，序列化后，会扩展成一个字典； 14 | ManyToManyField字段，序列化后，会扩展为一个列表，列表的元素下级obj的属性字典。 15 | 3、多余多级关系，下级属性也可遵循以上约定，实现递归取值； 16 | 4、日期时间类型默认按“%F %T”格式序列化； 17 | 可通过设置`self.date_format`, `self.time_format`, `self.datetime_format`属性来自定义 18 | """ 19 | 20 | def getDetail(self, model, identifier='id', obj=None, excluded_fields=None): 21 | obj = self.checked_params[identifier] if obj is None else obj 22 | self.data = {} 23 | detail_fields = getattr(model, 'detail_fields', None) 24 | if detail_fields is None: 25 | detail_fields = [f.name for f in model._meta.get_fields()] 26 | 27 | if isinstance(excluded_fields, list): 28 | detail_fields = [f for f in detail_fields if f not in excluded_fields] 29 | for field in detail_fields: 30 | k, v = self.getObjAttr(obj, field) 31 | self.data[k] = v 32 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/get_list_data_mixin.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Q 2 | from .defaults import DEFAULT_PAGE_LENGTH 3 | from .get_data_common import BaseSerializingMixin 4 | 5 | 6 | class ListDataMixin(BaseSerializingMixin): 7 | """ 8 | 集成了针对列表数据的序列化处理方法，支持search与filter列表数据过滤处理。 9 | 10 | 作为核心功能的扩展，必须和核心类`APIHandlerBase`一起使用，最终会将self.data设置为一个列表。 11 | 12 | 序列化约定： 13 | 1、model的定义中，需包定义一个`list_fields: list`属性 14 | 2、对于ForeignKey, MangToManyField, OneToOneField等关系型字段， 15 | 请在`list_fields`中以字典定义其取值层级关系，字段长度为1，key为field_name, value为下级属性； 16 | 如没有以字典做明确指定，则默认仅返回下级obj的id； 17 | ForeignKey， OneToOneField字段，序列化后，会扩展成一个字典； 18 | ManyToManyField字段，序列化后，会扩展为一个列表，列表的元素下级obj的属性字典。 19 | 3、多余多级关系，下级属性也可遵循以上约定，实现递归取值 20 | 4、日期时间类型默认按“%F %T”格式序列化， 21 | 可通过设置`self.date_format`, `self.time_format`, `self.datetime_format`属性来自定义。 22 | 23 | 分页约定： 24 | 1、post数据中需包含'page_index'，表示当前页码 25 | 2、post数据中可以包含'page_length'字段，表示单页的数据条数 26 | 3、若不指定'page_length'，默认长度为`DEFAULT_PAGE_LENGTH` 27 | 4、数据总条数存储在`data_total_length`属性中 28 | 29 | 搜索约定： 30 | 1、post数据中包含'search'字段时，调用getList，会触发多字段模糊搜索比配，具体哪些字段，请在model中定义'search_fields' 31 | 2、post数据中包含model定义的db字段，会触发，精确的filter过滤，具体哪些字段，请在model中定义'filter_fields' 32 | 3、关系型字段，请以'.'分隔表示层级关系 33 | 4、当search与filter的post传值为None，或者search为空字符串时，会当做未传值处理， 34 | 未传值则不依此来搜索/过滤，返回所有其他匹配条件的数据。 35 | 5、特别注意：post_fields定义时，要允许为None 36 | """ 37 | # 默认开启分页功能 38 | auto_pagination = True 39 | 40 | # 数据总长度，分页功能使用 41 | data_total_length = None 42 | 43 | def _makeSearchFilter(self, fields, value): 44 | """ 45 | search搜索多个字段、模糊匹配、不区分大小写 46 | """ 47 | q = Q(**{f"{fields.pop().replace('.', '__')}__icontains": value}) 48 | if fields: 49 | return q | self._makeSearchFilter(fields, value) 50 | else: 51 | return q 52 | 53 | def pagination(self, queryset, is_queryset=True): 54 | """ 55 | 对queryset或者list数据执行分页计算，填充`self.data_total_length`属性 56 | """ 57 | self.data_total_length = queryset.count() if is_queryset else len(queryset) 58 | page_index = self.checked_params['page_index'] 59 | page_length = self.checked_params['page_length'] if self.checked_params.get('page_length') else DEFAULT_PAGE_LENGTH 60 | return queryset[(page_index - 1) * page_length: page_index * page_length] 61 | 62 | def getQueryset( 63 | self, 64 | model, 65 | spec_qs=None, 66 | filters=None, 67 | search_value=None, 68 | search_fields=None, 69 | order_by=None, 70 | excludes=None, 71 | additional_filters=None): 72 | """ 73 | 执行过滤，搜索，返回一个真实queryset 74 | """ 75 | queryset = model.objects.all() if spec_qs is None else spec_qs 76 | 77 | # 先按filter精确过滤 78 | if filters: 79 | query_filter = {f.replace('.', '__'): filters[f] for f in filters.keys()} 80 | queryset = model.objects.filter(**query_filter) 81 | 82 | # 然后执行按search模糊搜索 83 | if search_value and search_fields: 84 | queryset = queryset.filter(self._makeSearchFilter(search_fields, search_value)) 85 | 86 | # 执行额外的条件过滤 87 | if isinstance(additional_filters, dict): 88 | queryset = queryset.filter(**additional_filters) 89 | 90 | # 排除过滤 91 | if excludes is not None: 92 | for k, v in excludes: 93 | queryset = queryset.exclude(**{k: v}) 94 | 95 | # 排序 96 | if order_by is not None: 97 | queryset = queryset.order_by(order_by) 98 | 99 | return queryset.distinct() 100 | 101 | def _id_unique(self, list_data): 102 | """ 103 | 基于ManyToManyField的下级属性做过滤，会造成数据重复，在这里保证行数据不重复 104 | """ 105 | tmp_set = set() 106 | data = [] 107 | for raw in list_data: 108 | if raw['id'] not in tmp_set(): 109 | data.append(raw) 110 | tmp_set.add(raw['id']) 111 | return data 112 | 113 | def makeListData(self, queryset, model): 114 | """ 115 | 将queryset基于model.list_fields设置，转换成可序列化的数据列表; 116 | 不管model.list_fields有没有指定id，都会返回id属性； 117 | 另外，基于ManyToManyField的下级属性做过滤，会造成数据重复，在这里会保证每条数据id不重复。 118 | """ 119 | list_fields = getattr(model, 'list_fields', None) 120 | if list_fields is None: 121 | list_fields = [f.name for f in model._meta.get_fields()] 122 | if 'id' not in list_fields: 123 | list_fields.append('id') 124 | 125 | list_data = [] 126 | for obj in queryset: 127 | raw = {} 128 | for field in list_fields: 129 | k, v = self.getObjAttr(obj, field) 130 | raw[k] = v 131 | list_data.append(raw) 132 | return list_data 133 | 134 | def getList(self, model, spec_qs=None, order_by=None, excludes=None, additional_filters=None): 135 | if self.checked_params is None: 136 | self.checked_params = {} 137 | search = { 138 | 'search_value': self.checked_params.get('search'), 139 | 'search_fields': list(getattr(model, 'search_fields', [])), 140 | 'filters': {f: self.checked_params[f] for f in getattr(model, 'filter_fields', []) if self.checked_params.get(f) is not None}, 141 | 'order_by': order_by, 142 | 'excludes': excludes, 143 | 'additional_filters': additional_filters, 144 | 'spec_qs': spec_qs, 145 | } 146 | queryset = self.getQueryset(model, **search) 147 | if not queryset.exists(): 148 | self.data = [] 149 | else: 150 | if self.auto_pagination and "page_index" not in self.checked_params: 151 | self.checked_params['page_index'] = 1 152 | if "page_index" in self.checked_params: 153 | queryset = self.pagination(queryset) 154 | self.data = self.makeListData(queryset, model) 155 | return self.data 156 | -------------------------------------------------------------------------------- /corelib/api_serializing_mixins/modify_data_mixin.py: -------------------------------------------------------------------------------- 1 | from django.db.models import ManyToManyField 2 | 3 | 4 | class ModifyDataMixin(object): 5 | """ 6 | 作为核心功能的扩展，必须和核心类`APIHandlerBase`一起使用。 7 | """ 8 | 9 | def modifyData(self, identifier='id', obj=None, success_msg=None, error_msg=None, update_fields=None): 10 | if obj is None: 11 | obj = self.checked_params.pop(identifier) 12 | changed = False 13 | for f in filter(lambda k: getattr(obj, k, None) != self.checked_params[k], self.checked_params): 14 | val = self.checked_params[f] 15 | if isinstance(obj._meta.get_field(f), ManyToManyField): # Handle m2m field. 16 | getattr(obj, f).set(val) 17 | else: 18 | setattr(obj, f, val) 19 | changed = True 20 | if changed: 21 | try: 22 | obj.save(update_fields=update_fields) 23 | except Exception as e: 24 | _msg = f"Failed to modify data with '{identifier}={self.params[identifier]}'. {str(e)}" if error_msg is None else error_msg 25 | return self.error(_msg, return_value=False) 26 | 27 | self.message = f"To modify data with '{identifier}={self.params[identifier]}' succeeded." if success_msg is None else success_msg 28 | return changed 29 | 30 | def onlyUpdate(self, model, filters, values): 31 | """ 32 | 作为修改数据的一种快捷方式，相对modifyData方法，DB执行效率更好。但是不够灵活。 33 | 返回匹配到的更新行数，Int型。 34 | """ 35 | try: 36 | rows = model.objects.filter(**filters).update(**values) 37 | except Exception as e: 38 | return self.error(f"ERROR: Failed to execute SQL update. {str(e)}") 39 | 40 | self.message = f"{rows} row updated." 41 | return rows 42 | -------------------------------------------------------------------------------- /corelib/asynctask/__init__.py: -------------------------------------------------------------------------------- 1 | from .lib.async_task_client import ClientResultException, ClientRunningException, AsyncClient 2 | from .lib.decorators import asynctask 3 | from .lib.async_task_server import MainServer 4 | 5 | __all__ = ('ClientResultException', 'ClientRunningException', 'AsyncClient', 'asynctask', 'MainServer') 6 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/asynctask/async_api/__init__.py -------------------------------------------------------------------------------- /corelib/asynctask/async_api/api.py: -------------------------------------------------------------------------------- 1 | from corelib import APIIngressBase 2 | from .handlers import AsyncTaskListAPI, AsyncTaskLogAPI 3 | 4 | 5 | class APIIngress(APIIngressBase): 6 | actions = { 7 | 'getList': AsyncTaskListAPI, 8 | 'delete': AsyncTaskListAPI, 9 | 'getLog': AsyncTaskLogAPI 10 | } 11 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/handlers.py: -------------------------------------------------------------------------------- 1 | from corelib import APIHandlerBase, pre_handler, IntType, StrType, ChoiceType, ObjectType 2 | from .models import AsyncTask 3 | from corelib.asynctask.config import ASYNC_TASK_LOGDIR, ASYNC_TASK_LOGFILE_PREFIX 4 | import os 5 | 6 | 7 | class AsyncTaskListAPI(APIHandlerBase): 8 | fields_defination = { 9 | 'search': StrType(name='search'), 10 | 'status': ChoiceType(*[item[0] for item in AsyncTask.STATUS_OPTIONS], allow_empty=True, name='status'), 11 | 'result': ChoiceType(*[item[0] for item in AsyncTask.RESULT_OPTIONS], allow_empty=True, name='result'), 12 | 'offset': IntType(min=1, name='offset'), 13 | 'limit': IntType(min=0, name='limit'), 14 | 'id': ObjectType(AsyncTask, name='id') 15 | } 16 | 17 | @pre_handler(opt=["search", "status", "result", "offset", "limit"]) 18 | def getList(self): 19 | self.baseGetList(model=AsyncTask) 20 | 21 | @pre_handler(req=["id"]) 22 | def delete(self): 23 | obj = self.checked_params['id'] 24 | id, name = obj.id, obj.name 25 | obj.delete() 26 | self.message = f"To delete async task with id='{id}', name='{name}' succeeded." 27 | 28 | 29 | class AsyncTaskLogAPI(APIHandlerBase): 30 | fields_defination = { 31 | 'id': ObjectType(AsyncTask, real_query=False, name='id') 32 | } 33 | @pre_handler(req=["id"]) 34 | def getLog(self): 35 | id = self.checked_params['id'] 36 | log_file = os.path.join(ASYNC_TASK_LOGDIR, f'{ASYNC_TASK_LOGFILE_PREFIX}_{id}.log') 37 | self.data = [f'ERROR: Missing log file: {log_file}'] 38 | if os.path.isfile(log_file): 39 | with open(log_file) as f: 40 | self.data = f.readlines() 41 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 07:52 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ] 10 | 11 | operations = [ 12 | ] 13 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/migrations/0002_asynctask.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations, models 4 | import jsonfield.fields 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [ 12 | ('async_api', '0001_initial'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='AsyncTask', 18 | fields=[ 19 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 20 | ('uuid', models.CharField(default='', max_length=128, unique=True, verbose_name='UUID')), 21 | ('name', models.CharField(default='', max_length=128, verbose_name='任务名称')), 22 | ('status', models.IntegerField(choices=[(0, '等待执行'), (1, '正在执行'), (2, '执行完毕')], default=0, verbose_name='执行状态')), 23 | ('result', models.BooleanField(default=True, verbose_name='执行结果')), 24 | ('result_data', jsonfield.fields.JSONField(default={}, verbose_name='执行返回数据')), 25 | ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), 26 | ('finish_time', models.DateTimeField(null=True, verbose_name='完成时间')), 27 | ], 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/asynctask/async_api/migrations/__init__.py -------------------------------------------------------------------------------- /corelib/asynctask/async_api/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from jsonfield import JSONField 3 | 4 | 5 | class AsyncTask(models.Model): 6 | """ 7 | To record serialized async-tasks. 8 | """ 9 | STATUS_OPTIONS = ((0, "等待执行"), (1, "正在执行"), (2, "执行完毕")) 10 | 11 | # db fields. 12 | id = models.AutoField('ID', primary_key=True) 13 | uuid = models.CharField('UUID', max_length=128, unique=True, default='') 14 | name = models.CharField("任务名称", max_length=128, default='') 15 | status = models.IntegerField("执行状态", choices=STATUS_OPTIONS, default=0) 16 | result = models.BooleanField("执行结果", default=True) 17 | result_data = JSONField("执行返回数据", default={}) 18 | create_time = models.DateTimeField("创建时间", auto_now_add=True) 19 | finish_time = models.DateTimeField("完成时间", null=True) 20 | 21 | # serializing settings. 22 | list_fields = ["id", "uuid", "name", "status", "result", "result_data", "create_time", "finish_time"] 23 | detail_fields = list_fields 24 | search_fields = ["uuid", "name"] 25 | filter_fields = ["status", "result"] 26 | -------------------------------------------------------------------------------- /corelib/asynctask/async_api/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from .api import APIIngress 3 | 4 | urlpatterns = [ 5 | path('api/v1', APIIngress.as_view(), name="asynctask_api"), 6 | ] 7 | -------------------------------------------------------------------------------- /corelib/asynctask/bin/asynctask_server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | import django 6 | import re 7 | 8 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 9 | sys.path.append(BASE_DIR) 10 | 11 | manage_file = os.path.join(BASE_DIR, 'manage.py') 12 | if not os.path.isfile(manage_file): 13 | raise SystemExit(f'FATAL: It seems not a django project in directory: {BASE_DIR}') 14 | 15 | DJANGO_SETTING = None 16 | with open(manage_file) as f: 17 | for line in f: 18 | if re.search(r"os.environ.setdefault\('DJANGO_SETTINGS_MODULE'", line): 19 | DJANGO_SETTING = line.split("'")[3] 20 | break 21 | if not DJANGO_SETTING: 22 | raise SystemExit(f"FATAL: It seems not a django `manage.py`: {manage_file}") 23 | 24 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", DJANGO_SETTING) 25 | django.setup() 26 | 27 | from corelib.asynctask import MainServer 28 | 29 | server = MainServer() 30 | server.main() 31 | -------------------------------------------------------------------------------- /corelib/asynctask/lib/__init__.py: -------------------------------------------------------------------------------- 1 | from .async_task_client import ClientResultException, ClientRunningException, AsyncClient 2 | from .decorators import asynctask 3 | from .async_task_server import MainServer 4 | 5 | __all__ = ('ClientResultException', 'ClientRunningException', 'AsyncClient', 'asynctask', 'MainServer') 6 | -------------------------------------------------------------------------------- /corelib/asynctask/lib/async_task_client.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from tornado import gen, tcpclient 3 | from .defaults import ASYNCTASK_CLIENT_CONNECT_ADDR, ASYNCTASK_CLIENT_CONNECT_PORT 4 | import json 5 | 6 | HEADER_PREFIX = 'DataLength:' 7 | HEADER_LENTGH = 24 8 | 9 | 10 | class ClientResultException(Exception): 11 | def __init__(self, err=None): 12 | self.err = "ERROR: Result from AsyncTaskServer is not OK!" if err is None else err 13 | 14 | def __str__(self): 15 | return self.err 16 | 17 | 18 | class ClientRunningException(Exception): 19 | def __init__(self, err=None): 20 | self.err = "ERROR: Some error happened when running async client." if err is None else err 21 | 22 | def __str__(self): 23 | return f"{self.err}" 24 | 25 | 26 | class AsyncClient(object): 27 | def __init__(self, server_addr=None, server_port=None): 28 | self.server_addr = ASYNCTASK_CLIENT_CONNECT_ADDR if server_addr is None else server_addr 29 | self.server_port = ASYNCTASK_CLIENT_CONNECT_PORT if server_port is None else server_port 30 | 31 | def pack(self, msg): 32 | """ 33 | To transfer string msg to bytes with fixed 24Bytes Header. 34 | 35 | :msg A string to pack. 36 | 37 | Return a result tuple: '(None, err)' or '(, None)'. 38 | """ 39 | value_len_max = HEADER_LENTGH - len(HEADER_PREFIX) 40 | header_value = str(len(msg) + HEADER_LENTGH) 41 | value_len = len(header_value) 42 | if value_len > value_len_max: 43 | return None, "Socket Data Packing Error: Data is too big!" 44 | header_value = '0' * (value_len_max - value_len) + header_value 45 | 46 | return (HEADER_PREFIX + header_value + msg).encode('utf-8'), None 47 | 48 | async def go(self, uuid, name, module, tracking=False, delaytime=0, *args, **kwargs): 49 | """ 50 | To send serialized func data to AsyncTaskServer. 51 | 52 | :uuid Task uuid. 53 | :name function name. For dynamic import in AsyncTaskServer. 54 | :module python module the function in. For dynamic import in AsyncTaskServer. 55 | :tracking Record in backend database or Not. 56 | :delay To run this task after `delay` seconds in AsyncTaskServer. 57 | 58 | `args` and `kwargs` are parameters for task function. 59 | 60 | Return None or err. 61 | """ 62 | # To make bytes data. 63 | data = { 64 | 'uuid': uuid, 65 | 'name': name, 66 | 'module': module, 67 | 'tracking': tracking, 68 | 'delaytime': delaytime, 69 | 'args': args, 70 | 'kwargs': kwargs 71 | } 72 | data_str = json.dumps(data) 73 | data_bytes, err = self.pack(data_str) 74 | if err is not None: 75 | raise ClientRunningException(f'ERROR: Client data_bytes packing failed: {err}') 76 | 77 | # Send data and handle result. 78 | count = 0 79 | stream = None 80 | while count < 3: 81 | count += 1 82 | try: 83 | stream = await tcpclient.TCPClient().connect(self.server_addr, self.server_port) 84 | break 85 | except Exception: 86 | await gen.sleep(3) 87 | if stream is None: 88 | raise ClientRunningException(f"Failed to connect to server. server_addr: {self.server_addr}, server_port: {self.server_port}") 89 | await stream.write(data_bytes) 90 | ret = await stream.read_bytes(256, partial=True) 91 | ret_str = ret.decode('utf-8') 92 | if ret_str != 'OK': 93 | raise ClientResultException(ret_str) 94 | 95 | stream.close() 96 | 97 | def record(self, uuid, name): 98 | """ 99 | To record async task into backend database. 100 | 101 | :uuid task uuid. 102 | :name a task readable name. 103 | 104 | Return None or err. 105 | """ 106 | if 'corelib.asynctask.async_api' in settings.INSTALLED_APPS: 107 | from corelib.asynctask.async_api.models import AsyncTask 108 | try: 109 | _, created = AsyncTask.objects.get_or_create(uuid=uuid, name=name) 110 | except Exception as e: 111 | return str(e) 112 | else: 113 | if not created: 114 | return "UUID_EXIST" 115 | -------------------------------------------------------------------------------- /corelib/asynctask/lib/async_task_server.py: -------------------------------------------------------------------------------- 1 | from corelib.tools.logger import Logger 2 | from .defaults import ( 3 | ASYNCTASK_BIND_ADDR, 4 | ASYNCTASK_BIND_PORT, 5 | ASYNCTASK_WORKERS, 6 | ASYNCTASK_REGISTER_MODULE, 7 | ASYNCTASK_LOG_LEVEL 8 | ) 9 | 10 | from .async_task_client import HEADER_LENTGH 11 | import json 12 | from django.conf import settings 13 | from importlib import import_module 14 | from tornado import gen, ioloop, tcpserver, iostream 15 | from functools import partial 16 | 17 | 18 | class AsyncServer(tcpserver.TCPServer): 19 | def __init__(self, chunk_size=None, logger=None, *args, **kwargs) -> None: 20 | super().__init__(*args, **kwargs) 21 | self.logger = Logger(trigger_level=ASYNCTASK_LOG_LEVEL) if logger is None else logger 22 | self.chunk_size = 1024 if chunk_size is None else chunk_size 23 | self.funcs = {} 24 | 25 | async def handle_stream(self, stream, address): 26 | self.logger.msg_prefix = 'AsyncServer.handle_stream(): ' 27 | 28 | # handle TCP connetion. 29 | while True: 30 | try: 31 | self.logger.log("To receive data from clients...", level='DEBUG') 32 | data_b = await stream.read_bytes(self.chunk_size, partial=True) 33 | self.logger.log("To parsing client data header...", level='DEBUG') 34 | header = data_b[:HEADER_LENTGH] 35 | data_len = int(header.decode('utf-8').split(':')[1]) 36 | while len(data_b) < data_len: 37 | data_b += await stream.read_bytes(self.chunk_size) 38 | self.logger.log("To parsing client data body...", level='DEBUG') 39 | data = json.loads(data_b[HEADER_LENTGH:].decode('utf-8')) 40 | self.logger.log(f"New task received: {str(data)}", level="INFO") 41 | result = 'OK' 42 | await stream.write(result.encode('utf-8')) 43 | except iostream.StreamClosedError: # Normally stoped connection by client. 44 | self.logger.log(f"Returned result for client with value: '{result}'", level='DEBUG') 45 | break 46 | except Exception as e: 47 | self.logger.log(str(e), level="ERROR") 48 | result = 'ERROR' 49 | break 50 | 51 | # To do the real work. 52 | if result == 'OK': 53 | self.logger.log(f"Now to prepare to run the blocking task.", level='DEBUG') 54 | uuid, name, module = data['uuid'], data['name'], data['module'] 55 | tracking, delaytime = data['tracking'], data['delaytime'] 56 | args, kwargs = data['args'], data['kwargs'] 57 | 58 | index = f"{module}.{name}" 59 | func = self.funcs[index] 60 | 61 | # Delays. 62 | if isinstance(delaytime, int) and delaytime > 0: 63 | self.logger.log(f"Task '{uuid}' delayed in {delaytime} seconds.") 64 | await gen.sleep(func.delaytime) 65 | elif func.delaytime > 0: # Task delay. 66 | self.logger.log(f"Task '{uuid}' delayed in {func.delaytime} seconds.") 67 | await gen.sleep(func.delaytime) 68 | 69 | # To run the real blocking work in IOLoop. 70 | self.logger.log(f"Task '{uuid}' start to run: func={index}, tracking={tracking}, args={args}, kwargs={kwargs}") 71 | func = partial(func, **kwargs) 72 | await ioloop.IOLoop.current().run_in_executor(None, func, *args) 73 | self.logger.log(f"Task '{uuid}' complete.") 74 | 75 | 76 | class MainServer(object): 77 | def __init__( 78 | self, 79 | bind_addr: str = None, 80 | bind_port: int = None, 81 | subps: int = None, 82 | logger: Logger = None 83 | ) -> None: 84 | 85 | self.bind_addr = ASYNCTASK_BIND_ADDR if bind_addr is None else bind_addr 86 | self.bind_port = ASYNCTASK_BIND_PORT if bind_port is None else bind_port 87 | self.subps = ASYNCTASK_WORKERS if subps is None else subps 88 | self.logger = Logger(trigger_level=ASYNCTASK_LOG_LEVEL) if logger is None else logger 89 | self.funcs = {} 90 | 91 | def registerFunctions(self): 92 | self.logger.msg_prefix = 'AsyncServer.registerFunctions(): ' 93 | 94 | for app in filter(lambda s: not s.startswith('django.'), settings.INSTALLED_APPS): 95 | mod_str = f'{app}.{ASYNCTASK_REGISTER_MODULE}' 96 | try: 97 | mod = import_module(mod_str) 98 | except Exception as e: 99 | self.logger.log(f"Ignore invalid django-installed app: '{app}'.") 100 | self.logger.log(f"{str(e)}", level='DEBUG') 101 | else: 102 | for attr_name in filter(lambda a: not a.startswith('_'), dir(mod)): 103 | func = getattr(mod, attr_name) 104 | if getattr(func, 'is_asynctask', False): 105 | index = f'{mod_str}.{attr_name}' 106 | self.funcs[index] = func 107 | self.logger.log(f"To register asynctask funcion '{index}' succeeded.") 108 | 109 | def main(self): 110 | self.logger.msg_prefix = 'AsyncServer.main(): ' 111 | self.logger.log("Server initializing...") 112 | 113 | # To register asynctask functions. 114 | self.registerFunctions() 115 | if not self.funcs: 116 | return self.logger.log("No asynctask funcions registered. AsyncServer now quit.", level="FATAL") 117 | 118 | # To init tornado tcpserver. 119 | server = AsyncServer() 120 | server.funcs = self.funcs 121 | server.listen(self.bind_port, self.bind_addr) 122 | server.start(self.subps) # To fork process 123 | ioloop.IOLoop.current().start() 124 | -------------------------------------------------------------------------------- /corelib/asynctask/lib/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps, partial 2 | from .async_task_client import AsyncClient 3 | from corelib.tools.func_tools import genUUID 4 | from tornado import ioloop 5 | from tornado.platform.asyncio import AnyThreadEventLoopPolicy 6 | from django.conf import settings 7 | from django.utils import timezone 8 | from django.db import connections 9 | import asyncio 10 | 11 | 12 | def asynctask(func=None, tracking=False, delaytime=0, name=None): 13 | """ 14 | A decorator to register a function as an Async Task with optional parameters. 15 | 16 | :tracking If True, to record task running status and function returns in database, for some reason like task-reviewing purpose. 17 | Or Just run task without any database record if False. 18 | :delaytime Seconds how long a task passed to AsyncTaskServer will delay to run. 19 | :name A readable name for this task function. If not specified, `func.__name__` will be used. 20 | Only useful if `tracking` is True. 21 | """ 22 | if func is None: 23 | return partial(asynctask, tracking=tracking, delaytime=delaytime, name=name) 24 | 25 | def delay(*args, **kwargs): 26 | delaytime = kwargs.get('delaytime', 0) 27 | client = AsyncClient() 28 | err = 'UUID_EXIST' 29 | while err == 'UUID_EXIST': 30 | uuid = genUUID(64) 31 | if not tracking: 32 | err = None 33 | break 34 | _name = name if name else func.__name__ 35 | err = client.record(uuid, name=_name) 36 | if err is not None and err != 'UUID_EXIST': 37 | return err 38 | if tracking: 39 | kwargs['__async_task_uuid__'] = uuid 40 | kwargs['__async_func_name__'] = func.__name__ 41 | main = partial(client.go, uuid, func.__name__, func.__module__, tracking, delaytime, *args, **kwargs) 42 | asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) # To make ioloop runnable in any thread within Django. 43 | try: 44 | ioloop.IOLoop.current().run_sync(main) 45 | except Exception as e: 46 | return str(e) 47 | 48 | @wraps(func) 49 | def wrapper(*args, **kwargs): 50 | # To record. 51 | _to_record = False 52 | if tracking and 'corelib.asynctask.async_api' in settings.INSTALLED_APPS: 53 | _to_record = True 54 | uuid = kwargs.pop('__async_task_uuid__') 55 | func_name = kwargs.pop('__async_func_name__') 56 | from corelib.asynctask.async_api.models import AsyncTask 57 | obj = AsyncTask.objects.filter(uuid=uuid).first() 58 | obj.status = 1 59 | obj.save() 60 | 61 | try: 62 | # To do the real work. 63 | res = func(*args, **kwargs) 64 | 65 | except Exception as e: 66 | if _to_record: 67 | obj.result = False 68 | obj.result_data = { 69 | 'result': False, 70 | 'error_msg': f"ERROR: To run asynctask '{func_name}' failed. task uuid: '{uuid}'. {str(e)}", 71 | 'data': None 72 | } 73 | else: 74 | raise e 75 | else: 76 | if _to_record: 77 | obj.result = True 78 | obj.return_data = {'result': True, 'data': res} 79 | finally: 80 | if _to_record: 81 | obj.status = 2 82 | obj.finish_time = timezone.now() 83 | obj.save() 84 | 85 | # To close db connections the way like handling django request_finished. 86 | for conn in connections.all(): 87 | conn.close_if_unusable_or_obsolete() 88 | 89 | return res 90 | 91 | wrapper.delay = delay 92 | wrapper.is_asynctask = True 93 | wrapper.delaytime = delaytime 94 | return wrapper 95 | -------------------------------------------------------------------------------- /corelib/asynctask/lib/defaults.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | # default values 4 | _ASYNCTASK_BIND_ADDR = '127.0.0.1' # listen ip 5 | _ASYNCTASK_BIND_PORT = 11118 # listen port 6 | _ASYNCTASK_WORKERS = 0 # 0 means auto fetch the number from OS CPU cores. 7 | _ASYNCTASK_REGISTER_MODULE = 'asynctasks' # This means you should write all async task functions in 'asynctasks.py' in django app's basedir. 8 | _ASYNCTASK_LOG_LEVEL = 'INFO' 9 | 10 | # To user value in settings, or default value. 11 | ASYNCTASK_BIND_ADDR = getattr(settings, 'ASYNCTASK_BIND_ADDR', _ASYNCTASK_BIND_ADDR) 12 | ASYNCTASK_BIND_PORT = getattr(settings, 'ASYNCTASK_BIND_PORT', _ASYNCTASK_BIND_PORT) 13 | ASYNCTASK_CLIENT_CONNECT_ADDR = getattr(settings, 'ASYNCTASK_CLIENT_CONNECT_ADDR', _ASYNCTASK_BIND_ADDR) 14 | ASYNCTASK_CLIENT_CONNECT_PORT = getattr(settings, 'ASYNCTASK_CLIENT_CONNECT_PORT', _ASYNCTASK_BIND_PORT) 15 | ASYNCTASK_WORKERS = getattr(settings, 'ASYNCTASK_WORKERS', _ASYNCTASK_WORKERS) 16 | ASYNCTASK_REGISTER_MODULE = getattr(settings, 'ASYNCTASK_REGISTER_MODULE', _ASYNCTASK_REGISTER_MODULE) 17 | ASYNCTASK_LOG_LEVEL = getattr(settings, 'ASYNCTASK_LOG_LEVEL', _ASYNCTASK_LOG_LEVEL) 18 | -------------------------------------------------------------------------------- /corelib/permission/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/permission/__init__.py -------------------------------------------------------------------------------- /corelib/permission/api.py: -------------------------------------------------------------------------------- 1 | from corelib import APIIngressBase 2 | from .handlers import PermissionGet, PermissionSet 3 | 4 | 5 | class APIIngress(APIIngressBase): 6 | actions = { 7 | 'getUserList': PermissionGet, 8 | 'getUserDetail': PermissionGet, 9 | 'getPermGroups': PermissionGet, 10 | 'getMyPerm': PermissionGet, 11 | 'setUserPerm': PermissionSet, 12 | } 13 | -------------------------------------------------------------------------------- /corelib/permission/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from .defaults import ( 3 | PERMISSION_GROUPS, CUSTOM_PERMISSION_MODEL, DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED, 4 | DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED, DEFAULT_USER_PASSWORD) 5 | from .tools import get_model 6 | 7 | from corelib.api_base.defaults import ACTION_AUTH_REQUIRED 8 | from django.contrib.auth.models import User 9 | 10 | 11 | def permissionChecker(perm=None): 12 | """ 13 | Can only be used for API action handlers. 14 | This decorator is used to check whether user has permission to visit this API or not. 15 | This decorator needs 'corelib.permission' to be installed as a django app. 16 | 17 | Params: 18 | perm A string define in `PERMISSION_GROUPS` setting. 19 | """ 20 | def decorator(func): 21 | @wraps(func) 22 | def checker(self, *args, **kwargs): 23 | if perm is not None and not self.by_pass_perm_check: 24 | # To get permission model. 25 | model = get_model() 26 | if not model: 27 | return self.error(f"ERROR: `CUSTOM_PERMISSION_MODEL` setting '{CUSTOM_PERMISSION_MODEL}' is not valid.", http_status=500) 28 | 29 | error_msg = f"ERROR: You are not allowed to call this action API '{self.action}'!" 30 | 31 | # To get user perm object. 32 | if not ACTION_AUTH_REQUIRED and self.request.user.is_anonymous: 33 | # For developing mode. 34 | user_defaults = { 35 | 'email': 'fake@example.lei', 36 | 'password': DEFAULT_USER_PASSWORD, 37 | } 38 | user, _ = User.objects.get_or_create(username=DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED, defaults=user_defaults) 39 | perm_defaults = { 40 | 'perm_group': DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED, 41 | } 42 | user_perm, _ = model.objects.get_or_create(user=user, defaults=perm_defaults) 43 | else: 44 | # Online mode. 45 | user_perm = model.objects.filter(user=self.request.user).first() 46 | if user_perm is None: 47 | return self.error(f"{error_msg} User_perm: None!", http_status=403) 48 | 49 | # To check user's permission 50 | self.user_perm = user_perm 51 | if PERMISSION_GROUPS[user_perm.perm_group] < PERMISSION_GROUPS[perm]: 52 | return self.error(error_msg, http_status=403) 53 | 54 | # To do action. 55 | func_result = func(self, *args, **kwargs) 56 | return func_result 57 | return checker 58 | return decorator 59 | -------------------------------------------------------------------------------- /corelib/permission/defaults.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from django.conf import settings 3 | 4 | 5 | # default permission groups. 6 | _PERMISSION_GROUPS = OrderedDict({ 7 | "normal": 1, 8 | "admin": 2, 9 | }) 10 | 11 | # This user and perm will be used for developing, when `ACTION_AUTH_REQUIRED` is set to 'False'. 12 | _DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED = 'superdeveloper' 13 | _DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED = 'admin' 14 | _DEFAULT_USER_PASSWORD = 'qwer!1234' 15 | 16 | # Tuple: (, ) 17 | _CUSTOM_PERMISSION_MODEL = None 18 | 19 | # Vars that really works. 20 | PERMISSION_GROUPS = getattr(settings, 'PERMISSION_GROUPS', _PERMISSION_GROUPS) 21 | CUSTOM_PERMISSION_MODEL = getattr(settings, 'CUSTOM_PERMISSION_MODEL', _CUSTOM_PERMISSION_MODEL) 22 | DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED = getattr(settings, 'DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED', _DEFAULT_USER_WHEN_AUTH_NOT_REQUIRED) 23 | DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED = getattr(settings, 'DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED', _DEFAULT_PERM_WHEN_AUTH_NOT_REQUIRED) 24 | DEFAULT_USER_PASSWORD = getattr(settings, 'DEFAULT_USER_PASSWORD', _DEFAULT_USER_PASSWORD) 25 | -------------------------------------------------------------------------------- /corelib/permission/handlers.py: -------------------------------------------------------------------------------- 1 | from corelib import APIHandlerBase, pre_handler, ObjectType, ChoiceType, StrType, IntType 2 | from corelib.api_serializing_mixins.get_list_data_mixin import ListDataMixin 3 | from corelib.api_serializing_mixins.get_detail_data_mixin import DetailDataMixin 4 | from corelib.api_serializing_mixins.modify_data_mixin import ModifyDataMixin 5 | from .defaults import PERMISSION_GROUPS 6 | from .tools import get_model 7 | 8 | 9 | class PermissionGet(APIHandlerBase, ListDataMixin, DetailDataMixin): 10 | post_fields = { 11 | "id": ObjectType(get_model()), 12 | "search": StrType(), 13 | "perm_group": ChoiceType(*PERMISSION_GROUPS.keys(), allow_empty=True), 14 | "page_index": IntType(min=1), 15 | "page_length": IntType(min=0), 16 | } 17 | 18 | @pre_handler(opt=["search", "perm_group", "page_index", "page_length"], perm='admin') 19 | def getUserList(self): 20 | self.getList(model=get_model()) 21 | 22 | @pre_handler(req=['id'], perm='admin') 23 | def getUserDetail(self): 24 | self.getDetail(model=get_model()) 25 | 26 | @pre_handler(perm='admin') 27 | def getPermGroups(self): 28 | self.data = list(PERMISSION_GROUPS.keys()) 29 | 30 | @pre_handler(perm='normal') 31 | def getMyPerm(self): 32 | self.data = {"perm_group": self.user_perm.perm_group, "username": self.user_perm.user.username} 33 | 34 | 35 | class PermissionSet(APIHandlerBase, ModifyDataMixin): 36 | post_fields = { 37 | "id": ObjectType(get_model()), 38 | "perm_group": ChoiceType(*PERMISSION_GROUPS.keys()), 39 | } 40 | @pre_handler(req=["id", "perm_group"], perm='admin', record=True) 41 | def setUserPerm(self): 42 | self.modifyData() 43 | -------------------------------------------------------------------------------- /corelib/permission/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ] 10 | 11 | operations = [ 12 | ] 13 | -------------------------------------------------------------------------------- /corelib/permission/migrations/0002_apipermission.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ('permission', '0001_initial'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='APIPermission', 20 | fields=[ 21 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 22 | ('realname', models.CharField(default='', max_length=64, verbose_name='中文姓名')), 23 | ('perm_group', models.CharField(default='', max_length=64, verbose_name='权限属组')), 24 | ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='api_perm', to=settings.AUTH_USER_MODEL)), 25 | ], 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /corelib/permission/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/permission/migrations/__init__.py -------------------------------------------------------------------------------- /corelib/permission/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.contrib.auth.models import User 3 | 4 | 5 | class APIPermission(models.Model): 6 | """ 7 | Based on django User. 8 | """ 9 | 10 | # db fields. 11 | id = models.AutoField('ID', primary_key=True) 12 | user = models.OneToOneField(User, on_delete=models.CASCADE, related_name="api_perm", unique=True) 13 | realname = models.CharField("中文姓名", max_length=64, default='') 14 | perm_group = models.CharField("权限属组", max_length=64, default='') 15 | 16 | # serializing settings. 17 | list_fields = ["id", "perm_group", {"user": ["username"]}, "realname"] 18 | detail_fields = list_fields 19 | search_fields = ["user.username"] 20 | filter_fields = ["perm_group"] 21 | -------------------------------------------------------------------------------- /corelib/permission/tools.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from .defaults import CUSTOM_PERMISSION_MODEL 3 | from .models import APIPermission 4 | 5 | 6 | def get_model(): 7 | if not CUSTOM_PERMISSION_MODEL: 8 | return APIPermission 9 | # print(CUSTOM_PERMISSION_MODEL) 10 | module_path = CUSTOM_PERMISSION_MODEL[0] 11 | model_name = CUSTOM_PERMISSION_MODEL[1] 12 | try: 13 | module = import_module(module_path) 14 | except Exception: 15 | return None 16 | 17 | return getattr(module, model_name, None) 18 | -------------------------------------------------------------------------------- /corelib/permission/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from .api import APIIngress 3 | 4 | urlpatterns = [ 5 | path('api/v1', APIIngress.as_view(), name="permission_api"), 6 | ] 7 | -------------------------------------------------------------------------------- /corelib/recorder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/recorder/__init__.py -------------------------------------------------------------------------------- /corelib/recorder/api.py: -------------------------------------------------------------------------------- 1 | from corelib import APIIngressBase 2 | from .handlers import APICallingRecordHandler 3 | 4 | 5 | class APIIngress(APIIngressBase): 6 | actions = { 7 | 'getRecordList': APICallingRecordHandler, 8 | } 9 | -------------------------------------------------------------------------------- /corelib/recorder/decorators.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from django.utils import timezone 3 | 4 | 5 | def recorder(record_label=None): 6 | """ 7 | Can only be used for API action handlers. 8 | This decorator is used to record api callings that need to be record. users permission. 9 | This decorator needs 'corelib.recorder' to be installed as a django app. 10 | """ 11 | def decorator(func): 12 | @wraps(func) 13 | def recording(self, *args, **kwargs): 14 | from corelib.recorder.models import APICallingRecord 15 | # To record this call. 16 | username = '' 17 | if self.auth_token: 18 | token_user = self.auth_token.split('.')[0] 19 | username = f"[TOKEN_USER]{token_user}" 20 | elif self.request and not self.request.user.is_anonymous: 21 | username = self.request.user.username 22 | if self.by_pass_bind_username: 23 | username = self.by_pass_bind_username 24 | 25 | record_data = { 26 | "username": username, 27 | "api": self.request.path, 28 | "action": self.action, 29 | "action_label": record_label if record_label else '', 30 | "post_data": self.params 31 | } 32 | record = APICallingRecord.objects.create(**record_data) 33 | 34 | # To do the real work. 35 | func_result = func(self, *args, **kwargs) 36 | 37 | # To record result of this call after doing real work. 38 | record.result = "SUCCESS" if self.result else "FAILED" 39 | record.message = self.message 40 | record.error_message = self.error_message 41 | record.finish_time = timezone.now() 42 | record.save() 43 | 44 | return func_result 45 | return recording 46 | return decorator 47 | -------------------------------------------------------------------------------- /corelib/recorder/handlers.py: -------------------------------------------------------------------------------- 1 | from corelib import APIHandlerBase, pre_handler, ChoiceType, StrType, IntType 2 | from corelib.api_serializing_mixins.get_list_data_mixin import ListDataMixin 3 | from .models import APICallingRecord 4 | 5 | 6 | class APICallingRecordHandler(APIHandlerBase, ListDataMixin): 7 | post_fields = { 8 | "search": StrType(), 9 | "result": ChoiceType("SUCCESS", "FAILED"), 10 | 'page_index': IntType(min=1), 11 | 'page_length': IntType(min=0), 12 | } 13 | 14 | @pre_handler(opt=["search", "result", "page_index", "page_length"], perm="admin") 15 | def getRecordList(self): 16 | self.getList(model=APICallingRecord) 17 | -------------------------------------------------------------------------------- /corelib/recorder/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ] 10 | 11 | operations = [ 12 | ] 13 | -------------------------------------------------------------------------------- /corelib/recorder/migrations/0002_apicallingrecord.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations, models 4 | import django.utils.timezone 5 | import jsonfield.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ('recorder', '0001_initial'), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='APICallingRecord', 19 | fields=[ 20 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 21 | ('username', models.CharField(default='', max_length=64, verbose_name='用户')), 22 | ('api', models.CharField(default='', max_length=128, verbose_name='调用API')), 23 | ('action', models.CharField(default='', max_length=64, verbose_name='执行动作')), 24 | ('action_label', models.CharField(default='', max_length=128, verbose_name='执行动作label')), 25 | ('post_data', jsonfield.fields.JSONField(default={}, verbose_name='操作内容')), 26 | ('result', models.CharField(default='', max_length=16, verbose_name='操作结果')), 27 | ('message', models.TextField(default='', verbose_name='成功消息')), 28 | ('error_message', models.TextField(default='', verbose_name='失败消息')), 29 | ('operating_time', models.DateTimeField(default=django.utils.timezone.now, verbose_name='操作时间')), 30 | ], 31 | options={ 32 | 'ordering': ['-id'], 33 | }, 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /corelib/recorder/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/recorder/migrations/__init__.py -------------------------------------------------------------------------------- /corelib/recorder/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from jsonfield import JSONField 3 | from django.utils import timezone 4 | 5 | 6 | class APICallingRecord(models.Model): 7 | # db fields. 8 | id = models.AutoField('ID', primary_key=True) 9 | username = models.CharField("用户", max_length=64, default="") 10 | api = models.CharField("调用API", max_length=128, default="") 11 | action = models.CharField("执行动作", max_length=64, default="") 12 | action_label = models.CharField("执行动作label", max_length=128, default="") 13 | post_data = JSONField("操作内容", default={}) 14 | result = models.CharField("操作结果", max_length=16, default="") # "SUCCESS" or "FAILED" 15 | message = models.TextField("成功消息", default="") 16 | error_message = models.TextField("失败消息", default="") 17 | operating_time = models.DateTimeField("操作时间", default=timezone.now) 18 | 19 | # serializing field.s 20 | list_fields = ["id", "username", "api", "action", "action_label", "post_data", "result", "message", "error_message", "operating_time"] 21 | detail_fields = list_fields 22 | search_fields = ["username", "api", "action", "action_label", "post_data", "message", "error_message"] 23 | filter_fields = ["result"] 24 | 25 | class Meta: 26 | ordering = ['-id'] 27 | -------------------------------------------------------------------------------- /corelib/recorder/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from .api import APIIngress 3 | 4 | urlpatterns = [ 5 | path('api/v1', APIIngress.as_view(), name="recorder_api"), 6 | ] 7 | -------------------------------------------------------------------------------- /corelib/timer/__init__.py: -------------------------------------------------------------------------------- 1 | from .lib.service import Timer 2 | from .lib.decorator import cron 3 | 4 | __all__ = ("Timer", "cron") 5 | -------------------------------------------------------------------------------- /corelib/timer/bin/timer_server: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | import sys 4 | import os 5 | import django 6 | import re 7 | 8 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 9 | sys.path.append(BASE_DIR) 10 | 11 | manage_file = os.path.join(BASE_DIR, 'manage.py') 12 | if not os.path.isfile(manage_file): 13 | raise SystemExit(f'FATAL: It seems not a django project in directory: {BASE_DIR}') 14 | 15 | DJANGO_SETTING = None 16 | with open(manage_file) as f: 17 | for line in f: 18 | if re.search(r"os.environ.setdefault\('DJANGO_SETTINGS_MODULE'", line): 19 | DJANGO_SETTING = line.split("'")[3] 20 | break 21 | if not DJANGO_SETTING: 22 | raise SystemExit(f"FATAL: It seems not a django `manage.py`: {manage_file}") 23 | 24 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", DJANGO_SETTING) 25 | django.setup() 26 | 27 | from corelib.timer import Timer 28 | from corelib.timer.lib.defaults import TIMER_WORKER_MOD 29 | 30 | if __name__ == '__main__': 31 | server = Timer(TIMER_WORKER_MOD) 32 | server.start() 33 | -------------------------------------------------------------------------------- /corelib/timer/lib/__init__.py: -------------------------------------------------------------------------------- 1 | from .decorator import cron 2 | from .service import Timer 3 | 4 | __all__ = ('cron', 'Timer') 5 | -------------------------------------------------------------------------------- /corelib/timer/lib/decorator.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | from django.utils import timezone 3 | from functools import wraps 4 | 5 | 6 | # 定时任务装饰器。 7 | def cron(every=None, crontab=None, enabled=True, dynamic=False): 8 | """ 9 | 一个cron装饰器，用于在timer中执行定时任务。 10 | 内置了self参数，只能作用于Timer与其子类的实例方法。 11 | 12 | 参数说明： 13 | every 大于0的秒数。提供数值后，timer启动时运行一次，以后每间隔every秒数再次执行。 14 | 15 | crontab 一个crontab字符串，六个字段，空格符为间隔。依次为：秒，分，时，日，月，周。 16 | 类似linux的crontab，支持'*'写法表示任意值；支持类似*/5'的间隔写法，间隔数需是正整数。 17 | 18 | enabled 用于直接设置是否启用此定时任务。（对于静态任务，修改之后需要重启timer服务。） 19 | 20 | dynamic 用于标记当前定时任务为动态定时任务。 21 | 标记为`dynamic`时，其他参数可不填。需要通过API或者相关Handler在数据库中做运行参数配置。 22 | 23 | 注意：every与crontab同时提供时，以every为准。静态任务不支持at_time. 24 | """ 25 | def decorator(func): 26 | # 设置工作函数内置属性 27 | func._is_a_timer_task = True 28 | func._timer_every = every 29 | func._timer_crontab = crontab 30 | func._timer_enabled = enabled 31 | func._is_dynamic = dynamic 32 | 33 | @wraps(func) 34 | def runInCron(self, name, *args, **kwargs): 35 | # 标记在运行 36 | if func._is_dynamic: 37 | self.dynamic_state[name]['running'] = True 38 | else: 39 | self.cron_state[name]['running'] = True 40 | 41 | # 开始执行 42 | self.logger.log(f"Task '{name}' start to run...") 43 | start_at = timezone.now() 44 | try: 45 | func(*args, **kwargs) 46 | except Exception: 47 | traceback.print_exc() 48 | result = 'failed' 49 | else: 50 | result = 'success' 51 | end_at = timezone.now() 52 | 53 | # 执行结果数据统计 54 | time_spend = round(end_at.timestamp() - start_at.timestamp(), 3) 55 | if func._is_dynamic and self.enable_dynamic: 56 | from corelib.timer.timer_api.models import CronJob 57 | self.dynamic_tasks[name]['total_run_count'] += 1 58 | _attrs = { 59 | 'last_run_start_at': start_at, 60 | 'last_run_spend_time': time_spend, 61 | 'last_run_result': result, 62 | 'total_run_count': self.dynamic_tasks[name]['total_run_count'] 63 | } 64 | CronJob.objects.filter(name=name).update(**_attrs) 65 | self.logger.log(f'dynamic task result updated: {_attrs}', level='DEBUG') 66 | self.logger.log(f"Task '{name}' finished. time_spend: {time_spend}s, result: {result}") 67 | 68 | # 标记已停止 69 | if func._is_dynamic: 70 | self.dynamic_state[name]['running'] = False 71 | else: 72 | self.cron_state[name]['running'] = False 73 | return result 74 | 75 | return runInCron 76 | return decorator 77 | -------------------------------------------------------------------------------- /corelib/timer/lib/defaults.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | """ 4 | 配置参数说明： 5 | 6 | TIMER_REGISTER_MODULE 定时任务注册模块，默认为'timer', 即，some_django_app/timer.py 7 | TIMER_LOG_LEVEL timer_server的日志级别，默认为'INFO', 可选值['DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL'] 8 | TIMER_WORKER_MOD timer_server的工作模式，默认为'thread'，表示以多线程方式执行各个定时任务。（不适合计算密集型的任务） 9 | 也可以是'process'，表示以子进程的方式执行各个并发任务。（可以用作计算密集型的任务） 10 | 11 | DYNAMIC_UPDATING_INTERVAL 动态任务同步数据库配置的周期，默认是60，表是一分钟检查一次动态数据的更新，不能小于60。 12 | """ 13 | 14 | _TIMER_REGISTER_MODULE = 'timer' # 要求在个django app下编写timer.py模块 15 | _TIMER_LOG_LEVEL = 'INFO' 16 | _TIMER_WORKER_MOD = 'thread' 17 | _DYNAMIC_UPDATING_INTERVAL = 60 18 | 19 | TIMER_REGISTER_MODULE = getattr(settings, 'TIMER_REGISTER_MODULE', _TIMER_REGISTER_MODULE) 20 | TIMER_LOG_LEVEL = getattr(settings, 'TIMER_LOG_LEVEL', _TIMER_LOG_LEVEL) 21 | TIMER_WORKER_MOD = getattr(settings, 'TIMER_WORKER_MOD', _TIMER_WORKER_MOD) 22 | DYNAMIC_UPDATING_INTERVAL = getattr(settings, 'DYNAMIC_UPDATING_INTERVAL', _DYNAMIC_UPDATING_INTERVAL) 23 | -------------------------------------------------------------------------------- /corelib/timer/lib/service.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | from multiprocessing import Process, Manager 4 | from threading import Thread 5 | from django.conf import settings 6 | from django.utils import timezone 7 | from importlib import import_module 8 | from .defaults import TIMER_LOG_LEVEL, TIMER_REGISTER_MODULE, DYNAMIC_UPDATING_INTERVAL 9 | from corelib.tools.logger import Logger 10 | 11 | 12 | class Timer(object): 13 | """ 14 | 定时任务分为两类： 15 | - 静态任务：配置数据无需入库，直接通过cron装饰器加载到timer进程中执行。 16 | - 动态任务：配置数据记录在数据库（此表中），需要在数据库中设定运行参数后，才能运行。 17 | 考虑到安全因素，不支持自定义脚本的上传执行。所有定时任务，都需系统开发人员编码。 18 | 19 | 静态任务： 20 | 相当于是一种简化的动态定时任务。无需入库，直接在编码时写死。 21 | 所以更加简单、易用，但缺乏灵活性，不可由用户来调度，也不可传参。 22 | 故，在定义静态任务时，不可为任务函数定义任何参数。 23 | 24 | 动态任务： 25 | 在timer服务启动时，仅做模块儿加载入库，不会直接开始运行。 26 | 需要在数据库中配置运行参数，才能开始执行。 27 | timer进程同步数据库中任务参数的时间间隔最小为1分钟（可做全局配置，不可小于1分钟）。 28 | 这即表示用户修改了数据库之后，会在1分钟内生效，而不是及时生效。 29 | 紧急情况下，要想及时生效，可以重启timer服务进程来实现。 30 | 31 | timer服务进程： 32 | timer将会并发地执行各个定时任务函数。支持'thread'与'process'两种并发模式。可在class初始化时做设定。 33 | 'thread'模式，适合IO等待型的任务处理。但，不适合高负荷的计算密集型。是Timer的默认工作模式。 34 | 'process'模式，可用于高负荷的计算密集型。但，比较重，最好单独找服务器独立部署。 35 | 36 | 另外，考虑到性能雪崩效应，timer中的任务，都不允许交叠执行。即同一个任务，上一次执行周期的任务执行还未结束时，则本次执行周期不执行。 37 | 故，对于需要重复执行的任务，请合理评估最小执行周期，否则某些执行周期会失效。 38 | """ 39 | 40 | def __init__(self, worker=None): 41 | """ 42 | :worker 合法值：'thread', 'process'，默认为'thread'； 43 | 'thread': 以multi-thread方式执行每个定时任务函数。 44 | 'process': 以multi-process方式执行每个定时任务函数。 45 | 46 | 当以'process'方式运行时，默认会起两个进程，一个父进程，一个manager子进程。manager子进程用于各工作子进程之间共享全局变量：`self.cron_state`. 47 | """ 48 | self.cron_pool = {} 49 | self.cron_state = {} 50 | self.dynamic_tasks = {} 51 | self.dynamic_state = {} 52 | self.cron_origin_start_time = None 53 | self.worker = 'thread' if worker is None else worker 54 | self.logger = Logger(trigger_level=TIMER_LOG_LEVEL, msg_prefix='Timer: ') 55 | if worker not in {'process', 'thread'}: 56 | raise SystemExit("ERROR: `worker` must be 'process' or 'thread'") 57 | self.is_process_mod = False 58 | if self.worker == 'process': 59 | self.is_process_mod = True 60 | self.manager = Manager() 61 | self.cron_state = self.manager.dict() 62 | self.dynamic_state = self.manager.dict() 63 | self.dynamic_tasks = self.manager.dict() 64 | 65 | self.enable_dynamic = False 66 | if 'corelib.timer.timer_api' in settings.INSTALLED_APPS: 67 | self.enable_dynamic = True 68 | 69 | def schedule_task(self, func, dynamic_task=None): 70 | """ 71 | 判定当前时间点，是否需要执行指定的定时任务。 72 | """ 73 | # 开始评估执行周期 74 | name = dynamic_task['name'] if dynamic_task else f'{func.__module__}.{func.__name__}' 75 | to_run = False 76 | every = dynamic_task['every'] if dynamic_task else func._timer_every 77 | crontab = dynamic_task['crontab'] if dynamic_task else func._timer_crontab 78 | at_time = dynamic_task['at_time'] if dynamic_task else None 79 | 80 | # 根据timer服务启动时间，计算every相对执行时间点。 81 | if every: 82 | if int(time.time() - self.cron_origin_start_time) % every == 0: 83 | to_run = True 84 | 85 | # 根据系统绝对时间，评估crontab字符串是否匹配。 86 | elif crontab: 87 | _crontab = crontab.split() 88 | if len(_crontab) == 7: 89 | now = timezone.now() 90 | now_str = now.strftime("%S %M %H %d %m %W %Y") 91 | now_list = [int(t) for t in now_str.split()] 92 | match_count = 0 93 | for i in range(7): 94 | if _crontab[i] == '*': 95 | match_count += 1 96 | elif re.search(r'^\*/[1-9][0-9]*$', _crontab[i]): 97 | _every = int(_crontab[i].split('/')[1]) 98 | if now_list[i] % _every == 0: 99 | match_count += 1 100 | else: 101 | try: 102 | num = int(_crontab[i]) 103 | except Exception: 104 | self.logger.log(f"Invalid crontab string: '{crontab}'. Crontab task '{name}' unable to run! ", level='ERROR') 105 | break 106 | 107 | if num == now_list[i]: 108 | match_count += 1 109 | 110 | # 每个字段都匹配成功，放可执行crontab任务。 111 | if match_count == 7: 112 | to_run = True 113 | 114 | # 根据系统绝对时间评估at_time单次任务。 115 | elif at_time: 116 | if not self.dynamic_state[name]['running'] and timezone.now() >= at_time: 117 | to_run = True 118 | 119 | # 不允许交叠执行。 120 | if to_run: 121 | _running = self.dynamic_state[name]['running'] if dynamic_task else self.cron_state[name]['running'] 122 | if _running: 123 | self.logger.log(f"Task '{name}' is still running, ignore it this time.", level='WARNING') 124 | return False 125 | 126 | return to_run 127 | 128 | def register(self): 129 | ''' 130 | 注册timer定时任务 131 | ''' 132 | if self.enable_dynamic: 133 | from corelib.timer.timer_api.models import AvailableTasks 134 | 135 | for app in filter(lambda s: not s.startswith('django.'), settings.INSTALLED_APPS): 136 | mod_str = f'{app}.{TIMER_REGISTER_MODULE}' 137 | try: 138 | mod = import_module(mod_str) 139 | except Exception as e: 140 | self.logger.log(f"Ignore invalid django-installed app: '{app}'.") 141 | self.logger.log(f"{str(e)}", level='DEBUG') 142 | else: 143 | for attr_name in filter(lambda a: not a.startswith('_'), dir(mod)): 144 | func = getattr(mod, attr_name) 145 | index = f'{mod_str}.{attr_name}' 146 | if getattr(func, '_is_a_timer_task', False): 147 | self.cron_pool[index] = func 148 | 149 | # 动态任务的注册/更新 150 | if self.enable_dynamic and getattr(func, '_is_dynamic', False): 151 | attrs = { 152 | 'module_path': index, 153 | 'usage_doc': '' if func.__doc__ is None else func.__doc__, 154 | } 155 | AvailableTasks.objects.update_or_create(**attrs) 156 | _type = 'dynamic' 157 | 158 | # 静态任务初始化运行状态 159 | else: 160 | _state = {"running": False} 161 | self.cron_state[index] = self.manager.dict(_state) if self.is_process_mod else _state 162 | _type = 'static' 163 | 164 | self.logger.log(f"To register {_type} task '{index}' succeeded.") 165 | 166 | def update_dynamic_tasks(self): 167 | """ 168 | 读取数据库配置，更新全局属性`dynamic_tasks`与`dynamic_state` 169 | """ 170 | self.logger.log('Try to update dynamic tasks from DB', 'DEBUG') 171 | from corelib.timer.timer_api.models import CronJob 172 | 173 | self.dynamic_tasks = self.manager.dict() if self.is_process_mod else {} 174 | qs = CronJob.objects.filter(enabled=1) 175 | for obj in qs: 176 | name = obj.name 177 | self.logger.log(f"Updating dynamic task '{name}' from DB.", 'DEBUG') 178 | # 加载定时任务 179 | _attr = { 180 | 'name': name, 181 | 'task': obj.task, 182 | 'args': obj.args, 183 | 'kwargs': obj.kwargs, 184 | 'every': obj.every, 185 | 'crontab': obj.crontab, 186 | 'at_time': obj.at_time, 187 | 'expired_time': obj.expired_time, 188 | 'expired_count': obj.expired_count, 189 | 'total_run_count': obj.total_run_count 190 | } 191 | self.dynamic_tasks[name] = self.manager.dict(_attr) if self.is_process_mod else _attr 192 | 193 | # 初始化state 194 | if name not in self.dynamic_state: 195 | _state = {'running': False} 196 | self.dynamic_state[name] = self.manager.dict(_state) if self.is_process_mod else _state 197 | self.logger.log('To update dynamic tasks from DB succeeded!', 'DEBUG') 198 | 199 | def is_expired(self, name): 200 | ''' 201 | 针对动态任务，根据配置，将已失效的任务自动设置为失效状态。 202 | ''' 203 | from corelib.timer.timer_api.models import CronJob 204 | 205 | expired_time = self.dynamic_tasks[name]['expired_time'] 206 | expired_count = self.dynamic_tasks[name]['expired_count'] 207 | total_run_count = self.dynamic_tasks[name]['total_run_count'] 208 | every = self.dynamic_tasks[name]['every'] 209 | crontab = self.dynamic_tasks[name]['crontab'] 210 | at_time = self.dynamic_tasks[name]['at_time'] 211 | 212 | if expired_time and expired_time <= timezone.now(): 213 | CronJob.objects.filter(name=name).update(enabled=0) 214 | self.logger.log(f"Dynamic task '{name}' has expired!") 215 | return True 216 | if expired_count and total_run_count >= expired_count: 217 | CronJob.objects.filter(name=name).update(enabled=0) 218 | self.logger.log(f"Dynamic task '{name}' has expired, according to count limit '{expired_count}'!") 219 | return True 220 | if not (every or crontab): 221 | if not at_time: 222 | CronJob.objects.filter(name=name).update(enabled=0) 223 | self.logger.log(f"Task '{name}' with no running way set is now be disabled!") 224 | return True 225 | if total_run_count >= 1: 226 | CronJob.objects.filter(name=name).update(enabled=0) 227 | self.logger.log(f"At time task '{name}' finished and will not run any more!") 228 | return True 229 | return False 230 | 231 | def run_static_tasks(self): 232 | ''' 233 | 并发执行静态任务 234 | ''' 235 | for task in self.cron_pool: 236 | func = self.cron_pool[task] 237 | if not func._timer_enabled: 238 | continue 239 | args = [self, f'{func.__module__}.{func.__name__}'] 240 | if self.schedule_task(func): 241 | if self.worker == 'thread': 242 | Thread(target=func, args=args).start() 243 | elif self.worker == 'process': 244 | Process(target=func, args=args).start() 245 | 246 | def run_dynamic_tasks(self): 247 | ''' 248 | 并发执行动态任务 249 | ''' 250 | expired_names = [] 251 | for name, task in self.dynamic_tasks.items(): 252 | # 先做过期检查 253 | if self.is_expired(name): 254 | expired_names.append(name) 255 | continue 256 | 257 | # 然后执行cron调度 258 | func = self.cron_pool[task['task']] 259 | if self.schedule_task(func, task): 260 | args = [self, name] 261 | for arg in task['args']: 262 | args.append(arg) 263 | if self.worker == 'thread': 264 | Thread(target=func, args=args, kwargs=task['kwargs']).start() 265 | elif self.worker == 'process': 266 | Process(target=func, args=args, kwargs=task['kwargs']).start() 267 | 268 | # 最后，清理过期的动态任务 269 | for name in expired_names: 270 | self.dynamic_tasks.pop(name) 271 | self.dynamic_state.pop(name) 272 | 273 | def start(self): 274 | """ 275 | timer服务启动函数 276 | """ 277 | # 注册timer工作函数 278 | self.logger.log('Initiallizing timer...') 279 | self.register() 280 | 281 | # 动态任务，首次更新 282 | if self.enable_dynamic: 283 | self.update_dynamic_tasks() 284 | 285 | # 启动主循环 286 | dynamic_update_internal = 60 if DYNAMIC_UPDATING_INTERVAL < 60 else DYNAMIC_UPDATING_INTERVAL 287 | self.cron_origin_start_time = time.time() 288 | time.sleep(1) # 延迟一秒，错开`cron_origin_start_time` 289 | last_time = int(self.cron_origin_start_time) 290 | while True: 291 | current_time = int(time.time()) 292 | if current_time > last_time: 293 | self.logger.log('timer start to check.', level='DEBUG') 294 | if self.enable_dynamic and current_time % dynamic_update_internal == 0: 295 | self.update_dynamic_tasks() 296 | self.logger.log(f'dynamic tasks: {self.dynamic_tasks}', 'DEBUG') 297 | self.run_static_tasks() 298 | if self.enable_dynamic: 299 | self.run_dynamic_tasks() 300 | last_time = current_time 301 | time.sleep(0.2) 302 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/timer/timer_api/__init__.py -------------------------------------------------------------------------------- /corelib/timer/timer_api/api.py: -------------------------------------------------------------------------------- 1 | from corelib import APIIngressBase 2 | from .handlers import CronjobReadHandler, CronjobWriteHandler, AvailableTasksHandler 3 | 4 | 5 | class APIIngress(APIIngressBase): 6 | actions = { 7 | 'getCronList': CronjobReadHandler, 8 | 'getFilterOptions': CronjobReadHandler, 9 | 'getCronDetail': CronjobReadHandler, 10 | 'addCron': CronjobWriteHandler, 11 | 'deleteCron': CronjobWriteHandler, 12 | 'modifyCron': CronjobWriteHandler, 13 | 'enableCron': CronjobWriteHandler, 14 | 'disableCron': CronjobWriteHandler, 15 | 'renewAtTimeTask': CronjobWriteHandler, 16 | 'getAvailableCronList': AvailableTasksHandler, 17 | } 18 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/choices.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | TASK_ENABLE_CHOICES = OrderedDict() 4 | TASK_ENABLE_CHOICES[0] = '已停用' 5 | TASK_ENABLE_CHOICES[1] = '已启用' 6 | 7 | TASK_RESULT_CHOICES = OrderedDict() 8 | TASK_RESULT_CHOICES[''] = '未知' 9 | TASK_RESULT_CHOICES['success'] = '成功' 10 | TASK_RESULT_CHOICES['failed'] = '失败' 11 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/handlers.py: -------------------------------------------------------------------------------- 1 | from corelib import APIHandlerBase, pre_handler, ChoiceType, StrType, ObjectType, ListType, DictType, IntType, DatetimeType 2 | from corelib.api_serializing_mixins import ListDataMixin, DetailDataMixin, AddDataMixin, DeleteDataMixin, ModifyDataMixin 3 | from .models import CronJob, AvailableTasks 4 | from .choices import TASK_RESULT_CHOICES, TASK_ENABLE_CHOICES 5 | 6 | 7 | class CronjobReadHandler(APIHandlerBase, ListDataMixin, DetailDataMixin): 8 | post_fields = { 9 | 'search': StrType(), 10 | 'enabled': ChoiceType(*TASK_ENABLE_CHOICES.keys()), 11 | 'last_run_result': ChoiceType(*TASK_RESULT_CHOICES.keys()), 12 | 'id': ObjectType(model=CronJob), 13 | } 14 | 15 | @pre_handler(opt=['search', 'enabled', 'last_run_result']) 16 | def getCronList(self): 17 | self.getList(model=CronJob) 18 | 19 | @pre_handler(req=['id']) 20 | def getCronDetail(self): 21 | self.getDetail(model=CronJob) 22 | 23 | def getFilterOptions(self): 24 | self.data = { 25 | 'enabled_options': TASK_ENABLE_CHOICES, 26 | 'task_result_options': TASK_RESULT_CHOICES, 27 | } 28 | 29 | 30 | class CronjobWriteHandler(APIHandlerBase, AddDataMixin, DeleteDataMixin, ModifyDataMixin): 31 | post_fields = { 32 | 'name': StrType(max_length=128), 33 | 'description': StrType(), 34 | 'task': StrType(regex=r'^(\w+\.)+\w+$'), 35 | 'args': ListType(), 36 | 'kwargs': DictType(key=StrType(regex=r'^\w+$')), 37 | 'every': IntType(min=0), 38 | 'at_time': DatetimeType(extra_allowed_values=[None, ]), 39 | 'crontab': StrType(regex=r'^[\*\/\d]+[ \t]+[\*\/\d]+[ \t]+[\*\/\d]+[ \t]+[\*\/\d]+[ \t]+[\*\/\d]+[ \t]+[\*\/\d]+[ \t]+[\*\/\d]+$|^$'), 40 | 'enabled': ChoiceType(*TASK_ENABLE_CHOICES.keys()), 41 | 'expired_count': IntType(min=0), 42 | 'expired_time': DatetimeType(extra_allowed_values=[None, ]), 43 | 'id': ObjectType(model=CronJob, ), 44 | } 45 | 46 | add_data_opt_fileds = ['description', 'args', 'kwargs', 'every', 'crontab', 'enabled', 'expired_count', 'expired_time', 'at_time'] 47 | modify_data_opt_fields = ['description', 'args', 'kwargs', 'every', 'crontab', 'expired_count', 'expired_time', 'at_time'] 48 | 49 | @pre_handler(req=['name', 'task'], opt=add_data_opt_fileds, perm='admin') 50 | def addCron(self): 51 | ''' 52 | 非开放的action，不提供API。 53 | 仅供timer主程序启动时，扫描、自动添加动态定时任务时使用。 54 | ''' 55 | if not self.checked_params.get('every') and not self.checked_params.get('crontab') and not self.checked_params.get('at_time'): 56 | return self.error('ERROR: `every`, `crontab` or `at_time` must be provided.') 57 | self.addData(model=CronJob) 58 | 59 | @pre_handler(req=['id'], perm='admin') 60 | def deleteCron(self): 61 | self.deleteData() 62 | 63 | @pre_handler(req=['id'], opt=modify_data_opt_fields, perm='admin') 64 | def modifyCron(self): 65 | obj = self.checked_params['id'] 66 | new_every = self.checked_params['every'] if 'every' in self.checked_params else obj.every 67 | new_crontab = self.checked_params['crontab'] if 'crontab' in self.checked_params else obj.crontab 68 | new_at = self.checked_params['at_time'] if 'at_time' in self.checked_params else obj.at_time 69 | if not new_every and not new_crontab and not new_at: 70 | return self.error('ERROR: `every`, `crontab` and `at_time` cannot be all empty!') 71 | self.modifyData() 72 | 73 | @pre_handler(req=['id'], perm='admin') 74 | def enableCron(self): 75 | obj = self.checked_params['id'] 76 | obj.enabled = 1 77 | obj.save(update_fields=['enabled']) 78 | self.message = 'To enable cron task succeeded.' 79 | 80 | @pre_handler(req=['id'], perm='admin') 81 | def disableCron(self): 82 | obj = self.checked_params['id'] 83 | obj.enabled = 0 84 | obj.save(update_fields=['enabled']) 85 | self.message = 'To disable cron task succeeded.' 86 | 87 | @pre_handler(req=['id'], opt=['at_time'], perm='admin') 88 | def renewAtTimeTask(self): 89 | obj = self.checked_params['id'] 90 | obj.total_run_count = 0 91 | obj.last_run_start_at = None 92 | obj.last_run_spend_time = 0 93 | obj.last_run_result = '' 94 | if self.checked_params.get('at_time'): 95 | obj.at_time = self.checked_params['at_time'] 96 | obj.save(update_fields=['total_run_count', 'last_run_start_at', 'last_run_spend_time', 'last_run_result', 'at_time']) 97 | 98 | 99 | class AvailableTasksHandler(APIHandlerBase, ListDataMixin, AddDataMixin): 100 | post_fields = { 101 | 'search': StrType() 102 | } 103 | 104 | @pre_handler(opt=['search']) 105 | def getAvailableCronList(self): 106 | self.getList(model=AvailableTasks) 107 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 07:52 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ] 10 | 11 | operations = [ 12 | ] 13 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/migrations/0002_availabletasks_cronjob.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.2 on 2021-05-08 08:04 2 | 3 | from django.db import migrations, models 4 | import django.utils.timezone 5 | import jsonfield.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ('timer_api', '0001_initial'), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='AvailableTasks', 19 | fields=[ 20 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 21 | ('module_path', models.TextField(default='', verbose_name='任务函数路径')), 22 | ('usage_doc', models.TextField(default='', verbose_name='使用说明')), 23 | ], 24 | ), 25 | migrations.CreateModel( 26 | name='CronJob', 27 | fields=[ 28 | ('id', models.AutoField(primary_key=True, serialize=False, verbose_name='ID')), 29 | ('name', models.CharField(default='', max_length=128, unique=True, verbose_name='任务名称')), 30 | ('description', models.TextField(default='', verbose_name='任务描述')), 31 | ('task', models.TextField(default='', verbose_name='任务函数')), 32 | ('args', jsonfield.fields.JSONField(default=[], verbose_name='位置参数')), 33 | ('kwargs', jsonfield.fields.JSONField(default={}, verbose_name='键值参数')), 34 | ('every', models.IntegerField(default=0, verbose_name='执行间隔')), 35 | ('crontab', models.TextField(default='', verbose_name='crontab配置')), 36 | ('at_time', models.DateTimeField(default=None, null=True, verbose_name='任务创建时间')), 37 | ('enabled', models.IntegerField(choices=[(0, '已停用'), (1, '已启用')], default=1, verbose_name='是否启用')), 38 | ('expired_count', models.IntegerField(default=0, verbose_name='失效次数设置')), 39 | ('expired_time', models.DateTimeField(default=None, null=True, verbose_name='失效日期设置')), 40 | ('last_run_start_at', models.DateTimeField(default=None, null=True, verbose_name='最后一次执行时间')), 41 | ('last_run_spend_time', models.FloatField(default=0, verbose_name='最后一次执行耗时')), 42 | ('last_run_result', models.CharField(choices=[('', '未知'), ('success', '成功'), ('failed', '失败')], default='', max_length=16, verbose_name='最后一次执行结果')), 43 | ('total_run_count', models.IntegerField(default=0, verbose_name='总共执行次数')), 44 | ('create_at', models.DateTimeField(default=django.utils.timezone.now, verbose_name='任务创建时间')), 45 | ], 46 | options={ 47 | 'ordering': ['-id'], 48 | }, 49 | ), 50 | ] 51 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/timer/timer_api/migrations/__init__.py -------------------------------------------------------------------------------- /corelib/timer/timer_api/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from jsonfield import JSONField 3 | from django.utils import timezone 4 | from .choices import TASK_ENABLE_CHOICES, TASK_RESULT_CHOICES 5 | 6 | 7 | class CronJob(models.Model): 8 | ''' 9 | 特别字段说明： 10 | every 执行间隔，单位为秒。默认为0，表示不设置。 11 | crontab 一个类似linux的crontab配置字符串。默认为''，空字符串，表示不设置。 12 | 具体分7个字段：'秒 分 时 日 月 周 年' 13 | 如： '31 * * * * * *'，表示每分钟的31秒时刻，启动执行 14 | '0 0 */2 * * * 2021'，表示当前时间的小时位，为2的倍数时执行，且仅在2021年生效。 15 | at_time 仅在此时间点执行一次。通过`last_run_at`来判断是否已经执行。 16 | 17 | 注意：every、crontab、at_time必须设置其中一个，否则该任务无效。 18 | 若都设置，按此优先级顺序生效: every, crontab, at_time 19 | 20 | expired_count 可用于设置运行多少次后自动失效。默认为0，表示不做限制。 21 | expired_time 设置自动失效时间点，过期后不再执行。 22 | 23 | 注意：当这两个同时设定时，谁先达到限制条件，就以谁为准。触发后，将自动设置`enabled`为0 24 | 25 | last_run_result 用于记录任务的执行结果。 26 | 27 | 注意，此表中只记录任务的执行结果，不做任何其他任务执行信息记录。定时任务函数的任务return值，都会被Timer主进程忽略。 28 | 若要跟踪定时任务的执行中间过程，请输出到Timer的日志中查看。 29 | ''' 30 | # db fields. 31 | id = models.AutoField('ID', primary_key=True) 32 | name = models.CharField('任务名称', max_length=128, default='', unique=True) 33 | description = models.TextField('任务描述', default='') 34 | task = models.TextField('任务函数', default='') 35 | args = JSONField('位置参数', default=[]) 36 | kwargs = JSONField('键值参数', default={}) 37 | every = models.IntegerField('执行间隔', default=0) 38 | crontab = models.TextField('crontab配置', default='') 39 | at_time = models.DateTimeField('任务创建时间', null=True, default=None) 40 | enabled = models.IntegerField('是否启用', choices=tuple(TASK_ENABLE_CHOICES.items()), default=1) 41 | expired_count = models.IntegerField('失效次数设置', default=0) 42 | expired_time = models.DateTimeField('失效日期设置', null=True, default=None) 43 | last_run_start_at = models.DateTimeField('最后一次执行时间', null=True, default=None) 44 | last_run_spend_time = models.FloatField('最后一次执行耗时', default=0) 45 | last_run_result = models.CharField('最后一次执行结果', max_length=16, choices=tuple(TASK_RESULT_CHOICES.items()), default='') 46 | total_run_count = models.IntegerField('总共执行次数', default=0) 47 | create_at = models.DateTimeField('任务创建时间', default=timezone.now) 48 | 49 | # serializing fields. 50 | search_fields = ['name', 'description', 'task', 'args', 'kwargs', 'crontab'] 51 | filter_fields = ['enabled', 'last_run_result'] 52 | 53 | class Meta: 54 | ordering = ['-id'] 55 | 56 | 57 | class AvailableTasks(models.Model): 58 | id = models.AutoField('ID', primary_key=True) 59 | module_path = models.TextField('任务函数路径', default='') 60 | usage_doc = models.TextField('使用说明', default='') 61 | 62 | # serializing fields. 63 | search_fields = ['id', 'module_path', 'usage_doc'] 64 | -------------------------------------------------------------------------------- /corelib/timer/timer_api/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from .api import APIIngress 3 | 4 | urlpatterns = [ 5 | path('api/v1', APIIngress.as_view(), name="cronjob_api"), 6 | ] 7 | -------------------------------------------------------------------------------- /corelib/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan011/django-action-api/83b6a0d7662abf19d7b596a795b501baa77934d3/corelib/tools/__init__.py -------------------------------------------------------------------------------- /corelib/tools/ansible_runner.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from django.conf import settings 4 | from threading import Thread 5 | 6 | 7 | class CmdJob(object): 8 | def __init__(self, cmd="", task_name=None): 9 | self.cmd = cmd 10 | self.task_name = task_name 11 | self.result = {} 12 | 13 | 14 | class AnsibleRunner(object): 15 | def __init__(self, ansible_root, ansible_bin, ansible_key, timeout=60, quiet=False, parallel=False): 16 | self.ansible_root = ansible_root 17 | self.bin = ansible_bin 18 | self.key = ansible_key 19 | self.jobs = [] 20 | self.timeout = timeout 21 | self.port = 22 if getattr(settings, 'ANSIBLE_SSH_PORT', None) is None else settings.ANSIBLE_SSH_PORT 22 | self.quiet = quiet 23 | self.parallel = parallel 24 | 25 | def add_job(self, task_name, playbook, hosts, vars=None, single_file_playbook=False): 26 | # To check playbook. 27 | if single_file_playbook: 28 | if not playbook.startswith('/'): 29 | raise Exception("AnsibleRunner.add_job(): Path of single file playbook should not be relatively.") 30 | _playbook = playbook 31 | else: 32 | _playbook = os.path.join(self.ansible_root, playbook) 33 | if not os.path.isfile(_playbook): 34 | raise Exception(f"AnsibleRunner.add_job(): Playbook '{_playbook}' not found.") 35 | 36 | # To check target hosts 37 | _hosts = ','.join(hosts) 38 | if not _hosts: 39 | raise Exception("AnsibleRunner.add_job(): No target hosts provided.") 40 | 41 | # To check vars and generate ansible command. 42 | if vars: 43 | _vars = json.dumps(vars) 44 | job = CmdJob( 45 | f"{self.bin} -i {_hosts}, -e \'{_vars}\' {_playbook} --private-key={self.key} -T {self.timeout} --ssh-extra-args \'-p {self.port}\'", 46 | task_name=task_name 47 | ) 48 | else: 49 | job = CmdJob( 50 | f"{self.bin} -i {_hosts}, {_playbook} --private-key={self.key} -T {self.timeout} --ssh-extra-args \'-p {self.port}\'", 51 | task_name=task_name 52 | ) 53 | 54 | # To add job. 55 | self.jobs.append(job) 56 | 57 | def ansible_result_parser(self, job, lines): 58 | """ 59 | To parse ansible result to a dict, with items like: 60 | { 61 | "192.168.1.2": {"result":"success", "log_lines": [...]}, 62 | "192.168.1.3": {"result":"failed", "log_lines": [...]}, 63 | } 64 | """ 65 | 66 | # To remove playbook retry line, when ansible-playbook runs failed. 67 | _i = None 68 | for line in filter(lambda l: l.strip().endswith(".retry"), lines): 69 | _i = lines.index(line) 70 | break 71 | if _i is not None: 72 | lines.pop(_i) 73 | 74 | # The rest of lines should be a valid json. 75 | str_result = ''.join(lines) 76 | if not self.quiet: 77 | print(str_result) 78 | try: 79 | ansible_result = json.loads(str_result) 80 | except Exception: 81 | # To store the string value as result, if not a json. 82 | job.result = str_result 83 | else: 84 | if not self.quiet: 85 | print("===> json load succeeded!") 86 | for task_result in filter(lambda t: t['task']['name'] == job.task_name, ansible_result['plays'][0]['tasks']): 87 | # print(task_result) 88 | for host_ip in task_result['hosts'].keys(): 89 | job.result[host_ip] = {"result": "success", "log_lines": []} 90 | if task_result["hosts"][host_ip].get('failed') or task_result["hosts"][host_ip].get('unreachable'): # Tag failed. 91 | job.result[host_ip]["result"] = 'failed' 92 | _stdout = task_result["hosts"][host_ip].get("stdout_lines", []) 93 | _stderr = task_result["hosts"][host_ip].get("stderr_lines", []) 94 | job.result[host_ip]["log_lines"] = _stdout + _stderr 95 | break 96 | 97 | def do_job(self, job): 98 | """ 99 | To run a single job. 100 | """ 101 | if not self.quiet: 102 | print(f"===> {job.cmd}") 103 | print(job.cmd) 104 | with os.popen(job.cmd) as f: 105 | self.ansible_result_parser(job, f.readlines()) 106 | 107 | def go(self): 108 | """ 109 | Start to run all jobs. 110 | """ 111 | print(self.jobs) 112 | if self.parallel: 113 | pool = [] 114 | for job in self.jobs: 115 | pool.append(Thread(target=self.do_job, args=[job])) 116 | print('lalala') 117 | for p in pool: 118 | p.start() 119 | for p in pool: 120 | p.join() 121 | else: 122 | for job in self.jobs: 123 | self.do_job(job) 124 | -------------------------------------------------------------------------------- /corelib/tools/coroutine.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | class CoroutineRunner(object): 5 | def __init__(self, works): 6 | """ 7 | works: 二维列表，每个子列表代表一个需要并发执行的任务；子列表第一个元素是执行的函数，其余元素是传给函数的参数; 8 | """ 9 | self.tasks = [] 10 | self.works = works 11 | 12 | async def concurrent(self): 13 | loop = asyncio.get_event_loop() 14 | for real_work in self.works: 15 | # print(f"===> {str(real_work)}") 16 | _func, *_params = real_work 17 | result = asyncio.gather(loop.run_in_executor(None, _func, *_params)) 18 | self.tasks.append(result) 19 | await asyncio.gather(*self.tasks) 20 | 21 | def run(self): 22 | loop = asyncio.get_event_loop() 23 | loop.run_until_complete(self.concurrent()) 24 | -------------------------------------------------------------------------------- /corelib/tools/func_tools.py: -------------------------------------------------------------------------------- 1 | from random import randint 2 | import time 3 | 4 | 5 | def timeSpend(func, *args, **kwargs): 6 | time_start = time.time() 7 | result = func(*args, **kwargs) 8 | time_end = time.time() 9 | time_spend = time_end - time_start 10 | return result, time_spend 11 | 12 | 13 | def tableSort(table_data, *columns, reverse=False): 14 | """ 15 | table_data: A list like [{'col1': 1, 'col2': 2, 'col3': 123}, ...], column keys must be all the same. 16 | *columns: column keys. 17 | reverse: True or False. 18 | 19 | return None. With table_data changed. 20 | """ 21 | sort_by = list(columns) 22 | sort_by.reverse() 23 | for k in sort_by: 24 | table_data.sort(key=lambda d: d[k], reverse=reverse) 25 | 26 | 27 | def tableIndexOf(table_data, get_first_match=True, **row_filter): 28 | """ 29 | row_filter: a dict to match table row data. 30 | 31 | returns: 32 | Return a list, which item is the index number of matched row, if `get_first_match` is False. 33 | Or just a number of the first mateched row's index. 34 | None if no row matched. 35 | 36 | """ 37 | matched_indexes = [] 38 | for i in range(len(table_data)): 39 | row = table_data[i] 40 | field_match_count = 0 41 | for k, v in row_filter.items(): 42 | if k in row and row[k] == v: 43 | field_match_count += 1 44 | if field_match_count == len(row_filter): 45 | matched_indexes.append(i) 46 | if not matched_indexes: 47 | return None 48 | if get_first_match: 49 | return matched_indexes[0] 50 | else: 51 | return matched_indexes 52 | 53 | 54 | def getattrRecursively(obj, attr, default=None): 55 | first, *attrs = attr.split('.') 56 | remain = '.'.join(attrs) 57 | value = getattr(obj, first, default) 58 | if attrs and value != default: 59 | return getattrRecursively(value, remain, default=default) 60 | else: 61 | return value 62 | 63 | 64 | def genUUID(length): 65 | seeds = ''.join(chr(i) for i in range(97, 123)) 66 | seeds += seeds.upper() 67 | seeds += '0123456789' 68 | seeds_len = len(seeds) 69 | return ''.join(seeds[randint(0, seeds_len - 1)] for i in range(length)) 70 | 71 | 72 | def choice_map(choices, value): 73 | for k, v in choices: 74 | if v == value: 75 | return k 76 | 77 | 78 | def groupArray(array, num): 79 | """ 80 | To group an array by `num`. 81 | 82 | :array An iterable object. 83 | :num How many items a sub-group may contained. 84 | 85 | Returns an generator to generate a list contains `num` of items for each iterable calls. 86 | """ 87 | tmp = [] 88 | count = 0 89 | for i in array: 90 | count += 1 91 | tmp.append(i) 92 | if count >= num: 93 | yield tmp 94 | tmp = [] 95 | count = 0 96 | yield tmp 97 | 98 | 99 | def get_item_from_table_list(target_list, filter): 100 | """ 101 | 参数说明： 102 | target_list: 为一个table list, 需满足以下结构，例如： 103 | [ 104 | {'id': 1, 'name': 'lalal', ...}, 105 | {'id': 2, 'name': 'lalalaa', ...}, 106 | ] 107 | 108 | filter: 条件过滤字典，比如： 109 | { 110 | 'name': 'lalal' 111 | } 112 | 113 | 返回第一个匹配到的元素。 114 | """ 115 | for item in target_list: 116 | matched = 0 117 | for k, v in filter.items(): 118 | if k in item and item[k] == v: 119 | matched += 1 120 | if matched == len(filter): 121 | return item 122 | 123 | 124 | def merge_table_list(*target_lists, identified_by='id', order=True, order_by='id'): 125 | identities = set() 126 | result = [] 127 | for table in target_lists: 128 | for item in table: 129 | if not isinstance(item, dict): 130 | return None, "ERROR: Invalid table list found. Item of a table list be all dict." 131 | identity = item.get(identified_by, None) 132 | if identity is None: 133 | return None, f"ERROR: Invalid row identifier '{identified_by}' found for table list row." 134 | if identity not in identities: 135 | identities.add(identity) 136 | result.append(item) 137 | reverse = True if order_by.startswith('-') else False 138 | tableSort(result, 'id', reverse=reverse) 139 | 140 | return result, None 141 | -------------------------------------------------------------------------------- /corelib/tools/gitlab_client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from datetime import datetime 3 | 4 | 5 | class GitLabClient(object): 6 | """ 7 | gitlab相关接口封装 8 | """ 9 | 10 | def __init__(self, gitlab_api_url_base, gitlab_private_token, *args, **kwargs): 11 | self.url_base = gitlab_api_url_base 12 | self.header_base = {'PRIVATE-TOKEN': gitlab_private_token} 13 | 14 | def get_project_id(self, code_repo): 15 | """ 16 | 通过项目名和代码库ssh地址匹配获取项目ID 17 | """ 18 | project_name = code_repo.split('/')[-1].split('.')[0] 19 | url_project_search = f"{self.url_base}/projects?search={project_name}" 20 | res = requests.get(url=url_project_search, headers=self.header_base, verify=False) 21 | if res.status_code != 200: 22 | raise Exception(f"ERROR: Failed to get project id gitlab api return status Code: {res.status_code}") 23 | projects = res.json() 24 | project_id = None 25 | for project in projects: 26 | if project['ssh_url_to_repo'] == code_repo: 27 | project_id = project['id'] 28 | break 29 | if not project_id: 30 | raise Exception(f"错误：在gitlab中没有找到项目{project_name}，请检查项目名和代码库是否匹配") 31 | return project_id 32 | 33 | def get_commit(self, project_id, code_branch): 34 | """ 35 | 获取项目commit列表 36 | """ 37 | url_commit_get = f"{self.url_base}/projects/{project_id}/repository/commits?ref_name={code_branch}" 38 | res = requests.get(url=url_commit_get, headers=self.header_base, verify=False) 39 | if res.status_code != 200: 40 | raise Exception(f"ERROR: Failed to get commit gitlab api return status Code: {res.status_code}") 41 | commit_res = res.json() 42 | commits = [] 43 | for commit in commit_res: 44 | commits.append({ 45 | "short_id": commit["id"][:8], 46 | "commit_id": commit["id"], 47 | "committer_name": commit["committer_name"], 48 | "committed_date": self._time_format(commit["committed_date"]), 49 | }) 50 | return commits 51 | 52 | def get_tag(self, project_id, code_branch): 53 | """ 54 | 获取项目tag列表 55 | """ 56 | url_tag_get = f"{self.url_base}/projects/{project_id}/repository/tags" 57 | res = requests.get(url=url_tag_get, headers=self.header_base, verify=False) 58 | if res.status_code != 200: 59 | raise Exception(f"ERROR: Failed to get tag gitlab api return status Code: {res.status_code}") 60 | tags_res = res.json() 61 | # 保留当前branch的tag 62 | commits = self.get_commit(project_id, code_branch) 63 | commits_set = set([item["commit_id"] for item in commits]) 64 | tags = [] 65 | for tag in tags_res: 66 | if tag["commit"]["id"] in commits_set: 67 | tags.append({ 68 | "tag_name": tag["name"], 69 | "short_id": tag["commit"]["id"][:8], 70 | "commit_id": tag["commit"]["id"], 71 | "committer_name": tag["commit"]["committer_name"], 72 | "committed_date": self._time_format(tag["commit"]["committed_date"]), 73 | }) 74 | return tags 75 | 76 | def _time_format(self, time): 77 | """ 78 | 格式化时间 79 | """ 80 | tmp = datetime.strptime(time.split("+")[0], "%Y-%m-%dT%H:%M:%S.%f") 81 | return datetime.strftime(tmp, "%Y-%m-%d %H:%M:%S") 82 | 83 | def add_project_hook(self, project_id=None, url=None, push_events_branch_filter=None, push_events=False, tag_push_events=False, *args, **kwargs): 84 | """ 85 | 添加项目hook 86 | Attribute Type Required Description 87 | confidential_issues_events boolean no Trigger hook on confidential issues events 88 | confidential_note_events boolean no Trigger hook on confidential note events 89 | deployment_events boolean no Trigger hook on deployment events 90 | enable_ssl_verification boolean no Do SSL verification when triggering the hook 91 | id integer/string yes The ID or URL-encoded path of the project 92 | issues_events boolean no Trigger hook on issues events 93 | job_events boolean no Trigger hook on job events 94 | merge_requests_events boolean no Trigger hook on merge requests events 95 | note_events boolean no Trigger hook on note events 96 | pipeline_events boolean no Trigger hook on pipeline events 97 | push_events_branch_filter string no Trigger hook on push events for matching branches only 98 | push_events boolean no Trigger hook on push events 99 | tag_push_events boolean no Trigger hook on tag push events 100 | token string no Secret token to validate received payloads; this is not returned in the response 101 | url string yes The hook URL 102 | wiki_page_events boolean no Trigger hook on wiki events 103 | """ 104 | url_add_hook = f"{self.url_base}/projects/{project_id}/hooks" 105 | data = {"url": url, "push_events": push_events, "tag_push_events": tag_push_events} 106 | if push_events_branch_filter: 107 | data.update({"push_events_branch_filter": push_events_branch_filter}) 108 | if kwargs: 109 | data.update(kwargs) 110 | res = requests.post(url=url_add_hook, headers=self.header_base, data=data, verify=False) 111 | hook_res = res.json() 112 | if hook_res.get("id"): 113 | return True 114 | elif hook_res.get("message"): 115 | print(hook_res.get("message")) 116 | else: 117 | print(hook_res.get("error")) 118 | return False 119 | 120 | def list_project_hook(self, project_id=None): 121 | """ 122 | 获取项目hook列表 123 | """ 124 | url_list_hook = f"{self.url_base}/projects/{project_id}/hooks" 125 | res = requests.get(url=url_list_hook, headers=self.header_base, verify=False) 126 | if res.status_code != 200: 127 | raise Exception(f"ERROR: Failed to get project list gitlab api return status Code: {res.status_code}") 128 | return res.json() 129 | 130 | def delete_project_hook(self, project_id=None, code_branch=None, hook_url=None): 131 | """ 132 | 删除项目hook 133 | """ 134 | hooks = self.list_project_hook(project_id=project_id) 135 | hook_id = None 136 | for hook in hooks: 137 | if hook["url"] == hook_url: 138 | if hook["push_events"] and hook["push_events_branch_filter"] == code_branch: 139 | # push hook 140 | hook_id = hook["id"] 141 | break 142 | else: 143 | # tag hook 144 | hook_id = hook["id"] 145 | break 146 | print(f"[GitlabClient.delete_project_hook()] hook_id: {hook_id}") 147 | if hook_id: 148 | url_delete_hook = f"{self.url_base}/projects/{project_id}/hooks/{hook_id}" 149 | res = requests.delete(url=url_delete_hook, headers=self.header_base, verify=False) 150 | if res.status_code == 204: 151 | return True 152 | else: 153 | print(f"[GitlabClient.delete_project_hook()] gitlab api response, status_code: {res.status_code}") 154 | print(f"[GitlabClient.delete_project_hook()] gitlab api response, status_code: {res.text}") 155 | return False 156 | 157 | def get_branches(self, project_id, search=None): 158 | """ 159 | 获取分支列表 160 | """ 161 | url_list_branch = f"{self.url_base}/projects/{project_id}/repository/branches" 162 | if search: 163 | url_list_branch += f"?search={search}" 164 | res = requests.get(url=url_list_branch, headers=self.header_base, verify=False) 165 | if res.status_code != 200: 166 | raise Exception(f"ERROR: Failed to get branch gitlab api return status Code: {res.status_code}") 167 | branches_res = res.json() 168 | branches = [] 169 | for branch in branches_res: 170 | branches.append({ 171 | "branch_name": branch["name"] 172 | }) 173 | return branches 174 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/__init__.py: -------------------------------------------------------------------------------- 1 | from .deployment_client import DeploymentClient 2 | from .daemonset_client import DaemonsetClient 3 | 4 | __all__ = ('DeploymentClient', 'DaemonsetClient') 5 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/client_base.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | class K8SClientBase(object): 5 | """ 6 | 仅适用于k8s-apiserver的直接认证调用。 7 | 8 | 对于各大云厂商的接口网关复杂认证方式，并不适用。 9 | """ 10 | 11 | def __init__(self, api_url_base, auth_params, auth_type=None, api_version_prefix=None, ignore_https_verify=False, *args, **kwargs): 12 | ''' 13 | 确定认证用的headers，以及与k8s-apiserver接口版本相关的基础url。 14 | 15 | 参数： 16 | api_url_base 字符串。如：'http://:', 'https://' 17 | 18 | auth_type 可选范围：'Bearer Token', 'CA Certs'; 19 | 默认为'Bearer Token'。 20 | 21 | auth_params 一个字典。记录的参数，用于传k8s接口认证。字典字段要求，随`auth_type`，有不同要求。 22 | 'Bearer Token'，要求：{'bearer_token': ''} 23 | 'CA Certs'，要求：{ 24 | 25 | } 26 | 27 | api_version_prefix 一个字符串。k8s-apiserver的版本前缀，用于资源域名拼接。 28 | 默认为：/apis/apps/v1 29 | 老版本的k8s还可能是/apis/extensions/v1beta1 30 | ignore_https_verify Bool类型。用于API调用时，对私有k8s集群，忽略https证书校验。 31 | 32 | ''' 33 | self.api_url_base = str(api_url_base) 34 | self.auth_params = auth_params 35 | self.auth_type = 'Bearer Token' if auth_type is None else auth_type 36 | self.api_version_prefix = '/apis/apps/v1' if api_version_prefix is None else str(api_version_prefix) 37 | 38 | # 设置接口认证参数 39 | self.headers = {} 40 | self.request_verify = None 41 | self.certs = None 42 | if self.auth_type == 'Bearer Token': 43 | if 'bearer_token' not in self.auth_params: 44 | raise Exception(f"K8SClientBase.__init__(): '{self.auth_type}' requires 'bearer_token' in arg `auth_params`!") 45 | self.headers = {"Authorization": "Bearer " + self.auth_params['bearer_token']} 46 | if ignore_https_verify: 47 | self.request_verify = False 48 | elif self.auth_type == 'CA Certs': 49 | for key in ['cert_file', 'cert_key_file']: 50 | if key not in self.auth_params: 51 | raise Exception(f"K8SClientBase.__init__(): '{self.auth_type}' requires '{key}' in arg `auth_params`!") 52 | self.request_verify = self.auth_params['ca_file'] if 'ca_file' in self.auth_params else False 53 | self.certs = (self.auth_params['cert_file'], self.auth_params['cert_key_file']) 54 | else: 55 | raise Exception(f"K8SClientBase.__init__(): Invalid auth_type '{auth_type}'!") 56 | 57 | # 拼接API基础URL 58 | if self.api_url_base.endswith('/'): 59 | self.api_url_base = self.api_url_base[:-1] 60 | if not self.api_version_prefix.startswith('/'): 61 | self.api_version_prefix = '/' + self.api_version_prefix 62 | if self.api_version_prefix.endswith('/'): 63 | self.api_version_prefix = self.api_version_prefix[:-1] 64 | self.api_url = f"{self.api_url_base}{self.api_version_prefix}" 65 | 66 | def call(self, method, url, data=None, additional_headers=None): 67 | """ 68 | 封装认证过程，统一发起k8s接口调用。 69 | """ 70 | headers = {} 71 | headers.update(self.headers) 72 | method = method.upper() 73 | if method == 'PATCH': 74 | headers.update({"Content-Type": "application/strategic-merge-patch+json"}) 75 | if isinstance(additional_headers, dict): 76 | headers.update(additional_headers) 77 | 78 | request_params = { 79 | 'method': method, 80 | 'url': url, 81 | 'headers': headers, 82 | } 83 | 84 | if isinstance(data, dict): 85 | request_params['json'] = data 86 | 87 | if self.request_verify is not None: 88 | request_params['verify'] = self.request_verify 89 | 90 | if self.auth_type == 'CA Certs': 91 | request_params['cert'] = self.certs 92 | 93 | return requests.request(**request_params) 94 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/configmap_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | 3 | 4 | class ConfigMapClient(K8SClientBase): 5 | """ 6 | ConfigMap的增删改查 7 | """ 8 | 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | pass 12 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/daemonset_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | from datetime import datetime 3 | 4 | 5 | class DaemonsetClient(K8SClientBase): 6 | """ 7 | 用于daemonset增删改查。 8 | 9 | 通常用于daemonset的部署与发版。 10 | """ 11 | 12 | def __init__(self, name, namespace, *args, **kwargs): 13 | super().__init__(*args, **kwargs) 14 | self.daemonset_name = name 15 | self.namespace = namespace 16 | 17 | # 变量拼接 18 | self.url_ds_base = f"{self.api_url}/namespaces/{namespace}/daemonsets" 19 | self.url_ds_object = f"{self.api_url}/namespaces/{namespace}/daemonsets/{name}" 20 | self.last_generation = None 21 | 22 | def get_daemonset(self, check_existence=False): 23 | """ 24 | 获取daemonset对象 25 | """ 26 | _pre = "DaemonsetClient.get_daemonset()" 27 | 28 | # 直接请求数据 29 | res = self.call(method='get', url=self.url_ds_object) 30 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 31 | if res.status_code != 200: 32 | if check_existence and res.status_code == 404: 33 | return '__NOT_EXIST__' 34 | else: 35 | raise Exception(f"{_pre} [ERROR]: Failed to get daemonset object '{self.daemonset_name}'. K8S API returns status Code: {res.status_code}") 36 | 37 | return res.json() 38 | 39 | def get_daemonset_status(self): 40 | """ 41 | 单独获取daemonset的status 42 | """ 43 | data = self.get_daemonset() 44 | if 'metadata' in data: 45 | if 'managedFields' in data['metadata']: 46 | data['metadata'].pop('managedFields') 47 | return data 48 | 49 | def apply(self, daemonset): 50 | """ 51 | 相当于kubectl的apply功能 52 | 53 | 参数： 54 | daemonset 一个字典。包含daemonset的所有定义参数。 55 | """ 56 | _pre = "DaemonsetClient.apply()" 57 | 58 | # 检查daemonset数据 59 | if not isinstance(daemonset, dict): 60 | raise Exception(f"{_pre} [ERROR]: Invalid daemonset data. It must be a Dict.") 61 | if daemonset.get('kind') != 'Daemonset': 62 | raise Exception(f"{_pre} [ERROR]: Invalid daemonset data. Not a Daemonset kind.") 63 | if not (isinstance(daemonset.get('metadata'), dict) and daemonset['metadata'].get('name') == self.daemonset_name): 64 | raise Exception(f"{_pre} [ERROR]: Invalid daemonset data. Name not match.") 65 | 66 | # 存在就做patch，否则做create 67 | daemonset_online = self.get_daemonset(check_existence=True) 68 | method = 'POST' if daemonset_online == '__NOT_EXIST__' else 'PATCH' 69 | url = self.url_ds_base if daemonset_online == '__NOT_EXIST__' else self.url_ds_object 70 | 71 | # 发起请求 72 | res = self.call(method=method, url=url, data=daemonset) 73 | print(f"{_pre} [ERROR]: K8S API Response status code: {res.status_code}") 74 | if res.status_code != 200 and res.status_code != 201 and res.status_code != 202: 75 | raise Exception(f"{_pre} [ERROR]: Failed to apply daemonset '{self.daemonset_name}'. K8S API returns status_code: {res.status_code}") 76 | 77 | def _set_maxunavailable(self, data, desiredNumber, deploy_proportion, max_unavailable): 78 | # 计算maxUnavailable 79 | _max = None 80 | if max_unavailable: 81 | _max = max_unavailable 82 | elif deploy_proportion: 83 | _max = int(desiredNumber / deploy_proportion) 84 | if _max < 1: 85 | _max = 1 86 | if _max: 87 | data['spec']['updateStrategy'] = { 88 | "type": "RollingUpdate", 89 | "rollingUpdate": { 90 | "maxUnavailable": _max 91 | } 92 | } 93 | 94 | def set_image(self, containers, deploy_proportion=None, max_unavailable=None): 95 | """ 96 | 相当于kubectl的set image指令。 97 | 98 | 参数： 99 | containers 一个列表，元素为一个字典。 100 | 字典描述了一个pod中的container该如何更新镜像。字典需包含两个字段: 101 | 'name': pod中容器的名称 102 | 'image': 该容器需更新的目标镜像 103 | deploy_proportion int型，部署比例系数。如'3'，就表示每次部署1/3 104 | max_unavailable int型，滚动更新实例个数。设置多少个，每次就更新多少个。 105 | 106 | 当deploy_proportion与max_unavailable同时存在时，以max_unavailable为准 107 | """ 108 | _pre = 'DaemonsetClient.set_image()' 109 | 110 | # 检查containers 111 | if not isinstance(containers, list): 112 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, must be a list.") 113 | for c in containers: 114 | if not isinstance(c, dict): 115 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, item must be a dict.") 116 | if 'name' not in c or 'image' not in c: 117 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, illegal item.") 118 | 119 | # 获取generation. 120 | obj_data = self.get_daemonset() 121 | self.last_generation = obj_data['metadata'].get('generation', 0) 122 | desiredNumber = obj_data['status']['desiredNumberScheduled'] 123 | 124 | # request data 125 | data = {"spec": {"template": {"spec": {"containers": containers}}}} 126 | self._set_maxunavailable(data, desiredNumber, deploy_proportion, max_unavailable) 127 | 128 | # 发起请求，分析结果 129 | res = self.call(method='patch', url=self.url_ds_object, data=data) 130 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 131 | print(f"{_pre}: K8S API Response status text: {res.text}") 132 | if res.status_code != 200: 133 | raise Exception(f"{_pre} [ERROR]: To set image failed. API returns status code: {res.status_code}") 134 | 135 | def patch_version(self, version=None, deploy_proportion=None, max_unavailable=None): 136 | """ 137 | 一般用于重启daemonset中的pod。也适用于依赖initContainer来更新部署应用程序的情况。 138 | """ 139 | _pre = "DaemonsetClient.patch_version()" 140 | 141 | version_date_suffix = datetime.now().strftime("patch-at-%Y%m%d_%H%M%S") 142 | version = version_date_suffix if version is None else f"{str(version)}.{version_date_suffix}" 143 | 144 | # 获取generation. 145 | obj_data = self.get_daemonset() 146 | self.last_generation = obj_data['metadata'].get('generation', 0) 147 | desiredNumber = obj_data['status']['desiredNumberScheduled'] 148 | 149 | # request data 150 | data = {"spec": {"template": {"metadata": {"labels": {"version": version}}}}} 151 | self._set_maxunavailable(data, desiredNumber, deploy_proportion, max_unavailable) 152 | 153 | # 发起请求，分析结果 154 | res = self.call(method='patch', url=self.url_ds_object, data=data) 155 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 156 | print(f"{_pre}: K8S API Response status text: {res.text}") 157 | if res.status_code != 200: 158 | raise Exception(f"{_pre} [ERROR]: To patch version failed. API returns status code: {res.status_code}") 159 | 160 | def get_updating_progress(self): 161 | """ 162 | 用于获取更新部署时，更新进度。 163 | """ 164 | _pre = 'DaemonsetClient.get_updating_progress()' 165 | if self.last_generation is None: 166 | raise Exception(f"{_pre} [ERROR]: `self.last_generation` is not set.") 167 | data = self.get_daemonset() 168 | this_generation = data['status'].get('observedGeneration', 0) 169 | updated = data['status'].get('updatedNumberScheduled', 0) if this_generation > self.last_generation else 0 170 | desired = data['status'].get('desiredNumberScheduled', 0) 171 | ready = data['status'].get('numberReady', 0) 172 | 173 | if updated <= ready: 174 | return updated, desired 175 | else: 176 | return ready, desired 177 | 178 | def delete_daemonset(self): 179 | """ 180 | 用于删除daemonset 181 | """ 182 | _pre = 'DaemonsetClient.delete_daemonset()' 183 | res = self.call(method='delete', url=self.url_ds_object) 184 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 185 | if res.status_code != 200 and res.status_code != 202: 186 | raise Exception(f"{_pre} [ERROR]: To delete daemonset failed. API returns status code: {res.status_code}") 187 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/deployment_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | from datetime import datetime 3 | 4 | 5 | class DeploymentClient(K8SClientBase): 6 | """ 7 | 用于deployment增删改查。 8 | 9 | 通常用于deployment的部署与发版。 10 | 11 | 目前，更新策略仅支持RollingUpdate。 12 | """ 13 | 14 | def __init__(self, name, namespace, *args, **kwargs): 15 | super().__init__(*args, **kwargs) 16 | self.deployment_name = name 17 | self.namespace = namespace 18 | 19 | # 变量拼接 20 | self.url_dp_base = f"{self.api_url}/namespaces/{namespace}/deployments" 21 | self.url_dp_object = f"{self.api_url}/namespaces/{namespace}/deployments/{name}" 22 | self.url_dp_status = f"{self.api_url}/namespaces/{namespace}/deployments/{name}/status" 23 | self.last_generation = None 24 | 25 | def get_deployment(self, check_existence=False): 26 | """ 27 | 获取deployment对象 28 | """ 29 | _pre = "DeploymentClient.get_deployment()" 30 | 31 | # 直接请求数据 32 | res = self.call(method='get', url=self.url_dp_object) 33 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 34 | if res.status_code != 200: 35 | if check_existence and res.status_code == 404: 36 | return '__NOT_EXIST__' 37 | else: 38 | raise Exception(f"{_pre} [ERROR]: Failed to get deployment object '{self.deployment_name}'. K8S API returns status Code: {res.status_code}") 39 | 40 | return res.json() 41 | 42 | def get_deployment_status(self): 43 | """ 44 | 单独获取deployment的status 45 | """ 46 | _pre = "DeploymentClient.get_deployment_status()" 47 | 48 | # 直接请求数据 49 | res = self.call(method='get', url=self.url_dp_status) 50 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 51 | if res.status_code != 200: 52 | raise Exception(f"{_pre} [ERROR]: Failed to get deployment status of '{self.deployment_name}'. K8S API returns status Code: {res.status_code}") 53 | 54 | data = res.json() 55 | if 'metadata' in data: 56 | if 'managedFields' in data['metadata']: 57 | data['metadata'].pop('managedFields') 58 | return data 59 | 60 | def apply(self, deployment): 61 | """ 62 | 相当于kubectl的apply功能 63 | 64 | 参数： 65 | deployment 一个字典。包含deployment的所有定义参数。 66 | """ 67 | _pre = "DeploymentClient.apply()" 68 | 69 | # 检查deployment数据 70 | if not isinstance(deployment, dict): 71 | raise Exception(f"{_pre} [ERROR]: Invalid deployment data. It must be a Dict.") 72 | if deployment.get('kind') != 'Deployment': 73 | raise Exception(f"{_pre} [ERROR]: Invalid deployment data. Not a Deployment kind.") 74 | if not (isinstance(deployment.get('metadata'), dict) and deployment['metadata'].get('name') == self.deployment_name): 75 | raise Exception(f"{_pre} [ERROR]: Invalid deployment data. Name not match.") 76 | 77 | # 存在就做patch，否则做create 78 | deployment_online = self.get_deployment(check_existence=True) 79 | method = 'POST' if deployment_online == '__NOT_EXIST__' else 'PATCH' 80 | url = self.url_dp_base if deployment_online == '__NOT_EXIST__' else self.url_dp_object 81 | 82 | # 发起请求 83 | res = self.call(method=method, url=url, data=deployment) 84 | print(f"{_pre} [ERROR]: K8S API Response status code: {res.status_code}") 85 | if res.status_code != 200 and res.status_code != 201 and res.status_code != 202: 86 | raise Exception(f"{_pre} [ERROR]: Failed to apply deployment '{self.deployment_name}'. K8S API returns status_code: {res.status_code}") 87 | 88 | def _set_maxunavailable(self, data, replicas, deploy_proportion, max_unavailable): 89 | # 计算maxUnavailable 90 | _max = None 91 | if max_unavailable: 92 | _max = max_unavailable 93 | elif deploy_proportion: 94 | _max = int(replicas / deploy_proportion) 95 | if _max < 1: 96 | _max = 1 97 | 98 | if _max: 99 | data['spec']['Strategy'] = { 100 | "type": "RollingUpdate", 101 | "rollingUpdate": { 102 | "maxUnavailable": _max 103 | } 104 | } 105 | 106 | def set_image(self, containers, deploy_proportion=None, max_unavailable=None): 107 | """ 108 | 相当于kubectl的set image指令。 109 | 110 | 参数： 111 | containers 一个列表，元素为一个字典。 112 | 字典描述了一个pod中的container该如何更新镜像。字典需包含两个字段: 113 | 'name': pod中容器的名称 114 | 'image': 该容器需更新的目标镜像 115 | deploy_proportion int型，部署比例系数。如'3'，就表示每次部署1/3 116 | max_unavailable int型，滚动更新实例个数。设置多少个，每次就更新多少个。 117 | 118 | 注意，对于默认的‘RollingUpdate’策略，当deploy_proportion与max_unavailable同时存在时，以max_unavailable为准 119 | """ 120 | _pre = 'DeploymentClient.set_image()' 121 | 122 | # 检查containers 123 | if not isinstance(containers, list): 124 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, must be a list.") 125 | for c in containers: 126 | if not isinstance(c, dict): 127 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, item must be a dict.") 128 | if 'name' not in c or 'image' not in c: 129 | raise Exception(f"{_pre} [ERROR]: Illigal Arg `containers`, illegal item.") 130 | 131 | # 获取generation. 132 | obj_data = self.get_deployment() 133 | self.last_generation = obj_data['metadata'].get('generation', 0) 134 | replicas = obj_data['spec']['replicas'] 135 | 136 | # request data 137 | data = {"spec": {"template": {"spec": {"containers": containers}}}} 138 | self._set_maxunavailable(data, replicas, deploy_proportion, max_unavailable) 139 | 140 | # 发起请求，分析结果 141 | res = self.call(method='patch', url=self.url_dp_object, data=data) 142 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 143 | print(f"{_pre}: K8S API Response status text: {res.text}") 144 | if res.status_code != 200: 145 | raise Exception(f"{_pre} [ERROR]: To set image failed. API returns status code: {res.status_code}") 146 | 147 | def patch_version(self, version=None, deploy_proportion=None, max_unavailable=None): 148 | """ 149 | 一般用于重启deployment中的pod。也适用于依赖initContainer来更新部署应用程序的情况。 150 | 151 | 参数： 152 | version 一个字符串，表征指定的版本号。 153 | 若不指定，则默认为"patch-at-<当前日期时间>" 154 | deploy_proportion int型，部署比例系数。如'3'，就表示每次部署1/3 155 | max_unavailable int型，滚动更新实例个数。设置多少个，每次就更新多少个。 156 | 157 | 注意，对于默认的‘RollingUpdate’策略，当deploy_proportion与max_unavailable同时存在时，以max_unavailable为准 158 | """ 159 | _pre = "DeploymentClient.patch_version()" 160 | 161 | version_date_suffix = datetime.now().strftime("patch-at-%Y%m%d_%H%M%S") 162 | version = version_date_suffix if version is None else f"{str(version)}.{version_date_suffix}" 163 | 164 | # 获取generation. 165 | obj_data = self.get_deployment() 166 | self.last_generation = obj_data['metadata'].get('generation', 0) 167 | replicas = obj_data['spec']['replicas'] 168 | 169 | # request data 170 | data = {"spec": {"template": {"metadata": {"labels": {"version": version}}}}} 171 | self._set_maxunavailable(data, replicas, deploy_proportion, max_unavailable) 172 | 173 | # 发起请求，分析结果 174 | res = self.call(method='patch', url=self.url_dp_object, data=data) 175 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 176 | print(f"{_pre}: K8S API Response status text: {res.text}") 177 | if res.status_code != 200: 178 | raise Exception(f"{_pre} [ERROR]: To patch version failed. API returns status code: {res.status_code}") 179 | 180 | def get_updating_progress(self): 181 | """ 182 | 用于获取更新部署时，更新进度。 183 | """ 184 | _pre = 'DeploymentClient.get_updating_progress()' 185 | if self.last_generation is None: 186 | raise Exception(f"{_pre} [ERROR]: `self.last_generation` is not set as expected!") 187 | status = self.get_deployment_status() 188 | this_generation = int(status['status'].get('observedGeneration', 0)) 189 | replicas = int(status['spec'].get('replicas', 0)) 190 | updated = int(status['status'].get('updatedReplicas', 0)) if this_generation > self.last_generation else 0 191 | return updated, replicas 192 | 193 | def delete_deployment(self): 194 | """ 195 | 用于删除deployment 196 | """ 197 | _pre = 'DeploymentClient.delete_deployment()' 198 | res = self.call(method='delete', url=self.url_dp_object) 199 | print(f"{_pre}: K8S API Response status code: {res.status_code}") 200 | if res.status_code != 200 and res.status_code != 202: 201 | raise Exception(f"{_pre} [ERROR]: To delete deployment failed. API returns status code: {res.status_code}") 202 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/ingress_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | 3 | 4 | class IngressClient(K8SClientBase): 5 | """ 6 | Ingress的增删改查 7 | """ 8 | 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | pass 12 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/node_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | 3 | 4 | class NodeClient(K8SClientBase): 5 | """ 6 | 集群节点的增删改查 7 | """ 8 | 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | pass 12 | -------------------------------------------------------------------------------- /corelib/tools/k8s_clients/service_client.py: -------------------------------------------------------------------------------- 1 | from .client_base import K8SClientBase 2 | 3 | 4 | class ServiceClient(K8SClientBase): 5 | """ 6 | Service的增删改查 7 | """ 8 | 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | pass 12 | -------------------------------------------------------------------------------- /corelib/tools/logger.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | from os import path, makedirs 4 | from datetime import datetime 5 | 6 | 7 | class Logger(object): 8 | level_define = { 9 | "DEBUG": 5, 10 | "INFO": 4, 11 | "WARNING": 3, 12 | "ERROR": 2, 13 | "FATAL": 1, 14 | } 15 | 16 | def __init__(self, log_file=sys.stdout, trigger_level='INFO', msg_prefix=None): 17 | self.log_file = log_file 18 | self.trigger_level = trigger_level 19 | self.pre_check() 20 | if log_file == sys.stdout: 21 | self.log_f_obj = sys.stdout 22 | else: 23 | try: 24 | self.log_f_obj = open(log_file, 'a') 25 | except Exception: 26 | self.log_f_obj = sys.stdout 27 | self.msg_prefix = msg_prefix 28 | 29 | def pre_check(self): 30 | if self.log_file == sys.stdout: 31 | return None 32 | log_dir = path.dirname(self.log_file) 33 | if not path.isdir(log_dir): 34 | makedirs(log_dir) 35 | 36 | def log(self, msg, level='INFO'): 37 | this_level = level.upper() 38 | msg = str(msg) 39 | msg = str(self.msg_prefix) + msg if self.msg_prefix is not None else msg 40 | if this_level in self.level_define and self.level_define[this_level] <= self.level_define[self.trigger_level.upper()]: 41 | log_time = time.strftime("%F %T", time.localtime(time.time())) 42 | log_line = ' '.join([log_time, f"[{level}]", msg]) 43 | print(log_line, file=self.log_f_obj, flush=True) 44 | 45 | def exit(self, msg): 46 | level = 'FATAL' 47 | log_time = time.strftime("%F %T", time.localtime(time.time())) 48 | log_line = ' '.join([log_time, level, msg]) 49 | raise SystemExit(log_line) 50 | 51 | 52 | def debug_print(msg, file=sys.stdout): 53 | time_now = datetime.now().strftime('%F %T') 54 | prefix = f"===> {time_now} [debug]" 55 | print(prefix, msg, flush=True, file=file) 56 | -------------------------------------------------------------------------------- /corelib/tools/mysql_client.py: -------------------------------------------------------------------------------- 1 | import MySQLdb 2 | 3 | 4 | class MysqlClient(object): 5 | def __init__(self, host, port, db, user, passwd, charset='utf8'): 6 | self.host = host 7 | self.port = port 8 | self.db = db 9 | self.user = user 10 | self.passwd = passwd 11 | self.charset = charset 12 | 13 | def connect(self, cursorclass=None): 14 | err = None 15 | try: 16 | self.client = MySQLdb.connect(host=self.host, port=self.port, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset) 17 | self.cursor = self.client.cursor(cursorclass=cursorclass) 18 | except Exception as e: 19 | err = str(e) 20 | return err 21 | 22 | def query(self, sql): 23 | c = self.cursor 24 | try: 25 | c.execute(sql) 26 | except Exception as e: 27 | return None, str(e) 28 | return c.fetchall(), None 29 | 30 | def disconnect(self): 31 | self.cursor.close() 32 | self.client.close() 33 | 34 | def commit(self): 35 | try: 36 | self.client.commit() 37 | except Exception as e: 38 | self.client.rollback() 39 | return 'SQL client commit failed: ' + str(e) 40 | 41 | def rollback(self): 42 | self.client.rollback() 43 | 44 | 45 | class AlpacaMysqlClient(MysqlClient): 46 | def __init__(self, host, port, db, user, passwd, charset='utf8'): 47 | MysqlClient.__init__(self, host, port, db, user, passwd, charset='utf8') 48 | 49 | def ssdQuery(self, sql=''): 50 | c = self.client.cursor(cursorclass=MySQLdb.cursors.SSDictCursor) 51 | c.execute(sql) 52 | for row in c: 53 | yield row 54 | 55 | def dictQuery(self, sql=''): 56 | c = self.client.cursor(cursorclass=MySQLdb.cursors.DictCursor) 57 | data = None 58 | while True: 59 | sql = (yield data) 60 | c.execute(sql) 61 | data = c.fetchall() 62 | -------------------------------------------------------------------------------- /corelib/tools/saltstack_runner.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | 4 | 5 | class SaltRunnerUsageERROR(Exception): 6 | pass 7 | 8 | 9 | class SaltRunner: 10 | """ 11 | 通过调用salt-api来执行saltstack集中控制指令的工具。 12 | """ 13 | saltapi_session = None 14 | 15 | def __init__(self, salt_api_url, salt_api_user, salt_api_passwd, eauth='pam', quiet=False): 16 | """ 17 | 登录salt-api，获得session 18 | """ 19 | self.salt_api_url = salt_api_url 20 | self.salt_api_auth_config = { 21 | 'username': salt_api_user, 22 | 'password': salt_api_passwd, 23 | 'eauth': eauth, 24 | } 25 | 26 | self.quiet = quiet 27 | salt_api_session = requests.Session() 28 | salt_api_session_auth = salt_api_session.post(salt_api_url + '/login', self.salt_api_auth_config, verify=False) 29 | if salt_api_session_auth.status_code != 200: 30 | raise Exception('SaltRunner.__init__(): ERROR: Login to salt-api failed. HTTP status: %d' % salt_api_session_auth.status_code) 31 | else: 32 | self.saltapi_session = salt_api_session 33 | self.result = {} 34 | 35 | def go(self, node_list, system_cmd=None, script=None, script_args=None): 36 | """ 37 | 用于调用salt-api来远程一个命令，或者一个脚本。 38 | 39 | 参数： 40 | node_list 一个列表，元素为ip字符串，代表要执行操作的目标机器。 41 | system_cmd 命令行字符串，远程执行的命令。 42 | script 字符串，表示saltstack可远程获取的脚本地址。 43 | 要求只能传入http开头的url，或者salt开头的存放于saltstack fileroot中的文件 44 | script_args 要求是一个list，每个元素会自动转化为字符串，将作为脚本的位置参数。 45 | 注意： 46 | a.如果参数包含空格，则会自动给参数加上"'"； 47 | b.如果参数包含空格与引号：包含双引号，则外围会使用单引号"'"；包含单引号，则外围会使用双引号'"'，如果单双引号都有，则抛错。 48 | 49 | """ 50 | # 解析参数 51 | nodes = ','.join(node_list) 52 | if system_cmd is not None: 53 | salt_func = 'cmd.run_all' 54 | elif script is not None: 55 | salt_func = 'cmd.script' 56 | if re.search('^(http|salt).*://.*', script) is None: 57 | raise SaltRunnerUsageERROR(f"SaltRunner.go(): ERROR: saltstack cannot get remote script with content: {script}") 58 | else: 59 | raise SaltRunnerUsageERROR("SaltRunner.go(): ERROR: Params `system_cmd` or `script` must be provided.") 60 | 61 | salt_api_action = { 62 | "client": "local", 63 | "tgt": nodes, 64 | "fun": salt_func, 65 | "arg": system_cmd if system_cmd else script, 66 | "expr_form": "list" 67 | } # To make salt api called like system cmd: "salt -L node1,node2 ...." 68 | if script_args: 69 | if not isinstance(script_args, list): 70 | raise SaltRunnerUsageERROR(f"SaltRunner.go(): ERROR: `script_args` must be a list.") 71 | 72 | # 给包含空字符的参数，加上引号。 73 | args = '' 74 | for arg in script_args: 75 | _arg = str(arg) 76 | if re.search(r'\s', _arg): 77 | if not ((_arg.startswith('"') and _arg.endswith('"')) or (_arg.startswith("'") and _arg.endswith("'"))): 78 | if re.search(r'"', _arg) and re.search(r"'", _arg): 79 | raise Exception("SaltRunner.go(): ERROR: SaltRunner cannot handle complex script args which contains both `'` and `\"`.") 80 | elif re.search(r'"', _arg): 81 | _arg = f"'{_arg}'" 82 | else: 83 | _arg = f'"{_arg}"' 84 | args += _arg 85 | args += ' ' 86 | salt_api_action['arg'] = [salt_api_action['arg'], args] 87 | 88 | # 发起salt-api调用，获取调用结果 89 | res = self.saltapi_session.post(self.salt_api_url, salt_api_action, verify=False) 90 | if res.status_code != 200: 91 | if not self.quiet: 92 | print(res.text) 93 | raise Exception(f"SaltRunner.go(): ERROR: Failed to call salt-api. status_code: {res.status_code}.") 94 | self.result.update(res.json()['return'][0]) 95 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Django Action API 2 | 3 | ## 简介 4 | 5 | Action API是一个基于django的API框架，原生支持异步任务处理，不依赖celery这种三方组件。 6 | 7 | API封装了request与reponse处理过程，开发者只需写对应action的handler处理方法即可。 8 | 9 | API非RESTful设计，这是因为，设计这套框架的初衷，是为了在运维开发领域中使用。而运维开发，操作类偏多，资源类相对偏少，把所有操作抽象为资源的方式不够便捷、不够直观。故，这里把所有接口都抽象为一个action，都用POST方法通过传JSON来交互，每个action对应的一个handler处理函数。这样设计，更符合运维开发领域的思维逻辑。 10 | 11 | 除了运维开发领域的各类系统外，此框架也适用于其他各种业务后台管理系统的开发。 12 | 13 | 在核心模块之外，提供了作为管理系统常用的扩展模块，可插拔式启用（即，注册django的INSTALLED_APP的方式来启用，具体参见后文详细说明。） 14 | 15 | 最后，这是一个小巧、（核心代码）精简、灵活、开放的API框架，鼓励各位开发者在自己的项目中阅读、修改框架源码，并在此提出宝贵意见。 16 | 17 | 也可以给我发邮件：alan001lhl@163.com 18 | 19 | ## 主要特点 20 | 21 | * **原生支持异步** 22 | 23 | 集成了tornado异步非阻塞处理模块，支持高并发任务处理。采用`TCP C/S`模式处理异步任务，不依赖中间件broker，无需安装三方组件，即插即用，没有过于复杂的配置。 24 | 25 | 值得一提的是，django+celery的异步模式中常见的DB链接不释放的问题，在本框架的异步模块中，也得到完美解决（采用django原生方式释放DB连接，支持django的`CONN_MAX_AGE`全局配置参数）。 26 | 27 | * **原生提供定时任务模块** 28 | 29 | 集成timer定时任务模块，提供简易的静态定时任务功能，支持crontab与every两种方式，来运行定时任务。 30 | 31 | 也提供较复杂动态定时任务功能，把定时任务配置数据都入库，并记录最后一次执行的执行结果。除了crontab与every外，还支持一次性定时任务at_time类型。 32 | 33 | * **请求数据的递归校验** 34 | 35 | 对于请求的JSON数据参数，支持绝大多数场景的递归嵌套校验。 36 | 37 | * **灵活的数据序列化** 38 | 39 | Action API以Mixin Class的方式提供序列化工具，对常规的增、删、改、查数据操作，提供灵活的、分层的序列化支持。 40 | 41 | 分层封装，可按需从不同层级接入序列化工具，开发灵活。 42 | 43 | 另外，列表数据的序列化支持分页操作。 44 | 45 | * **提供接口权限管理控制模块** 46 | 47 | * **提供action请求记录模块，一般用于系统的操作审计** 48 | 49 | * **提供API的token管理模块** 50 | 51 | * **提供一系列运维开发领域的常用工具** 52 | 53 | ## 安装与配置 54 | 55 | ### 安装 56 | 57 | 将corelib目录整个放到django项目根目录即可。 58 | 59 | corelib依赖软件包如下： 60 | 61 | ```text 62 | Django 63 | mysqlclient 64 | tornado 65 | jsonfield 66 | # ansible # 如果需要用到corelib/tools/ansible_runner.py工具的话。 67 | ``` 68 | 69 | 关于python环境，请使用python3.6以上的版本; 70 | 71 | Django尽量使用2.0以上的版本。 72 | 73 | ### 配置 74 | 75 | corelib中的各个模块，支持一系列的配置选项，只需在django的全局settings.py中配置即可。 76 | 77 | 具体各模块儿支持的配置选项，请参考各个模块的`defaults.py`模块。 78 | 79 | ## 基础用法、核心模块 80 | 81 | Action API中以下三个模块为核心与基础： 82 | 83 | ```shell 84 | 85 | corelib/api_auth # 负责接口的认证工作 86 | corelib/api_base # 基础封装，请求数据字段校验类型的封装。 87 | corelib/api_serializing_mixin # 以mixin-class的方式提供一系列常规增删改查序列化工具。 88 | 89 | ``` 90 | 91 | 下面以一个示例django app，展示其用法。实现了对数据表的常规增、删、改、查操作的一组action接口。 92 | 93 | 代码结构： 94 | 95 | ```script 96 | some_django_app/ 97 | api.py 98 | handlers.py 99 | models.py 100 | urls.py 101 | ``` 102 | 103 | 包含序列化设置的models.py 104 | 105 | ```python 106 | from django.db import models 107 | from django.utils import timezone 108 | 109 | 110 | class HostENV(models.Model): 111 | id = models.AutoField('ID', primary_key=True) 112 | name = models.CharField('环境名称', max_length=32, default='') 113 | 114 | 115 | class CMDBHost(models.Model): 116 | id = models.AutoField('ID', primary_key=True) 117 | hostname = models.CharField('主机名', max_length=64, default='') 118 | hostip = models.CharField('主机IP', max_length=32, default='') 119 | env = models.ForeignKey(HostENV, on_delete=models.SET_NULL, related_name='hosts', null=True) 120 | status = models.CharField('主机状态', max_length=32, default='') 121 | create_time = models.DateTimeField('创建时间', default=timezone.now) 122 | 123 | # 序列化设置 124 | list_fields = [ 125 | 'id', 'hostname', 'hostip', # 直接取值的字段 126 | {'env': ['id', 'name']}, # 关系型字段的序列化，支持多级嵌套 127 | ] 128 | search_fields = ['hostname', 'hostip'] # 搜索设置。列表序列化器，将会在这两个字段中做模糊搜索，取并集。 129 | filter_fields = [ # 精确过滤字段，多个字段取交集。 130 | 'status', 131 | 'env.name', # 关系型用'.'来串联层级关系，支持多级串联。 132 | ] 133 | detail_fields = list_fields + ['create_time'] # 时间类型序列化时，将自动转换为对应格式的字符串，默认格式'%F %T'，支持自定义。 134 | 135 | # 若不提供list_fields与detail_fields，序列化时，默认展示所有字段。 136 | ``` 137 | 138 | 常规增、删、改、查的handlers.py示例 139 | 140 | ```python 141 | from corelib import APIHandlerBase, ChoiceType, pre_handler, StrType, IntType, ObjectType, IPType 142 | from corelib.api_serializing_mixins import ListDataMixin, AddDataMixin, DetailDataMixin, ModifyDataMixin, DeleteDataMixin 143 | from .models import HostENV, CMDBHost 144 | 145 | 146 | class HostGetHandler(APIHandlerBase, ListDataMixin, DetailDataMixin): 147 | post_fields = { # post数据自动校验设置 148 | 'search': StrType(), # 表示接受任意字符串 149 | 'env.name': StrType(), # list数据的filter设置。需跟models中的filter_fields保持一致。 150 | 'status': ChoiceType('running', 'stopped'), # list数据的filter设置。只能传递这两个值之一，否则校验返回失败 151 | 'id': ObjectType(model=CMDBHost), # 校验之后将得到一个db数据对象 152 | } 153 | 154 | @pre_handler(opt=['search', 'env.name', 'status']) 155 | def getHostList(self): # action处理函数 156 | self.getList(model=CMDBHost) 157 | 158 | @pre_handler(req=['id']) 159 | def getHostDetail(self): 160 | self.getDetail(model=CMDBHost) 161 | 162 | 163 | class HostWriteHandler(APIHandlerBase, AddDataMixin, ModifyDataMixin, DeleteDataMixin): 164 | post_fields = { # post数据自动校验设置 165 | 'hostname': StrType(regex='^H', min_length=16, max_length=32), # 必须以H开头，长度介于16到32的字符串 166 | 'hostip': IPType(), # IP格式的校验。 167 | 'env': ObjectType(model=HostENV), 168 | 'id': ObjectType(model=CMDBHost), 169 | } 170 | 171 | @pre_handler(req=['hostname', 'env'], opt=['hostip']) # req为post数据中必须提供的字段，opt为可选。不在这两个列表中的字段，将被自动忽略。 172 | def addHost(self): 173 | self.addData(model=CMDBHost) 174 | 175 | @pre_handler(req=['id']) 176 | def deleteHost(self): 177 | self.deleteData() 178 | 179 | @pre_handler(req=['id'], opt=['hostname', 'env', 'hostip']) 180 | def modiyHost(self): 181 | self.modifyData() 182 | 183 | # 所有的handler函数无需return，处理结果设置在handler的固有属性即可，比如： 184 | # 若不用序列化工具，可自行做ORM数据查询，然后设置`self.data`与`self.message`即可， 185 | # 关于这点，以及如何抛错，后面会有详细示例。 186 | # 187 | # pre_handler装饰器，集成了post数据自动校验、权限检查、action请求记录等功能， 188 | # 如果数据校验失败，在pre_handler中就会直接返回`status_code=400`的错误，action函数不会执行。 189 | # 若一个action接口不需要pre_handler中的这些功能，action方法也可不用这个装饰器。 190 | # 191 | # 一般将get和write做分开定义，方便做权限控制。权限控制请参考后续的示例代码。 192 | # 193 | # 更多字段校验格式请查阅源码: `corelib/api_base/api_field_types.py`。 194 | # 也可修改源码，自定义数据校验格式。 195 | # 注意：自定义字段校验格式需满足check约定。 196 | 197 | ``` 198 | 199 | 将定义好的handler与action处理函数注册到api.py中 200 | 201 | ```python 202 | from corelib import APIIngressBase 203 | from .handlers import HostGetHandler, HostWriteHandler 204 | 205 | 206 | class APIIngress(APIIngressBase): # 请求会自动根据此处的actions定义来分发到handler 207 | actions = { 208 | 'getHostList': HostGetHandler, # key需保持和实际的处理函数名称一至。 209 | 'getHostDetail': HostGetHandler, 210 | 'addHost': HostWriteHandler, 211 | 'deleteHost': HostWriteHandler, 212 | 'modiyHost': HostWriteHandler, 213 | } 214 | 215 | ``` 216 | 217 | 一个基本上固定不变的urls.py 218 | 219 | ```python 220 | from django.urls import path 221 | from .api import APIIngress 222 | 223 | urlpatterns = [ 224 | path('api/v1', APIIngress.as_view()), 225 | ] 226 | 227 | ``` 228 | 229 | 最后，别忘了在django的settings.py中添加此app，以及将此处的urls.py注册到django的全局urls.py中。 230 | 231 | 注意：一般一个django app只需在全局urls.py定义中，定义一个path即可。可有效避免过于复杂的url匹配规则，引起意料之外的错误。 232 | 233 | 以上即展示了基础模块的用法。 234 | 235 | 下面，对请求数据、返回数据，以及token管理，分别加以说明。 236 | 237 | ### 请求体数据格式与返回数据格式 238 | 239 | 采用POST方法请求，post数据采用json格式传递，并满足以下要求： 240 | 241 | ```python 242 | { 243 | "action": "getHost", # 对应的action 244 | # "auth_token": "xxxxxxxxxxxxxx", # 若提供，则走token认证，否则走session认证。 245 | # ...其他字段 246 | } 247 | ``` 248 | 249 | 特别说明：`auth_token`字段用于接口的token认证，无需用户登录。故可归类于外部接口。 250 | 若想定义某一个handler作为私有接口，即，必须有用户登录才能访问（也即是必须走session认证），可以通过`pre_handler`的`private`参数来设置，例如： 251 | 252 | ```python 253 | 254 | @pre_hander(private=True) 255 | def some_action(self): 256 | pass 257 | 258 | ``` 259 | 260 | 关于数据返回，若处理正确则返回一个JSON字典， status_code为200： 261 | 262 | ```python 263 | 264 | { 265 | "result": "SUCCESS", # Or "FAILED", 如果有内容上的错误。 266 | "data": ... # 任意数据 267 | "message": "一条文字消息" 268 | } 269 | 270 | ``` 271 | 272 | 如果出现了系统级别的错误，则会返回一个字符串，status_code将为4xx或5xx（可自定义）。 273 | 274 | 基础模块的可选配置参数，请分别参考以下defaults模块： 275 | 276 | ```shell 277 | 278 | corelib/api_auth/defaults.py # 接口认证相关配置。 279 | corelib/api_base/defaults.py # 绕过接口认证的相关配置。 280 | corelib/api_serializing_mixins/defaults.py # 目前只有list查询时，分页默认行为的相关配置。 281 | 282 | ``` 283 | 284 | ### token管理说明 285 | 286 | token分两种： 287 | 288 | * 静态token 289 | 290 | 直接写在配置文件`settings.py`中的token，通过参数`ACTION_STATIC_TOKENS`来指定，例如 291 | 292 | ``` python 293 | 294 | ACTION_STATIC_TOKENS = ['lalalalalalalalalalalallalalalalalalalallalal', 'test2222222222222222222'] 295 | 296 | ``` 297 | 298 | 静态token直接配置，不会入库，也没有user归属，可随意配置。 299 | 300 | 一般用于测试，不建议在生产环境中使用静态token。 301 | 302 | * 动态token 303 | 304 | api_auth模块中集成了一个动态token管理工具，将token数据记录在数据库中，并可通过命令行，或者API，动态的管理token。 305 | 306 | 动态token是一个字符串，由两部分组成： 307 | 308 | ```text 309 | .<64位随机码> 310 | 311 | 随机码由系统自动生成，不会包含字符‘.’ 312 | ``` 313 | 314 | 这两个字段分别对应数据库中的两个字段，做存储。 315 | 316 | 这样，可根据此处的`username`来关联真实用户，做权限控制。 317 | 318 | 命令行的使用，请查看help文档： 319 | 320 | ```shell 321 | 322 | python3 corelib/api_auth/bin/token_manager --help 323 | 324 | ``` 325 | 326 | api的启用，需要注册django项目settings的`INSTALLED_APPS`: 327 | 328 | ```python 329 | INSTALLED_APPS = [ 330 | ... 331 | 'corelib.api_auth.token_api', 332 | ] 333 | ``` 334 | 335 | 并注册url： 336 | 337 | ```python 338 | from django.urls import path, include 339 | 340 | urlpatterns = [ 341 | path('token-manager/', include('corelib.api_auth.token_api.urls')) 342 | ] 343 | ``` 344 | 345 | 提供固定URI: `/token-manager/api/v1` 346 | 347 | 提供token的增删改查actions，请参考模块: `corelib/api_auth/token/api.py` 348 | 349 | ## 异步任务 350 | 351 | 由于django是个同步框架，无法直接在view函数中实现异步逻辑，仍然需要借助一个独立进程来跑异步任务。 352 | 353 | Action API异步模块提供一个独立运行的异步服务，通RPC调用来执行异步任务。 354 | 355 | 异步模块，主要用于执行一些耗时长的阻塞任务，在tornado的ioloop中，将放到线程类executor中执行。 356 | 357 | 异步模块: `corelib.asynctask` 358 | 359 | 异步服务程序：`corelib/asynctask/bin/asynctask_server` 360 | 361 | ### 一个简单示例 362 | 363 | 异步代码结构要求： 364 | 365 | ```script 366 | some_django_app/ 367 | api.py 368 | asynctasks.py # 异步任务模块，模块名称可通过配置项`ASYNCTASK_REGISTER_MODULE`在settings中设定 369 | handlers.py 370 | models.py 371 | urls.py 372 | ``` 373 | 374 | 一个没啥用的异步任务，仅用做示例（asynctasks.py模块内容）: 375 | 376 | ```python 377 | from corelib.asynctask import asynctask 378 | from corelib.tools.logger import Logger 379 | import time 380 | 381 | @asynctask # 注册为一个异步任务，异步服务启动时，将以此为基准来自动注册RPC调用函数 382 | def test_task(task_id): 383 | logger = Logger(trigger_level='INFO', msg_prefix='[asynctask] test_task: ') # 一个简易的日志工具 384 | logger.log('start...') 385 | time.sleep(5) # 运行一个阻塞任务 386 | print(task_id) 387 | logger.log('end!') 388 | 389 | ``` 390 | 391 | 在handler中执行异步任务 392 | 393 | ```python 394 | from .asynctask import test_task 395 | 396 | class AsyncAPIHandler(APIHandlerBase): 397 | post_fields = {'task_id': IntType(min=0)} 398 | 399 | @pre_handler(req=['task_id']) 400 | def asyncAction(self): 401 | task_id = self.checked_params['task_id'] # 校验之后的参数，将被填充到self.checked_params字典中。 402 | err = test_task.delay(task_id=task_id) # delay方法将发起RPC调用，故，只能传可序列化的参数。 403 | if err is not None: 404 | return self.error(err) # 表示启动异步任务出错，self.error()方法自动设置处理结果为错误，status_code默认为400 405 | self.message = '异步任务已启动' 406 | 407 | ``` 408 | 409 | 启动异步服务 410 | 411 | ```script 412 | 413 | cd /path/to/your_django_project/ 414 | 415 | python3 corelib/asynctask/bin/asynctask_server 416 | 417 | # 启动过程中，异步服务会自动注册各APP，`asynctasks.py`中定义的异步任务处理函数。 418 | ``` 419 | 420 | ### 延迟调用 421 | 422 | delay()方法支持固有参数delaytime，用于名副其实的延迟调用。 423 | 424 | 延迟sleep在异步模块中执行，delay()方法本身不阻塞。 425 | 426 | ```python 427 | 428 | ... 429 | err = test_task.delay(task_id=task_id, delaytime=10) # 表示10秒后再执行异步任务。 430 | ... 431 | 432 | ``` 433 | 434 | 可以用此方法，来执行一些一次性的定时任务。 435 | 436 | ```python 437 | from django.utils import timezone 438 | ... 439 | run_at = 440 | delaytime = int(run_at.timestamp() - timezone.now().timestamp()) 441 | err = test_task.delay(task_id=task_id, delaytime=delaytime) 442 | ... 443 | 444 | ``` 445 | 446 | 如果需要固定的delay时间，还可以这样设置 447 | 448 | ```python 449 | @asynctask(delaytime=10) # 设置默认延迟10秒（而不是调用时才指定）。 450 | def your_async_func(): 451 | ... 452 | 453 | ``` 454 | 455 | 但是，最好不要利用这个特性来做一些长时的延迟调用（延迟调用时间建议不要超过1分钟）。如果有长时延迟调用需求，请使用后面介绍的“定时任务”模块。 456 | 457 | 因为，这里仅仅是依赖asynctask-server进程做简单的sleep等待。若等待期间asynctask-server进程被重启，则调用最终就不会被执行。 458 | 459 | ### 任务状态跟踪、执行结果记录 460 | 461 | `corelib.asynctask.async_api`提供一张表，与一组action接口，可将任务执行过程与结果记录到数据库中，并可通过固有API来做查询。 462 | 463 | 要启用此功能，需要将`corelib.asynctask.async_api`注册到django settings.py中的`INSTALLED_APPS`中: 464 | 465 | ```python 466 | INSTALLED_APPS = [ 467 | ... 468 | corelib.asynctask.async_api, 469 | ... 470 | ] 471 | ``` 472 | 473 | 以及全局urls.py中: 474 | 475 | ```python 476 | from django.urls import path, include 477 | 478 | urlpatterns = [ 479 | ... 480 | path('asynctask/', include('corelib.asynctask.async_api.urls')], 481 | ``` 482 | 483 | 然后，`asynctask`装饰器通过`tracking`参数来启用记录功能。 484 | 485 | ```python 486 | 487 | @asynctask(tracking=True) 488 | def your_async_func(): 489 | ... 490 | 491 | ``` 492 | 493 | 若是第一次添加，别忘了执行数据库的`migrate`。 494 | 495 | 最后，（按照之前的配置）可以通过以下内置API来管理异步任务结果： 496 | 497 | 固定URL: `/asynctask/api/v1` 498 | 499 | 支持的action，请参考api模块: `corelib/asynctask/api/api.py` 500 | 501 | 支持的配置参数，请参考defaults模块：`corelib/asynctask/lib/defaults.py` 502 | 503 | ## 定时任务 504 | 505 | 根据配置数据入不入库，将定时任务分为两类： 506 | 507 | * 静态任务 508 | 509 | 定时任务配置直接在编码时固定写死，配置数据不入库。 510 | timer服务启动后立即开始按配置执行。 511 | 修改定时配置，必须重启timer进程。 512 | 513 | * 动态任务 514 | 515 | 定时任务配置数据写入数据库，可通过内置API动态修改（也可开发配套的前端页面做常规增删改查）。 516 | 启动timer服务是，代码中标注为动态任务的任务函数，只是可配置的task模块，需要通过API进一步配置时间参数，才能开始执行。 517 | 动态任务数据更新后，需等待timer做定时同步（即，加载数据库中的定时任务数据），默认同步周期为1分钟。 518 | 这意味着，修改了动态任务数据，会在1分钟内自动生效（不会立即生效），无需重启timer进程。 519 | 如果想立即生效，也可以通过重启timer进程来实现。 520 | 521 | 定时任务可以有三种执行方式： 522 | 523 | * every 524 | 525 | 周期执行，指定执行间隔，单位为秒，最小间隔1秒，0表示不启用。 526 | 527 | * crontab 528 | 529 | 类似linux的crontab字符串，拥有7个字段： 530 | 531 | ```shell 532 | * * * * * * * 533 | 秒 分 时 日 月 周 年 534 | ``` 535 | 536 | * at_time 537 | 538 | 在指定时间，执行一次的定时任务。 539 | 540 | 仅动态任务支持这种方式，静态任务不支持。 541 | 542 | 下面分别举例说明。 543 | 544 | ### 静态任务 545 | 546 | 代码结构（类似asynctask模块）： 547 | 548 | ```script 549 | some_django_app/ 550 | api.py 551 | asynctasks.py 552 | timer.py # 异步任务模块，模块名称可通过配置项`TIMER_REGISTER_MODULE`在settings中设定 553 | handlers.py 554 | models.py 555 | urls.py 556 | ``` 557 | 558 | 编码示例： 559 | 560 | ```python 561 | from corelib.timer.lib import cron 562 | from corelib.tools.logger import Logger 563 | 564 | 565 | # 每5秒执行一次 566 | @cron(every=5) 567 | def a_test_task(): 568 | logger = Logger(msg_prefix='a_test_task(): ') 569 | logger.log("start...") 570 | time.sleep(1) 571 | logger.log("end!") 572 | 573 | 574 | # 2021年，每天凌晨2点开始执行（其他年份自动失效） 575 | @cron(crontab="0 0 2 * * * 2021") 576 | def crontab_task(): 577 | logger = Logger(msg_prefix='crontab_task(): ') 578 | logger.log("start...") 579 | time.sleep(10) 580 | logger.log("end!") 581 | 582 | 583 | # 表示当时间的描述为0 20 40时开始执行。 584 | # 注意'*/'写法跟every不一样： 585 | # 这里，表示当前时间位的数值，对取余为0时才开始执行。 586 | # 故，若在秒位写上'*/40'，表示秒数在’0 40'时执行; 587 | # 如果写上'*/70'，则只能秒数只在0时执行。 588 | # 其他时间位，依次类推 589 | @cron(crontab="*/20 * * * * * *") 590 | def crontab_task(): 591 | logger = Logger(msg_prefix='crontab_task(): ') 592 | logger.log("start...") 593 | time.sleep(10) 594 | logger.log("end!") 595 | 596 | ``` 597 | 598 | 值得一提的是，静态任务函数不可定义参数，不能传参（故，叫静态），也无法记录执行结果，只能在日志中查看执行成功还是失败。若要传参，以及记录执行结果，请使用动态任务。 599 | 600 | 最后，启动timer服务进程 601 | 602 | ```shell 603 | cd /path/to/your/django_project/ 604 | 605 | python3 corelib/timer/bin/timer_server 606 | ``` 607 | 608 | ### 动态任务 609 | 610 | 动态任务要求先在django的`INSTALLED_APPS`中注册timer的内置api: 611 | 612 | ```python 613 | INSTALLED_APPS = [ 614 | ... 615 | 'corelib.timer.timer_api', 616 | ] 617 | 618 | ``` 619 | 620 | 以及内置URL： 621 | 622 | ```python 623 | from django.urls import path, include 624 | 625 | urlpatterns = [ 626 | ... 627 | path('timer/', include('corelib.timer.timer_api.urls')) 628 | ] 629 | 630 | ``` 631 | 632 | 别忘记做migration，来创建动态任务用的数据表。 633 | 634 | 代码组织结构、启动方式都跟静态任务一致，可跟静态任务写在同一个模块文件中。 635 | 636 | 编码示例： 637 | 638 | ```python 639 | 640 | ... 641 | 642 | @cron(dynamic=True) 643 | def unusable_task(foo, bar=None): 644 | ''' 645 | 这是一个测试用的动态任务 646 | 647 | 参数： 648 | 649 | foo 字符串，必填。 650 | bar 字符串，可选。默认为None。 651 | ''' 652 | logger = Logger(msg_prefix='unusable_task(): ') 653 | logger.log('start...') 654 | print(foo, bar) 655 | time.sleep(4) 656 | logger.log('end!') 657 | 658 | # 注意：目前动态任务仍然不会记录任务函数的return值（考虑到对于return值，做序列化的复杂性，当前版本暂时不做此项支持）。 659 | # 数据库中记录的所谓“执行结果”，仅仅指的是任务执行成功了，还是失败了。以及其他各项任务执行结果指标（具体请通过API查看） 660 | ``` 661 | 662 | 这样，当timer_server启动时，便会把动态任务函数，注册到DB中，供用户做进一步配置。 663 | 664 | 特别注意： 665 | 这里的动态任务函数，不是一个具体的可执行的任务，它更应当看做是一个任务模板，可以基于此函数来配置具体的动态任务。 666 | 建议每一个动态任务，都编写文档说明，这样，用户在创建具体的动态任务时，才能知道如何给函数传参。 667 | 668 | 当前以注册了哪些动态任务（模板），可以通过API查看（我们编写了一个接口测试脚本）： 669 | 670 | ```python 671 | import requests 672 | 673 | 674 | def call(action, data=None): 675 | _data = { 676 | 'action': action, 677 | 678 | # 要求在django的settings.py中设定一个静态token 679 | # ACTION_STATIC_TOKENS = ['lalalalalalalalalalalallalalalalalalalallalal'] 680 | # 具体，请参考token管理说明。 681 | 'auth_token': 'lalalalalalalalalalalallalalalalalalalallalal', 682 | } 683 | if data: 684 | _data.update(data) 685 | res = requests.post(url='http://127.0.0.1:8888/timer/api/v1', json=_data) # 假设django的web服务起本地的8888端口 686 | 687 | print(f"status_code: {res.status_code}") 688 | print(f"data: {res.json()}") 689 | 690 | 691 | # action: getAvailableCronList 692 | # 用于返回，当前可用的动态任务（模板）函数，以及其文档说明 693 | def get_availables(): 694 | call('getAvailableCronList', {}) 695 | 696 | 697 | if __name__ == '__main__': 698 | get_availables() 699 | 700 | ``` 701 | 702 | 然后便可使用已注册的动态任务（模板）来定义具体的动态任务： 703 | 704 | ```python 705 | 706 | # action: addCron 707 | # 新建一个名为test00001的动态定时任务。每5分钟执行一次。 708 | def add_task(): 709 | data1 = { 710 | 'name': 'test00001', 711 | 'description': '第一个测试任务', 712 | 'task': 'testapp.timer.unusable_task', # 动态任务模块（模板） 713 | 'args': ['SRE'], # 传给参数foo 714 | 'kwargs': {'bar': '666'}, # 传给参数bar 715 | 'every': 60 * 5 716 | } 717 | call('addCron', data1) 718 | 719 | # 成功添加后，timer_server会自动加载新的任务，并开始按计划执行。无需重启timer_server 720 | 721 | 722 | # action: getCronList 723 | # 用户获取实际的动态任务列表 724 | def get_tasks(): 725 | call('getCronList', {}) 726 | 727 | 728 | # action: disableCron 729 | # 禁用任务 730 | def disable_task(): 731 | data = {'id': 1} 732 | call('disableCron', data) 733 | 734 | 735 | # action: enableCron 736 | # 启用任务 737 | def enable_task(): 738 | data = {'id': 1} 739 | call('enableCron', data) 740 | 741 | 742 | # action: modifyCron 743 | # 限制只能运行100次，之后自动失效。 744 | def modify_task(): 745 | data1 = { 746 | 'id': 1, 747 | 'expired_count': 100, # 0表示无限制 748 | } 749 | call('modifyCron', data1) 750 | 751 | 752 | # action: modifyCron 753 | # 直接指定失效时间，过期自动失效。 754 | def modify_task(): 755 | data1 = { 756 | 'id': 1, 757 | 'expired_count': 0, # 0表示无限制 758 | 'expired_time': '2021-05-10 00:00:00', 759 | } 760 | call('modifyCron', data1) 761 | 762 | # expired_time与expired_count同时设定时，谁先达到即已谁为准。 763 | 764 | # action: modifyCron 765 | # 修改任务为一个at_time类型的任务。 766 | # at_time类型的任务，会在指定时间点执行一次，然后立即失效。 767 | # 如果执行时间，早于当前时间，timer进程加载到新配置后，会立即执行。 768 | def modify_task(): 769 | data1 = { 770 | 'id': 1, 771 | 'every': 0, # 0 表示取消every设定 772 | 'crontab': '', # 空字符表示取消crontab设定。 773 | 'at_time': '2021-05-07 07:03:00', # 当at_time不为空，切every，crontab都没有设置时，表示是一个at_time任务。 774 | } 775 | call('modifyCron', data1) 776 | 777 | 778 | # action: renewAtTimeTask 779 | # 当一个at_time任务执行结束自动失效后，此action可用来重置一个at_time任务。 780 | # 重置时，不会自动启用这个任务，别忘记重新启用此任务。 781 | def renew(): 782 | data = { 783 | 'id': 1, 784 | 'at_time': '2021-05-07 07:11:05' # 可选参数。若不提供，则表示扔采用原来的时间。 785 | } 786 | 787 | call('renewAtTimeTask', data) 788 | ``` 789 | 790 | 固定URI: `/timer/api/v1` 791 | 792 | action请参考api模块: `corelib/timer/api/api.py` 793 | 794 | timer支持的配置选项请参考defaults模块: `corelib/timer/lib/defaults.py` 795 | 796 | ## 基于Action的权限检查 797 | 798 | Action API框架通过`corelib.permission`模块，可以对每个定义的action做自动化的请求权限检查与校验。 799 | 800 | 考虑到权限方面的复杂性，Action API框架仅将权限按组划分，每个组代表一个权限等级，等级高低由等级序号判定，序号越大权限越高。 801 | 802 | 内置以下两个权限组： 803 | 804 | ```python 805 | PERMISSION_GROUPS = { 806 | "admin": 2, 807 | "normal": 1, 808 | } 809 | ``` 810 | 811 | 若应用程序遵循此逻辑，需要增加其他权限组，请在django的全局设置settings.py中设置配置项`PERMISSION_GROUPS`。 812 | 813 | 若有更复杂的权限需求，请自行定制。 814 | 815 | 要启用此模块，请在django settings.py中启用此app: 816 | 817 | ```python 818 | INSTALLED_APPS = [ 819 | ... 820 | corelib.permission, 821 | ... 822 | ] 823 | ``` 824 | 825 | 如此，便可在装饰器`pre_handler`中，启用权限检查。 826 | 827 | 默认权限组配置，实际上将权限切割为三个等级，如下： 828 | 829 | ```python 830 | ... 831 | @pre_handler(opt=['search', 'group.name', 'coding_type', 'page_length', 'page_index']) 832 | def getHostList(self): # 无需权限检查，所有用户（包括没有明确配置权限组的用户）都可访问。 833 | self.getList(model=CMDBHost) 834 | 835 | @pre_handler(req=['id'], perm='normal') # 仅允许normal与admin组的用户访问 836 | def getHostDetail(self): 837 | self.getDetail(model=CMDBHost) 838 | ... 839 | 840 | @pre_handler(req=['hostname', 'env'], opt=['hostip'], perm='admin') # 仅允许admin组的用户访问 841 | def addHost(self): 842 | self.addData(model=CMDBHost) 843 | ... 844 | ``` 845 | 846 | 当权限检查不通过时，Action API会返回403错误。 847 | 848 | permission模块，也提供一组内置API，注册全局urls.py便可启用: 849 | 850 | ```python 851 | from django.urls import path, include 852 | 853 | ... 854 | 855 | urlpatterns = [ 856 | ... 857 | path('permission/', include('corelib.permission.urls')], 858 | ... 859 | ``` 860 | 861 | 别忘记做数据库的migration. 862 | 863 | 固定URI: `/permission/api/v1` 864 | 865 | action请参考api模块：`corelib/permission/api.py` 866 | 867 | 支持的配置选项，请参考defaults模块： `corelib/permission/defaults.py` 868 | 869 | ## 基于Action的请求记录 870 | 871 | Action API框架可通过`corelib.record`模块来对action的每一次请求做记录、入库，用作操作审计，用以监管用户的操作行为。 872 | 873 | 注意，此功能适用于请求量较小、操作会产生一定影响的API。对于请求量较大的API，请不要启用此功能，会压垮数据库。 874 | 875 | 同样需要注册APP来启用： 876 | 877 | ```python 878 | INSTALLED_APPS = [ 879 | ... 880 | corelib.recorder, 881 | ... 882 | ] 883 | ``` 884 | 885 | 注册内置API相关URL： 886 | 887 | ```python 888 | urlpatterns = [ 889 | ... 890 | path('record/', include('corelib.recorder.urls')), 891 | ... 892 | ] 893 | 894 | ``` 895 | 896 | 然后数据库做migrate。 897 | 898 | 如此，便可在装饰器`pre_handler`中，启用操作请求记录： 899 | 900 | ```python 901 | ... 902 | @pre_handler(opt=['search', 'group.name', 'coding_type', 'page_length', 'page_index']) 903 | def getHostList(self): # 默认不做记录 904 | self.getList(model=CMDBHost) 905 | ... 906 | 907 | @pre_handler(req=['hostname', 'env'], opt=['hostip'], record=True, record_label="新增host") 908 | def addHost(self): # 启用请求操作记录，record_label用于对action添加可读描述。不指定则为空。 909 | self.addData(model=CMDBHost) 910 | ... 911 | ``` 912 | 913 | 内置API的固定URI：`/record/api/v1` 914 | 915 | 支持的actions请参考模块：`corelib/recorder/api.py` 916 | 917 | ## 其他说明 918 | 919 | 最后，关于代码风格，附上corelib在开发过程中的，vscode中Python编码配置： 920 | 921 | ```json 922 | "python.linting.enabled": true, 923 | "python.linting.flake8Enabled": true, 924 | "python.linting.flake8Args": [ 925 | "--max-line-length=160", 926 | "--ignore=E402,F403,F405,W503,E126,E902", 927 | ], 928 | ``` 929 | 930 | 这样，阅读corelib代码时，vscode显示就会很干净。 931 | --------------------------------------------------------------------------------