├── CONTRIBUTORS.txt ├── ank ├── utils │ ├── __init__.py │ ├── config_handle.py │ ├── logger.py │ ├── cmd_helpers.py │ ├── naming_services.py │ ├── api_helpers.py │ └── crontab_time.py ├── components │ ├── __init__.py │ ├── split_app.py │ ├── log_app.py │ ├── join_app.py │ ├── kafka_producer.py │ ├── kafka_consumer.py │ ├── redis_subscribe.py │ ├── zmq_producer.py │ ├── zmq_consumer.py │ ├── rabbitmq_consumer.py │ ├── schedule_app.py │ ├── rabbitmq_producer.py │ ├── pipe_app.py │ └── api_app.py ├── templates │ ├── baseapp_settings.tpy │ ├── apiapp_settings.tpy │ ├── __init__.py │ ├── schedule_settings.tpy │ ├── baseapp_services.tpy │ ├── apiapp_services.tpy │ ├── schedule_services.tpy │ ├── readme.tpy │ ├── baseapp_processor.tpy │ ├── apiapp_processor.tpy │ ├── apiapp_endpoint.tpy │ ├── docker.tpy │ ├── scheduleapp_processor.tpy │ └── unittest.tpy ├── tests │ ├── __init__.py │ ├── settings.yml │ ├── services.yml │ ├── test_deploy.py │ └── processor.py ├── __init__.py ├── generate_setting.py ├── program_loader.py ├── chain_process.py ├── generate_processor.py ├── cli.py └── daemon.py ├── examples ├── __init__.py ├── schedule_app │ ├── requirements.txt │ ├── __init__.py │ ├── settings.yml │ ├── services.yml │ └── processor.py ├── api_app │ ├── __init__.py │ ├── requirements.txt │ ├── settings.yml │ ├── services.yml │ ├── processor.py │ └── endpoint.py └── streaming_app │ ├── requirements.txt │ ├── __init__.py │ ├── settings.yml │ ├── services.yml │ └── processor.py ├── MANIFEST.in ├── requirements.txt ├── TODO ├── .gitignore ├── CHANGES.md ├── Dockerfile ├── LICENSE ├── setup.py └── README.md /CONTRIBUTORS.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ank/utils/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md CHANGES.md LICENSE -------------------------------------------------------------------------------- /ank/components/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /ank/templates/baseapp_settings.tpy: -------------------------------------------------------------------------------- 1 | parameters: -------------------------------------------------------------------------------- /ank/tests/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | -------------------------------------------------------------------------------- /examples/schedule_app/requirements.txt: -------------------------------------------------------------------------------- 1 | ank>=1.5.5 -------------------------------------------------------------------------------- /examples/api_app/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /examples/streaming_app/requirements.txt: -------------------------------------------------------------------------------- 1 | ank>=1.5.5 -------------------------------------------------------------------------------- /examples/api_app/requirements.txt: -------------------------------------------------------------------------------- 1 | ank>=1.5.5 2 | pymongo -------------------------------------------------------------------------------- /examples/schedule_app/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /examples/streaming_app/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' -------------------------------------------------------------------------------- /ank/templates/apiapp_settings.tpy: -------------------------------------------------------------------------------- 1 | parameters: 2 | host: localhost 3 | port: {0} -------------------------------------------------------------------------------- /ank/templates/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import os 5 | TEMPLATES_PATH = os.getcwd() -------------------------------------------------------------------------------- /ank/templates/schedule_settings.tpy: -------------------------------------------------------------------------------- 1 | parameters: 2 | crontab_time: '0 0 * * 5' 3 | start_now: True -------------------------------------------------------------------------------- /examples/api_app/settings.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | 3 | mongo_host: localhost 4 | mongo_port: 27017 5 | -------------------------------------------------------------------------------- /ank/tests/settings.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | 3 | range_from: 20 4 | range_to: 100 5 | 6 | batch_size: 2 -------------------------------------------------------------------------------- /examples/schedule_app/settings.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | 3 | crontab_time: '0 0 * * 5' 4 | start_now: True -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml==5.4 2 | redis==4.5.4 3 | flask==2.3.2 4 | pika==0.10.0 5 | pyzmq==16.0.2 6 | kafka-python==0.9.5 -------------------------------------------------------------------------------- /ank/templates/baseapp_services.tpy: -------------------------------------------------------------------------------- 1 | services: 2 | {0}: 3 | class: processor.{0} 4 | arguments: ~ 5 | 6 | 7 | chains: 8 | - {0} -------------------------------------------------------------------------------- /ank/templates/apiapp_services.tpy: -------------------------------------------------------------------------------- 1 | services: 2 | {0}: 3 | class: processor.{0} 4 | arguments: ['%host%', '%port%'] 5 | 6 | 7 | chains: 8 | - {0} -------------------------------------------------------------------------------- /ank/templates/schedule_services.tpy: -------------------------------------------------------------------------------- 1 | services: 2 | {0}: 3 | class: processor.{0} 4 | arguments: ['%crontab_time%', '%start_now%'] 5 | 6 | 7 | chains: 8 | - {0} -------------------------------------------------------------------------------- /examples/streaming_app/settings.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | 3 | redis_host: localhost 4 | redis_port: 6379 5 | 6 | range_from: 20 7 | range_to: 100 8 | 9 | batch_size: 2 -------------------------------------------------------------------------------- /examples/schedule_app/services.yml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | ScheduleExample: 4 | class: processor.ScheduleExample 5 | arguments: ['%crontab_time%', '%start_now%'] 6 | 7 | 8 | chains: 9 | - ScheduleExample -------------------------------------------------------------------------------- /ank/templates/readme.tpy: -------------------------------------------------------------------------------- 1 | ## {0}: 2 | Service descriptions 3 | 4 | 5 | # generator: 6 | - generate template settings.yml: `ank -s` 7 | - generate _processor.py: `ank -p` 8 | 9 | # deploy: 10 | - build: `ank -b` 11 | - test: `ank -t -f test-settings.yml` 12 | - run: `ank -r` -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | [x] base apps: Base, API, Schedule 2 | [x] supported chain: join, split 3 | [x] supported consumer/producer: redis, kafka, zmq, rabbitmq 4 | [x] generate: processor.py, settings.yml 5 | [ ] management/admin 6 | [ ] build/deploy 7 | [x] sample: PipeApp, ScheduleApp, APIApp 8 | [x] test -------------------------------------------------------------------------------- /examples/api_app/services.yml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | ExampleApp: 4 | class: processor.ExampleApp 5 | arguments: [$MongoClient] 6 | 7 | MongoClient: 8 | class: pymongo.MongoClient 9 | arguments: ['%mongo_host%', '%mongo_port%'] 10 | 11 | chains: 12 | - ExampleApp -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python modules. 2 | .idea 3 | *.DS_Store 4 | *.pyc 5 | 6 | # Setuptools distribution folder. 7 | src/* 8 | env/* 9 | venv/* 10 | build/* 11 | dist/* 12 | 13 | # Examples 14 | *.log 15 | examples/*/worker/* 16 | 17 | # Python egg metadata, regenerated from source files by setuptools. 18 | *.egg-info 19 | *.egg -------------------------------------------------------------------------------- /ank/templates/baseapp_processor.tpy: -------------------------------------------------------------------------------- 1 | from ank.components.app import BaseApp 2 | 3 | 4 | class {0}(BaseApp): 5 | 6 | def __init__(self, agrs, **kwagrs): 7 | super({0}, self).__init__() 8 | 9 | def run(self, process=None): 10 | super({0}, self).run(process) 11 | 12 | def process(message=None): 13 | return message 14 | -------------------------------------------------------------------------------- /ank/templates/apiapp_processor.tpy: -------------------------------------------------------------------------------- 1 | from endpoint import {0}Endpoint 2 | from ank.components.pipe_app import PipeApp 3 | 4 | 5 | class {0}(PipeApp): 6 | 7 | def __init__(self, agrs, **kwagrs): 8 | super({0}, self).__init__() 9 | 10 | def run(self, process=None): 11 | api_app = {0}Endpoint(host='localhost', port={1}) 12 | api_app.run() 13 | -------------------------------------------------------------------------------- /ank/components/split_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class SplitApp(PipeApp): 8 | 9 | def init_app(self): 10 | pass 11 | 12 | def start(self): 13 | self.logger.info('Start {}'.format(self.__class__.__name__)) 14 | 15 | def process(self, message=None): 16 | return message 17 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | - 2015/10/15 -- Initial project 2 | - 2016/06/21 -- Public project 3 | - 2016/06/25 -- Upload to pip 4 | - 2016/06/29 -- Add kafka 5 | - 2016/06/30 -- Add zeroMQ 6 | - 2016/10/04 -- Support python3 7 | - 2016/12/17 -- Generate template, docker 8 | - 2017/03/20 -- Using pika for rabbitMQ 9 | - 2017/03/22 -- Support branching 10 | - 2018/06/20 -- Refactor, edit example 11 | -------------------------------------------------------------------------------- /ank/components/log_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class LogApp(PipeApp): 8 | 9 | def init_app(self): 10 | pass 11 | 12 | def start(self): 13 | self.logger.info('Start {}'.format(self.__class__.__name__)) 14 | 15 | def process(self, message=None): 16 | self.logger.info(message) 17 | return message 18 | -------------------------------------------------------------------------------- /ank/templates/apiapp_endpoint.tpy: -------------------------------------------------------------------------------- 1 | from ank.components.api_app import APIApp 2 | 3 | 4 | class {0}Endpoint(APIApp): 5 | 6 | def __init__(self, host='localhost', port={1}): 7 | super({0}Endpoint, self).__init__(host=host, port=port) 8 | 9 | def hello(self, params): 10 | # host:port/hello?name=YourInput 11 | your_input = params.get('name') 12 | return {'message': 'Hello: %s' % (your_input)} 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM sunary/python-2.7-alpine:0.1 3 | MAINTAINER Nhat Vo Van "v2nhat@gmail.com" 4 | 5 | #addition apk for image 6 | RUN apk --update add py-pip libffi-dev openssl-dev 7 | RUN apk --update add gettext gcc libpq python-dev git && rm -rf /var/cache/apk/* 8 | 9 | RUN pip install --upgrade pip 10 | 11 | RUN mkdir -p /srv/logs 12 | WORKDIR /srv/ank 13 | RUN pip install -r requirements.txt 14 | 15 | ADD . ./ 16 | ENTRYPOINT [] 17 | CMD [] -------------------------------------------------------------------------------- /examples/api_app/processor.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from endpoint import ExampleAPI 5 | from ank.components.pipe_app import PipeApp 6 | 7 | 8 | class ExampleApp(PipeApp): 9 | 10 | def __init__(self, agrs, **kwagrs): 11 | super(ExampleApp, self).__init__() 12 | 13 | def start(self): 14 | api_app = ExampleAPI(host='localhost', port=5372, mongo_client=self.mongo, mongo_db='demo') 15 | api_app.run() 16 | -------------------------------------------------------------------------------- /ank/templates/docker.tpy: -------------------------------------------------------------------------------- 1 | FROM alpine:3.3 2 | MAINTAINER Developer "name@company.com" 3 | 4 | #addition apk for image 5 | RUN apk --update add py-pip libffi-dev openssl-dev 6 | RUN apk --update add gettext gcc libpq python-dev git && rm -rf /var/cache/apk/* 7 | 8 | RUN pip install --upgrade pip 9 | 10 | RUN mkdir -p /srv/logs 11 | RUN mkdir -p /srv/{0} 12 | ADD . /srv/{0} 13 | WORKDIR /srv/{0} 14 | RUN pip install -r requirements.txt 15 | 16 | ENTRYPOINT [] 17 | CMD [] -------------------------------------------------------------------------------- /ank/utils/config_handle.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import yaml 5 | 6 | 7 | def load(file_name, object_name): 8 | with open(file_name, 'r') as fo: 9 | service_loader = yaml.load(fo) 10 | return service_loader[object_name] 11 | 12 | 13 | def save(file_name, key, data): 14 | output = yaml.dump({key: data}, default_flow_style=False) 15 | with open(file_name, 'w') as of: 16 | of.write(output) 17 | 18 | return output 19 | 20 | -------------------------------------------------------------------------------- /examples/schedule_app/processor.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.schedule_app import ScheduleApp 5 | from datetime import datetime 6 | 7 | 8 | class ScheduleExample(ScheduleApp): 9 | 10 | def init_app(self, crontab_time='', start_now=False): 11 | super(ScheduleExample, self).init_app(crontab_time, start_now) 12 | 13 | def start(self): 14 | pass 15 | 16 | def process(self, messages=None): 17 | print('Now is {}'.format(datetime.now())) -------------------------------------------------------------------------------- /ank/templates/scheduleapp_processor.tpy: -------------------------------------------------------------------------------- 1 | from ank.components.schedule_app import ScheduleApp 2 | from datetime import datetime 3 | 4 | 5 | class {0}(ScheduleApp): 6 | 7 | def __init__(self, crontab_time='', start_now=False, **kwargs): 8 | super({0}, self).__init__(crontab_time=crontab_time, start_now=start_now) 9 | 10 | def run(self, process=None): 11 | super(ScheduleApp, self).run(process) 12 | 13 | def process(self, message=None): 14 | print('Now is %s' % (datetime.now())) 15 | -------------------------------------------------------------------------------- /ank/components/join_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | from ank.chain_process import CONTENT_KEY 6 | 7 | 8 | class JoinApp(PipeApp): 9 | """ 10 | Join messages from previous processor 11 | """ 12 | 13 | def init_app(self, batch_size=None): 14 | self.batch_size = batch_size 15 | self.stored_messages = {CONTENT_KEY: []} 16 | 17 | def start(self): 18 | self.logger.info('Start {}'.format(self.__class__.__name__)) 19 | 20 | def process(self, message=None): 21 | return message 22 | -------------------------------------------------------------------------------- /ank/components/kafka_producer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class KafkaAnkProducer(PipeApp): 8 | """ 9 | Push message to queue 10 | """ 11 | 12 | def init_app(self, producer=None, topic=''): 13 | """ 14 | Args: 15 | producer (kafka.KafkaProducer): kafka producer 16 | topic (string): kafka subscriber topic 17 | """ 18 | self.producer = producer 19 | self.topic = topic 20 | 21 | def start(self): 22 | self.logger.info('Start {}'.format(self.__class__.__name__)) 23 | 24 | def process(self, message=None): 25 | self.producer.send(self.topic, message) 26 | return message 27 | -------------------------------------------------------------------------------- /ank/components/kafka_consumer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class KafkaAnkConsumer(PipeApp): 8 | """ 9 | Message was received from consumer.subscribe 10 | """ 11 | 12 | def ini_app(self, consumer=None, topic=None): 13 | """ 14 | Args: 15 | consumer (kafka.KafkaConsumer): kafka consumer 16 | topic (string): kafka subscriber topic 17 | """ 18 | self.consumer = consumer 19 | self.topic = topic 20 | 21 | def start(self): 22 | self.consumer.subscribe(self.topic) 23 | 24 | for message in self.consumer: 25 | self.chain_process(message) 26 | 27 | def process(self, message=None): 28 | return message 29 | -------------------------------------------------------------------------------- /ank/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | VERSION = '1.6.0' 5 | API_DEFAULT_PORT = '5372' 6 | 7 | 8 | from components.pipe_app import PipeApp 9 | from components.api_app import APIApp 10 | from components.schedule_app import ScheduleApp 11 | 12 | from components.join_app import JoinApp 13 | from components.split_app import SplitApp 14 | from components.log_app import LogApp 15 | 16 | from components.redis_subscribe import RedisSubscribe 17 | from components.kafka_consumer import KafkaAnkConsumer 18 | from components.rabbitmq_consumer import RabbitmqConsumer 19 | from components.zmq_consumer import ZeroMqConsumer 20 | 21 | from components.kafka_producer import KafkaAnkProducer 22 | from components.rabbitmq_producer import RabbitmqProducer 23 | from components.zmq_producer import ZeroMqProducer 24 | -------------------------------------------------------------------------------- /ank/tests/services.yml: -------------------------------------------------------------------------------- 1 | 2 | services: 3 | 4 | FirstApp: 5 | class: processor.FirstApp 6 | arguments: ['%range_from%', 'range_to'] 7 | 8 | PrintApp: 9 | class: processor.PrintApp 10 | arguments: ~ 11 | 12 | ConditionalApp: 13 | class: processor.ConditionalApp 14 | arguments: ~ 15 | 16 | OddApp: 17 | class: processor.OddApp 18 | arguments: ~ 19 | 20 | EvenApp: 21 | class: processor.EvenApp 22 | arguments: ~ 23 | 24 | LogApp: 25 | class: apps.log_app.LogApp 26 | arguments: ~ 27 | 28 | JoinApp: 29 | class: apps.join_app.JoinApp 30 | arguments: '%batch_size%' 31 | 32 | SplitApp: 33 | class: apps.split_app.SplitApp 34 | arguments: ~ 35 | 36 | 37 | chains: 38 | - FirstApp 39 | # - LogApp 40 | - JoinApp 41 | # - LogApp 42 | - SplitApp 43 | - ConditionalApp 44 | - [EvenApp, OddApp] 45 | - PrintApp -------------------------------------------------------------------------------- /ank/components/redis_subscribe.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class RedisSubscribe(PipeApp): 8 | """ 9 | Subscribe message from redis pubsub 10 | """ 11 | 12 | def ini_app(self, redis=None, topic=''): 13 | """ 14 | Args: 15 | redis (redis.StrictRedis): redis client 16 | topic (string): redis subscriber topic 17 | """ 18 | self.redis = redis 19 | self.topic = topic 20 | 21 | def start(self): 22 | self.logger.info('Start {}'.format(self.__class__.__name__)) 23 | pubsub = self.redis.pubsub(self.topic) 24 | 25 | while True: 26 | message = pubsub.get_message() 27 | if message: 28 | self.chain_process(message.get('data')) 29 | 30 | def process(self, message=None): 31 | return message 32 | -------------------------------------------------------------------------------- /ank/components/zmq_producer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | try: 6 | import zmq 7 | except ImportError: 8 | raise ImportError('pyzmq not found') 9 | 10 | 11 | class ZeroMqProducer(PipeApp): 12 | """ 13 | Push message to queue 14 | """ 15 | 16 | def init_app(self, uri='', topic=''): 17 | """ 18 | Args: 19 | uri (string): connection uri 20 | topic (string): topic name 21 | """ 22 | 23 | context = zmq.Context() 24 | 25 | self.sock = context.socket(zmq.PUSH) 26 | self.sock.bind(uri) 27 | self.sock.setsockopt(zmq.PULL, topic) 28 | 29 | def start(self): 30 | self.logger.info('Start {}'.format(self.__class__.__name__)) 31 | 32 | def process(self, message=None): 33 | self.sock.send(message) 34 | return message 35 | -------------------------------------------------------------------------------- /ank/templates/unittest.tpy: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from ank import program_loader 4 | from processor import {0} 5 | 6 | 7 | class TestService(unittest.TestCase): 8 | 9 | def test_something(self): 10 | self.assertEqual(True, False) 11 | 12 | def test_function(self): 13 | service = {0}() 14 | 15 | message = None # Need initialize variable 16 | expected_result = None # Need initialize variable 17 | 18 | output_service = service.process(message) 19 | 20 | self.assertEqual(output_service, expected_result) 21 | 22 | def test_chain_processor(self): 23 | if os.path.exists('_processor.py'): 24 | import _processor 25 | else: 26 | print('generate _processor.py before run this test') 27 | 28 | def test_chain(self): 29 | program_loader.main('settings.yml') # TODO replace by your file settings 30 | 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /ank/tests/test_deploy.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import unittest 5 | from ank.program_loader import ProgramLoader 6 | from ank.generate_processor import GenerateProcessor 7 | from ank.generate_setting import GenerateSetting 8 | 9 | 10 | class TestDeploy(unittest.TestCase): 11 | """ 12 | To run this test, copy services.yml, settings.yml and processor.py to root folder 13 | """ 14 | 15 | def test_program_loader(self): 16 | loader = ProgramLoader() 17 | print('\n\n## Head Dependency Injection:') 18 | loader.start() 19 | 20 | def test_gen_processor(self): 21 | generate_processor = GenerateProcessor() 22 | print('\n\n## Generate Processor:') 23 | print(generate_processor.process()) 24 | 25 | def test_gen_setting(self): 26 | generate_setting = GenerateSetting() 27 | print('\n\n## Generate Setting:') 28 | print(generate_setting.process()) 29 | 30 | 31 | if __name__ == '__main__': 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /ank/components/zmq_consumer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | try: 6 | import zmq 7 | except ImportError: 8 | raise ImportError('pyzqm not found') 9 | 10 | 11 | class ZeroMqConsumer(PipeApp): 12 | """ 13 | Message was received from sock.recv 14 | """ 15 | 16 | def init_app(self, uri='', topic=''): 17 | """ 18 | Args: 19 | uri (string): connection uri 20 | topic (string): topic name 21 | """ 22 | context = zmq.Context() 23 | 24 | self.sock = context.socket(zmq.PULL) 25 | self.sock.bind(uri) 26 | self.topic = topic 27 | 28 | def start(self): 29 | self.logger.info('Start {}'.format(self.__class__.__name__)) 30 | self.sock.setsockopt(zmq.PULL, self.topic) 31 | while True: 32 | message = self.sock.recv() 33 | self.chain_process(message) 34 | 35 | def process(self, message=None): 36 | return message 37 | -------------------------------------------------------------------------------- /examples/api_app/endpoint.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.api_app import APIApp 5 | 6 | 7 | class ExampleAPI(APIApp): 8 | 9 | def __init__(self, host='localhost', port=5372, mongo_client=None, mongo_db=''): 10 | super(ExampleAPI, self).__init__(host=host, port=port) 11 | 12 | self.db = mongo_client[mongo_db] 13 | 14 | # path: host:port/api/add?a=1&b=2 15 | def add(self, params): 16 | a = int(params.get('a')) 17 | b = int(params.get('b')) 18 | result = a + b 19 | 20 | collection = self.db['add'] 21 | collection.insert({'param1': a, 'param2': b, 'result': result}) 22 | return {'result': result} 23 | 24 | # path: host:port/api/sub?a=105&b=17 25 | def sub(self, params): 26 | a = int(params.get('a')) 27 | b = int(params.get('b')) 28 | result = a - b 29 | 30 | collection = self.db['sub'] 31 | collection.insert({'param1': a, 'param2': b, 'result': result}) 32 | return {'result': result} 33 | -------------------------------------------------------------------------------- /examples/streaming_app/services.yml: -------------------------------------------------------------------------------- 1 | 2 | services: 3 | 4 | FirstApp: 5 | class: processor.FirstApp 6 | arguments: [$Redis, '%range_from%', '%range_to%'] 7 | 8 | Redis: 9 | class: redis.client.StrictRedis 10 | arguments: ['%redis_host%', '%redis_port%'] 11 | 12 | PrintApp: 13 | class: processor.PrintApp 14 | arguments: ~ 15 | 16 | ConditionalApp: 17 | class: processor.ConditionalApp 18 | arguments: ~ 19 | 20 | OddApp: 21 | class: processor.OddApp 22 | arguments: ~ 23 | 24 | EvenApp: 25 | class: processor.EvenApp 26 | arguments: ~ 27 | 28 | LogApp: 29 | class: ank.components.log_app.LogApp 30 | arguments: ~ 31 | 32 | JoinApp: 33 | class: ank.components.join_app.JoinApp 34 | arguments: '%batch_size%' 35 | 36 | SplitApp: 37 | class: ank.components.split_app.SplitApp 38 | arguments: ~ 39 | 40 | 41 | chains: 42 | - FirstApp 43 | # - LogApp 44 | - JoinApp 45 | # - LogApp 46 | - SplitApp 47 | - ConditionalApp 48 | - [EvenApp, OddApp] 49 | - PrintApp -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | All code is released under the MIT License: 2 | 3 | Copyright 2016 Nhat Vo Van (a.k.a Sunary) and contributors (see CONTRIBUTORS.txt) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a 6 | copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /ank/components/rabbitmq_consumer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | try: 6 | import pika 7 | except ImportError: 8 | raise ImportError('pika not found') 9 | 10 | 11 | class RabbitmqConsumer(PipeApp): 12 | """ 13 | Message was received from queue by on_messages_received() method 14 | """ 15 | 16 | def init_app(self, uri='', queue=''): 17 | """ 18 | Args: 19 | uri (string): connection uri 20 | queue (string): queue name 21 | Examples: 22 | >>> RabbitmqConsumer(uri='amqp://username:password@host:5672', 23 | ... queue='ExampleQueue') 24 | """ 25 | self.connection = pika.BlockingConnection(pika.URLParameters(uri)) 26 | self.queue = queue 27 | 28 | def start(self): 29 | self.logger.info('Start {}'.format(self.__class__.__name__)) 30 | channel = self.connection.channel() 31 | channel.basic_consume(self.call_back, queue=self.queue, no_ack=True) 32 | 33 | channel.start_consuming() 34 | 35 | def call_back(self, ch, method, properties, message): 36 | return self.process(message) 37 | 38 | def process(self, message=None): 39 | return message 40 | -------------------------------------------------------------------------------- /ank/components/schedule_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import time 5 | from ank.components.pipe_app import PipeApp 6 | from datetime import datetime 7 | from ank.utils.crontab_time import CrontabTimeReader 8 | 9 | 10 | class ScheduleApp(PipeApp): 11 | """ 12 | Schedule process using crontab time format 13 | """ 14 | 15 | def init_app(self, crontab_time='', start_now=False): 16 | """ 17 | Args: 18 | crontab_time (string): cronjob time format 19 | start_now (bool): is start after init 20 | """ 21 | self.crontab_reader = CrontabTimeReader(crontab_time) 22 | self.start_now = start_now 23 | self.logger.info('crontad time after extract: %s', self.crontab_reader) 24 | 25 | def start(self): 26 | if self.start_now: 27 | self.process() 28 | time.sleep(60) 29 | 30 | while True: 31 | next_time = self.crontab_reader.get_next_time() 32 | self.logger.info('sleep to %s' % next_time) 33 | 34 | second_to_wait = (next_time - datetime.utcnow()).total_seconds() 35 | time.sleep(second_to_wait) 36 | self.process() 37 | 38 | def process(self, message=None): 39 | return message 40 | -------------------------------------------------------------------------------- /ank/utils/logger.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'sunary' 3 | 4 | 5 | import os 6 | import logging 7 | import logging.handlers 8 | 9 | 10 | def init_logger(name, log_level=None, log_file=None): 11 | _logger = logging.getLogger(name) 12 | 13 | if log_level: 14 | _logger.setLevel(log_level) 15 | else: 16 | _logger.setLevel(logging.INFO) 17 | 18 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s: %(message)s') 19 | 20 | if not log_file: 21 | log_file = '{}.log'.format(name) 22 | 23 | if log_file: 24 | if not os.path.exists(log_file): 25 | with open(log_file, 'w') as of: 26 | of.write('') 27 | 28 | fh = logging.handlers.RotatingFileHandler(log_file, maxBytes=2 * 1024*1024, backupCount=3) 29 | fh.setFormatter(formatter) 30 | _logger.addHandler(fh) 31 | # else: 32 | # sh = logging.StreamHandler() 33 | # sh.setFormatter(formatter) 34 | # logger.addHandler(sh) 35 | 36 | return _logger 37 | 38 | 39 | if __name__ == '__main__': 40 | logger = init_logger('test', logging.INFO) 41 | logger.info('info') 42 | logger.debug('debug') 43 | logger.warning('warning') 44 | logger.error('error') 45 | -------------------------------------------------------------------------------- /ank/components/rabbitmq_producer.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | try: 6 | import pika 7 | except ImportError: 8 | raise ImportError('pika not found') 9 | 10 | 11 | class RabbitmqProducer(PipeApp): 12 | """ 13 | Push message to queue 14 | """ 15 | def init_app(self, uri='', exchange='', routing_key=''): 16 | """ 17 | Args: 18 | uri (string): connection uri 19 | exchange (string): exchange name 20 | routing_key (string): routing key 21 | Examples: 22 | >>> RabbitmqProducer(uri='amqp://username:password@host:5672', 23 | ... exchange='ExampleExchange', 24 | ... routing_key='ExchangeToQueue') 25 | """ 26 | self.exchange = exchange 27 | self.routing_key = routing_key 28 | self.connection = pika.BlockingConnection(pika.URLParameters(uri)) 29 | self.channel = self.connection.channel() 30 | 31 | def start(self): 32 | self.logger.info('Start {}'.format(self.__class__.__name__)) 33 | 34 | def process(self, message=None): 35 | self.channel.basic_publish(exchange=self.exchange, 36 | routing_key=self.routing_key, 37 | body=message) 38 | return message 39 | 40 | def close_connection(self): 41 | self.connection.close() 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import os 5 | from setuptools import setup, find_packages 6 | from ank import VERSION 7 | 8 | 9 | def __path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | 13 | with open('README.md') as fo: 14 | readme = fo.read() 15 | 16 | with open('LICENSE') as fo: 17 | license = fo.read() 18 | 19 | with open('CHANGES.md') as fo: 20 | changes = fo.read() 21 | 22 | reqs = ['pyyaml==3.12', 23 | 'flask==1.0.2', 24 | 'redis==2.10.5', 25 | 'pika==0.10.0', 26 | 'pyzmq==16.0.2', 27 | 'kafka-python==0.9.5'] 28 | 29 | setup( 30 | name='ank', 31 | version=VERSION, 32 | author='Sunary [Nhat Vo Van]', 33 | author_email='v2nhat@gmail.com', 34 | maintainer='Sunary [Nhat Vo Van]', 35 | maintainer_email='v2nhat@gmail.com', 36 | platforms='any', 37 | description='Python Streaming system, REST-API and Schedule task using queue message', 38 | long_description='Python Streaming system, REST-API and Schedule task using queue message\n', 39 | license=license, 40 | keywords='ank, streaming, microservice, pipeline, schedule task', 41 | url='https://github.com/sunary/ank', 42 | packages=find_packages(exclude=['docs', 'examples', 'tests']), 43 | package_data={'templates': ['*.tpy']}, 44 | install_requires=reqs, 45 | entry_points={ 46 | 'console_scripts': ['ank = ank.cli:main'] 47 | }, 48 | ) -------------------------------------------------------------------------------- /ank/utils/cmd_helpers.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import os 5 | import subprocess 6 | 7 | 8 | class chdir(object): 9 | 10 | def __init__(self, path): 11 | self.path = path 12 | self.last_dir = os.getcwd() 13 | 14 | def __enter__(self): 15 | if self.path: 16 | os.chdir(self.path) 17 | 18 | def __exit__(self, exc_type, exc_val, exc_tb): 19 | os.chdir(self.last_dir) 20 | 21 | 22 | def run_cmd(cmd, **kwargs): 23 | try: 24 | process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 25 | return_code = process.wait() 26 | result = 'returned {0}'.format(return_code) 27 | result += ' ' + process.stdout.read() 28 | return {'success': True, 29 | 'message': result} 30 | except Exception as e: 31 | return {'success': False, 32 | 'message': 'Error when execute command: {}'.format(cmd), 33 | 'detail': str(e)} 34 | 35 | 36 | def run_cmds(cmds, **kwargs): 37 | if not cmds: 38 | return {'success': True} 39 | 40 | output = {'success': True, 41 | 'message': []} 42 | for cmd in cmds: 43 | output_cmd = run_cmd(cmd, **kwargs) 44 | output['message'].append(output_cmd['message']) 45 | if not output_cmd['success']: 46 | output['success'] = False 47 | output['detail'] = output_cmd['detail'] 48 | break 49 | 50 | output['message'] = '###############'.join(output['message']) 51 | return output 52 | -------------------------------------------------------------------------------- /ank/components/pipe_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.utils import logger 5 | 6 | 7 | class PipeApp(object): 8 | """ 9 | Start app from run() method 10 | Process message in process() method 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | self.logger = logger.init_logger(self.__class__.__name__) 15 | 16 | self.chain_process = None 17 | self.init_app(*args, **kwargs) 18 | 19 | def init_app(self, *args, **kwargs): 20 | pass 21 | 22 | def run(self, process=None): 23 | self.chain_process = process or self.process 24 | 25 | self.start() 26 | 27 | def start(self): 28 | """ 29 | Your self.process become self.chain_process 30 | Start run everything from it 31 | """ 32 | 33 | def process(self, message=None): 34 | """ 35 | Args: 36 | message: {'content': (*) 'content of message', 37 | 'flags': (list|tuple) 'define next process will be use'} 38 | raise TypeError if you don't declare this in return of before branching-processor 39 | if 'flags' == [True, True]: process both in next branching-processors 40 | if 'flags' == [True, False]: process 1st processor in next branching-processors 41 | if 'flags' == [False, True]: process 2nd processor in next branching-processors 42 | if 'flags' == [False, False]: no processor, stop chain 43 | is None: stop chain 44 | """ 45 | return message 46 | -------------------------------------------------------------------------------- /ank/components/api_app.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import os 5 | from ank import API_DEFAULT_PORT 6 | from ank.components.pipe_app import PipeApp 7 | try: 8 | from flask import Flask 9 | except ImportError: 10 | ImportError('flask not found') 11 | from ank.utils import api_helpers 12 | 13 | 14 | def is_production(): 15 | return os.environ.get('ENV', 'dev').lower == 'production' 16 | 17 | 18 | class APIApp(PipeApp): 19 | """ 20 | API App 21 | Add function to extend class to create new API 22 | """ 23 | 24 | def init_app(self, host='localhost', port=API_DEFAULT_PORT): 25 | """ 26 | Args: 27 | host (string): RestAPI host, default 'localhost' 28 | port (int): RestAPI host, default API_DEFAULT_PORT=5372 29 | """ 30 | self.host = host 31 | self.port = port 32 | 33 | def start(self): 34 | flask_app = Flask(__name__) 35 | 36 | @flask_app.route('/') 37 | def index(): 38 | return 'Hello world!' 39 | 40 | @api_helpers.crossdomain(origin='*') 41 | @flask_app.route('/api/', methods=['GET', 'POST']) 42 | def _(method): 43 | try: 44 | params = api_helpers.get_options() 45 | method = getattr(self, method) 46 | return method(params) 47 | except Exception as e: 48 | self.logger.error(e) 49 | 50 | return api_helpers.failed(return_json={'detail': str(e)}, status_code=404, 51 | message='The requested url does not exist') 52 | 53 | flask_app.run(host=self.host, port=self.port, debug=not is_production()) 54 | -------------------------------------------------------------------------------- /ank/tests/processor.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class FirstApp(PipeApp): 8 | 9 | def init_app(self, range_from=0, range_to=0): 10 | self.range_from = range_from 11 | self.range_to = range_to 12 | 13 | def start(self): 14 | print('Start chain') 15 | for i in range(self.range_from, self.range_to): 16 | print('---start') 17 | self.chain_process({'content': i}) 18 | print('---end') 19 | 20 | def process(self, message=None): 21 | print('start app {}'.format(message)) 22 | return message 23 | 24 | 25 | class PrintApp(PipeApp): 26 | 27 | def init_app(self, *args): 28 | pass 29 | 30 | def run(self, process=None): 31 | print('From print app') 32 | 33 | def process(self, message=None): 34 | print('print only {}'.format(message)) 35 | return message 36 | 37 | 38 | class ConditionalApp(PipeApp): 39 | 40 | def init_app(self, *args): 41 | pass 42 | 43 | def run(self, process=None): 44 | print('From conditional app') 45 | 46 | def process(self, message=None): 47 | print('condition check {}'.format(message)) 48 | 49 | if message['content'] % 2: 50 | message.update({'flags': [False, True]}) 51 | else: 52 | message.update({'flags': [True, False]}) 53 | 54 | return message 55 | 56 | 57 | class OddApp(PipeApp): 58 | 59 | def init_app(self, *agrs): 60 | pass 61 | 62 | def run(self, process=None): 63 | print('From odd app') 64 | 65 | def process(self, message=None): 66 | print('odd {}'.format(message)) 67 | message['content'] = (message['content'] + 1)/2 68 | return message 69 | 70 | 71 | class EvenApp(PipeApp): 72 | 73 | def init_app(self, *args): 74 | pass 75 | 76 | def run(self, process=None): 77 | print('From even app') 78 | 79 | def process(self, message=None): 80 | print('even {}'.format(message)) 81 | message['content'] /= 2 82 | return message 83 | -------------------------------------------------------------------------------- /ank/utils/naming_services.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | def normalize_service_argument(argument): 5 | """ 6 | normalize service name and set type: $object, $variable$ 7 | Examples: 8 | >>> normalize_service_argument('$Mongodb') 9 | ['Mongodb', 'object'] 10 | >>> normalize_service_argument('%batch_size%') 11 | ['batch_size', 'variable'] 12 | """ 13 | if isinstance(argument, dict): 14 | dict_argument = {} 15 | 16 | for key, value in argument.items(): 17 | if value.startswith('$'): 18 | dict_argument[key] = [value[1:], 'object'] 19 | elif value.startswith('%') and value.endswith('%'): 20 | dict_argument[key] = [value[1:-1], 'variable'] 21 | 22 | return [dict_argument, 'dict'] 23 | elif type(argument) is str: 24 | if argument.startswith('$'): 25 | return [argument[1:], 'object'] 26 | elif argument.startswith('%') and argument.endswith('%'): 27 | return [argument[1:-1], 'variable'] 28 | 29 | return [None] 30 | 31 | 32 | def class_name_extract(class_full_name): 33 | """ 34 | extract class name to write import 35 | Examples: 36 | >>> class_name_extract('redis.client.StrictRedis') 37 | ('redis.client', 'StrictRedis') 38 | """ 39 | class_split = class_full_name.split('.') 40 | module_name = '.'.join(class_split[:-1]) 41 | class_name = class_split[-1] 42 | 43 | return module_name, class_name 44 | 45 | 46 | def get_deliver_from_class(class_name): 47 | """ 48 | Create deliver name from Class Name 49 | Examples: 50 | >>> get_deliver_from_class('DemoApp') 51 | '_demo_app' 52 | """ 53 | import string 54 | 55 | split_text = [] 56 | last_position = 0 57 | for i in range(1, len(class_name)): 58 | if class_name[i] in string.ascii_uppercase: 59 | split_text.append(class_name[last_position:i].lower()) 60 | last_position = i 61 | 62 | split_text.append(class_name[last_position:len(class_name)].lower()) 63 | 64 | return '_' + '_'.join(split_text) 65 | 66 | 67 | if __name__ == '__main__': 68 | import doctest 69 | doctest.testmod() 70 | -------------------------------------------------------------------------------- /examples/streaming_app/processor.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.components.pipe_app import PipeApp 5 | 6 | 7 | class FirstApp(PipeApp): 8 | 9 | def init_app(self, redis=None, range_from=0, range_to=0): 10 | self.redis = redis 11 | self.range_from = range_from 12 | self.range_to = range_to 13 | 14 | def start(self): 15 | redis_key = 'redis_key' 16 | self.redis.set(redis_key, 'hello world') 17 | print('Message from redis: {}'.format(self.redis.get(redis_key))) 18 | 19 | print('Start chain') 20 | for i in range(self.range_from, self.range_to): 21 | print('---start') 22 | self.chain_process({'content': i}) 23 | print('---end') 24 | 25 | def process(self, message=None): 26 | print('start app {}'.format(message)) 27 | return message 28 | 29 | 30 | class PrintApp(PipeApp): 31 | 32 | def init_app(self, *args): 33 | pass 34 | 35 | def run(self, process=None): 36 | print('From print app') 37 | 38 | def process(self, message=None): 39 | print('print only {}'.format(message)) 40 | return message 41 | 42 | 43 | class ConditionalApp(PipeApp): 44 | 45 | def init_app(self, *args): 46 | pass 47 | 48 | def run(self, process=None): 49 | print('From conditional app') 50 | 51 | def process(self, message=None): 52 | print('condition check {}'.format(message)) 53 | 54 | if message['content'] % 2: 55 | message.update({'flags': [False, True]}) 56 | else: 57 | message.update({'flags': [True, False]}) 58 | 59 | return message 60 | 61 | 62 | class OddApp(PipeApp): 63 | 64 | def init_app(self, *agrs): 65 | pass 66 | 67 | def run(self, process=None): 68 | print('From odd app') 69 | 70 | def process(self, message=None): 71 | print('odd {}'.format(message)) 72 | message['content'] = (message['content'] + 1)/2 73 | return message 74 | 75 | 76 | class EvenApp(PipeApp): 77 | 78 | def init_app(self, *args): 79 | pass 80 | 81 | def run(self, process=None): 82 | print('From even app') 83 | 84 | def process(self, message=None): 85 | print('even {}'.format(message)) 86 | message['content'] /= 2 87 | return message 88 | -------------------------------------------------------------------------------- /ank/generate_setting.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.utils import naming_services, config_handle, logger 5 | 6 | 7 | class GenerateSetting(object): 8 | """ 9 | Generate settings.yml template 10 | """ 11 | 12 | def __init__(self): 13 | self.logger = logger.init_logger(self.__class__.__name__) 14 | 15 | self.setting_parameters = {} 16 | 17 | def process(self, file_setting='_settings.yml'): 18 | try: 19 | self.service_loader = config_handle.load('services.yml', 'services') 20 | except IOError: 21 | self.logger.error('IOError: file \'services.yml\' not found') 22 | raise IOError('file \'services.yml\' not found') 23 | except KeyError: 24 | self.logger.error('KeyError: \'services\'') 25 | raise KeyError('services') 26 | 27 | try: 28 | chain_loader = config_handle.load('services.yml', 'chains') 29 | except KeyError: 30 | self.logger.error('KeyError: \'chains\'') 31 | raise KeyError('chains') 32 | 33 | for process_name in chain_loader: 34 | if isinstance(process_name, list): 35 | for proc in process_name: 36 | self.from_class(proc) 37 | else: 38 | self.from_class(process_name) 39 | 40 | # read old settings 41 | try: 42 | setting_loader = config_handle.load('settings.yml', 'parameters') 43 | except IOError: 44 | self.logger.error('IOError: file \'settings.yml\' not found') 45 | raise IOError('settings.yml not found') 46 | except KeyError: 47 | self.logger.error('KeyError: \'parameters\'') 48 | raise KeyError('parameters') 49 | 50 | for key, value in setting_loader.items(): 51 | if self.setting_parameters.get(key, '') is None: 52 | self.setting_parameters[key] = value 53 | 54 | output = config_handle.save(file_setting, 'parameters', self.setting_parameters) 55 | 56 | return output 57 | 58 | def from_object(self, argument): 59 | argument = naming_services.normalize_service_argument(argument) 60 | if argument[-1] == 'dict': 61 | for key, value in argument[0].items(): 62 | if value[-1] == 'object': 63 | self.from_class(value[0]) 64 | elif value[-1] == 'variable': 65 | self.setting_parameters[value[0]] = None 66 | 67 | elif argument[-1] == 'object': 68 | self.from_class(argument[0]) 69 | elif argument[-1] == 'variable': 70 | self.setting_parameters[argument[0]] = None 71 | 72 | def from_class(self, str_class): 73 | object_arguments = self.service_loader[str_class]['arguments'] 74 | object_arguments = object_arguments if type(object_arguments) is list else [object_arguments] 75 | [self.from_object(x) for x in object_arguments] 76 | 77 | 78 | def main(file_setting='_settings.yml'): 79 | generate_setting = GenerateSetting() 80 | print(generate_setting.process(file_setting)) 81 | 82 | 83 | if __name__ == '__main__': 84 | main(file_setting='_settings.yml') 85 | -------------------------------------------------------------------------------- /ank/program_loader.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import os 5 | import sys 6 | sys.path.append(os.getcwd()) 7 | 8 | import importlib 9 | from ank.chain_process import ChainProcess 10 | from ank.daemon import Daemon 11 | from ank.utils import naming_services, logger, config_handle 12 | 13 | 14 | class ProgramLoader(object): 15 | 16 | def __init__(self): 17 | self.logger = logger.init_logger(self.__class__.__name__) 18 | self.service_loader = {} 19 | self.setting_loader = {} 20 | 21 | def start(self, file_setting='settings.yml'): 22 | self.service_loader = config_handle.load('services.yml', 'services') 23 | self.setting_loader = config_handle.load(file_setting, 'parameters') 24 | 25 | chain_processor = ChainProcess() 26 | chain_loader = config_handle.load('services.yml', 'chains') 27 | 28 | for process_name in chain_loader: 29 | if isinstance(process_name, list): 30 | self.logger.info('processors: [{}]'.format(', '.join(process_name))) 31 | 32 | processors = [] 33 | for proc in process_name: 34 | processors.append(self.get_class(proc)) 35 | 36 | chain_processor.add_processor(processors) 37 | else: 38 | self.logger.info('processor: {}'.format(process_name)) 39 | 40 | processor = self.get_class(process_name) 41 | chain_processor.add_processor(processor) 42 | 43 | chain_processor.methods[0][0].run(chain_processor.process) 44 | 45 | def get_object(self, argument): 46 | """ 47 | Args: 48 | argument: list: [..., data_type] 49 | """ 50 | argument = naming_services.normalize_service_argument(argument) 51 | 52 | if argument[-1] == 'dict': 53 | dict_argument = {} 54 | for key, value in argument[0].items(): 55 | if value[-1] == 'object': 56 | dict_argument[key] = self.get_class(value[0]) 57 | elif value[-1] == 'variable': 58 | dict_argument[key] = self.setting_loader[value[0]] 59 | 60 | return dict_argument 61 | elif argument[-1] == 'object': 62 | return self.get_class(argument[0]) 63 | elif argument[-1] == 'variable': 64 | return self.setting_loader[argument[0]] 65 | 66 | return None 67 | 68 | def get_class(self, str_class): 69 | object_name = self.service_loader[str_class]['class'] 70 | 71 | object_arguments = self.service_loader[str_class]['arguments'] 72 | object_arguments = object_arguments if (type(object_arguments) is list) else [object_arguments] 73 | 74 | object_parameters = tuple([self.get_object(x) for x in object_arguments]) 75 | 76 | processor = self.load_class(object_name, object_parameters) 77 | 78 | return processor 79 | 80 | @staticmethod 81 | def load_class(class_full_name, parameters): 82 | module_name, class_name = naming_services.class_name_extract(class_full_name) 83 | module = importlib.import_module(module_name) 84 | 85 | _class = getattr(module, class_name) 86 | 87 | if parameters[0] is None: 88 | return _class() 89 | 90 | return _class(*parameters) 91 | 92 | 93 | def main(file_setting='settings.yml'): 94 | daemon = Daemon('daemon.pid') 95 | daemon.start() 96 | print(os.getpid()) 97 | loader = ProgramLoader() 98 | print(loader.start(file_setting)) 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /ank/utils/api_helpers.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from datetime import timedelta 5 | try: 6 | from flask import make_response, request, current_app, json 7 | except ImportError: 8 | raise ImportError('flask not found') 9 | from functools import update_wrapper 10 | 11 | 12 | STATUS_CODE = {200: 'success', 13 | 201: 'created', 14 | 202: 'accepted', 15 | 204: 'no_content', 16 | 302: 'redirect', 17 | 304: 'not_modified', 18 | 400: 'bad_request', 19 | 401: 'unauthorized', 20 | 403: 'forbidden', 21 | 404: 'not_found', 22 | 405: 'method_not_allowed', 23 | 409: 'conflict', 24 | 412: 'precondition_failed', 25 | 429: 'too_many_requests', 26 | 500: 'internal_server_error', 27 | 503: 'unavailable'} 28 | 29 | 30 | def crossdomain(origin=None, methods=None, headers=None, max_age=21600, 31 | attach_to_all=True, automatic_options=True): 32 | if methods is not None: 33 | methods = ', '.join(sorted(x.upper() for x in methods)) 34 | if headers is not None and not isinstance(headers, basestring): 35 | headers = ', '.join(x.upper() for x in headers) 36 | if not isinstance(origin, basestring): 37 | origin = ', '.join(origin) 38 | if isinstance(max_age, timedelta): 39 | max_age = max_age.total_seconds() 40 | 41 | def get_methods(): 42 | if methods is not None: 43 | return methods 44 | 45 | options_resp = current_app.make_default_options_response() 46 | return options_resp.headers['allow'] 47 | 48 | def decorator(f): 49 | def wrapped_function(*args, **kwargs): 50 | if automatic_options and request.method == 'OPTIONS': 51 | resp = current_app.make_default_options_response() 52 | else: 53 | resp = make_response(f(*args, **kwargs)) 54 | if not attach_to_all and request.method != 'OPTIONS': 55 | return resp 56 | 57 | h = resp.headers 58 | 59 | h['Access-Control-Allow-Origin'] = origin 60 | h['Access-Control-Allow-Methods'] = get_methods() 61 | h['Access-Control-Max-Age'] = str(max_age) 62 | if headers is not None: 63 | h['Access-Control-Allow-Headers'] = headers 64 | return resp 65 | 66 | f.provide_automatic_options = False 67 | return update_wrapper(wrapped_function, f) 68 | return decorator 69 | 70 | 71 | def get_options(): 72 | options = {} 73 | 74 | if request.method == 'POST': 75 | get_json = request.get_json() 76 | 77 | if get_json: 78 | for k, v in get_json.items(): 79 | options[k] = str(v) 80 | 81 | for field in request.form: 82 | options[field] = str(request.form.get(field)) 83 | 84 | elif request.method == 'GET': 85 | for field in request.args: 86 | options[field] = str(request.args.get(field)) 87 | 88 | return options 89 | 90 | 91 | def failed(return_json={}, status_code=400, message=None): 92 | return_json['ok'] = False 93 | return_json['status_code'] = status_code 94 | return_json['message'] = message or STATUS_CODE.get(status_code) 95 | 96 | return json.jsonify(return_json) 97 | 98 | 99 | def success(return_json={}, status_code=200, message=None): 100 | return_json['ok'] = True 101 | return_json['status_code'] = status_code 102 | return_json['message'] = message or STATUS_CODE.get(status_code) 103 | 104 | return json.jsonify(return_json) 105 | 106 | 107 | def is_success(api_json): 108 | return api_json.get('ok') or api_json.get('status_code') == 200 109 | -------------------------------------------------------------------------------- /ank/chain_process.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.utils import logger 5 | import copy 6 | 7 | 8 | CONTENT_KEY = 'content' 9 | FLAGS_KEY = 'flags' 10 | 11 | 12 | class ChainProcess(object): 13 | """ 14 | Run chain processors 15 | Output of a processor will be input of the next processor 16 | """ 17 | 18 | def __init__(self): 19 | self.logger = logger.init_logger(self.__class__.__name__) 20 | 21 | self.methods = [] 22 | 23 | def run(self): 24 | if self.methods: 25 | self.methods[0][0].run(self.process()) 26 | else: 27 | self.logger.error('KeyError: methods empty') 28 | raise KeyError('methods empty') 29 | 30 | def add_processor(self, processor, method='process'): 31 | """ 32 | add processor(s) and method(n) into chain 33 | """ 34 | if isinstance(processor, list): 35 | list_methods = [] 36 | for proc in processor: 37 | list_methods.append(getattr(proc, method)) 38 | 39 | self.methods.append((processor, list_methods)) 40 | else: 41 | self.methods.append((processor, getattr(processor, method))) 42 | 43 | def process(self, message=None, chain_methods=None): 44 | """ 45 | Process chain depend methods registered 46 | """ 47 | _message = copy.deepcopy(message) 48 | chain_methods = chain_methods or self.methods 49 | 50 | for i, (current_processor, current_method) in enumerate(chain_methods): 51 | if _message is None: 52 | return None 53 | 54 | if isinstance(current_processor, (list, tuple)): 55 | if not _message.get(FLAGS_KEY) or not isinstance(_message[FLAGS_KEY], (list, tuple)): 56 | _log_msg = 'message must to have FLAGS_KEY attribute with type is list or tuple' 57 | self.logger.error('TypeError ' + _log_msg) 58 | raise TypeError(_log_msg) 59 | 60 | temp_message = copy.deepcopy(_message) 61 | _message.pop(FLAGS_KEY) 62 | for j, status in enumerate(temp_message[FLAGS_KEY]): 63 | if status: 64 | processor_name = current_processor[j].__class__.__name__ 65 | self.logger.info('Run processor: {}'.format(processor_name)) 66 | 67 | try: 68 | self.process(_message, chain_methods=chain_methods[i + 1:]) 69 | except Exception as e: 70 | _log_msg = 'when run process {}'.format(processor_name) 71 | self.logger.error(type(e).__name__ + ' ' + _log_msg) 72 | self.logger.error(e) 73 | raise type(e)(_log_msg + '\n' + str(e)) 74 | 75 | return None 76 | 77 | else: 78 | processor_name = current_processor.__class__.__name__ 79 | self.logger.info('Run processor: {}'.format(processor_name)) 80 | 81 | try: 82 | if processor_name == 'JoinApp': 83 | current_processor.stored_messages[CONTENT_KEY].append(_message[CONTENT_KEY]) 84 | 85 | # get chunk batch_size 86 | if len(current_processor.stored_messages[CONTENT_KEY]) >= current_processor.batch_size: 87 | _message = copy.deepcopy(current_processor.stored_messages) 88 | current_processor.stored_messages[CONTENT_KEY] = [] 89 | else: 90 | return None 91 | 92 | elif processor_name == 'SplitApp': 93 | for msg in _message[CONTENT_KEY]: 94 | self.process({CONTENT_KEY: msg}, chain_methods=chain_methods[i + 1:]) 95 | 96 | return None 97 | else: 98 | _message = current_method(_message) 99 | 100 | except Exception as e: 101 | _log_msg = 'when run process {}'.format(processor_name) 102 | self.logger.error(type(e).__name__ + ' ' + _log_msg) 103 | self.logger.error(e) 104 | raise type(e)(_log_msg + '\n' + str(e)) 105 | -------------------------------------------------------------------------------- /ank/utils/crontab_time.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from datetime import datetime, timedelta 5 | 6 | 7 | class CrontabTimeReader(object): 8 | """ 9 | Calculate next time from the crontab time format: 10 | * * * * * * 11 | | | | | | | 12 | | | | | | +-- Year (range: 1900-3000) 13 | | | | | +---- Day of the Week (range: 1-7, 1 standing for Monday) 14 | | | | +------ Month of the Year (range: 1-12) 15 | | | +-------- Day of the Month (range: 1-31) 16 | | +---------- Hour (range: 0-23) 17 | +------------ Minute (range: 0-59) 18 | """ 19 | 20 | def __init__(self, time_str): 21 | time_str = time_str.split(' ') 22 | 23 | assert len(time_str) == 5, 'Crontab time much have 5 fields' 24 | self.minute = self.extract_value(time_str[0]) 25 | self.hour = self.extract_value(time_str[1]) 26 | self.day = self.extract_value(time_str[2]) 27 | self.month = self.extract_value(time_str[3]) 28 | self.day_of_week = self.extract_value(time_str[3]) 29 | 30 | @staticmethod 31 | def extract_value(value): 32 | ret_value = [] 33 | if '-' in value: 34 | value = value.split('-') 35 | for n in range(int(value[0]), int(value[1]) + 1): 36 | ret_value.append(n) 37 | 38 | elif ',' in value: 39 | value = value.split(',') 40 | for n in value: 41 | ret_value.append(int(n)) 42 | 43 | elif value == '*': 44 | return '*' 45 | else: 46 | return [int(value)] 47 | 48 | return ret_value 49 | 50 | def get_next_time(self): 51 | now = datetime.utcnow() 52 | valid_hours = range(24) if self.hour == '*' else self.hour 53 | valid_minutes = range(60) if self.minute == '*' else self.minute 54 | 55 | combine_time = [] 56 | for h in valid_hours: 57 | for m in valid_minutes: 58 | combine_time.append([h, m]) 59 | 60 | combine_time.sort() 61 | combine_time_now = [now.hour, now.minute] 62 | 63 | if self.day != '*': 64 | if (self.month == '*' or now.month in self.month) and now.day in self.day: 65 | for t in combine_time: 66 | if t > combine_time_now: 67 | return now.replace(hour=t[0], minute=t[1]) 68 | elif self.day_of_week != '*': 69 | if now.weekday() in self.day_of_week: 70 | for t in combine_time: 71 | if t > combine_time_now: 72 | return now.replace(hour=t[0], minute=t[1]) 73 | else: 74 | for t in combine_time: 75 | if t > combine_time_now: 76 | return now.replace(hour=t[0], minute=t[1]) 77 | 78 | if self.day != '*': 79 | combine_day = [] 80 | valid_month = range(1, 13) if self.month == '*' else self.month 81 | for m in valid_month: 82 | for d in self.day: 83 | combine_day.append([m, d]) 84 | 85 | combine_day.sort() 86 | combine_day_now = [now.month, now.day] 87 | for d in combine_day: 88 | if d > combine_day_now: 89 | return now.replace(month=d[0], day=d[1], hour=combine_time[0][0], minute=combine_time[0][1]) 90 | 91 | return now.replace(year=now.year + 1, month=combine_day[0][0], 92 | day=combine_day[0][1], hour=combine_time[0][0], minute=combine_time[0][1]) 93 | elif self.day_of_week != '*': 94 | num_day = min(map(lambda x: x - now.weekday() if x > now.weekday() 95 | else x + 7 - now.weekday(), self.day_of_week)) 96 | return (now + timedelta(hours=24 * num_day)).replace(hour=combine_time[0][0], minute=combine_time[0][1]) 97 | else: 98 | return (now + timedelta(hours=24)).replace(hour=combine_time[0][0], minute=combine_time[0][1]) 99 | 100 | def __str__(self): 101 | return 'minute: {}, hour: {}, day: {}, month: {}, day of week: {}'.format( 102 | self.day, self.hour, self.day, self.month, self.day_of_week 103 | ) 104 | 105 | 106 | if __name__ == '__main__': 107 | timer_reader = CrontabTimeReader('55 23 2,4,6 3 *') 108 | print(timer_reader.get_next_time()) 109 | timer_reader = CrontabTimeReader('0 2,3 * * 0-5') 110 | print(timer_reader.get_next_time()) 111 | -------------------------------------------------------------------------------- /ank/generate_processor.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | from ank.utils import naming_services, config_handle 5 | 6 | 7 | class GenerateProcessor(object): 8 | """ 9 | Generate processor.py help you run directly 10 | """ 11 | 12 | def __init__(self): 13 | self.import_libs = ['from deploy.chain_processor import ChainProcessor'] 14 | self.implement_classes = ['chain_processor = ChainProcessor()'] 15 | 16 | def process(self, file_setting='settings.yml'): 17 | self.service_loader = config_handle.load('services.yml', 'services') 18 | self.setting_loader = config_handle.load(file_setting, 'parameters') 19 | 20 | str_processor = "__author__ = 'ank_generator'\n\n" 21 | 22 | implement_workers = [] 23 | chain_loader = config_handle.load('services.yml', 'chains') 24 | for process_name in chain_loader: 25 | if isinstance(process_name, list): 26 | class_names = [] 27 | for proc in process_name: 28 | class_names.append(self.get_class(proc)) 29 | 30 | implement_workers.append(class_names) 31 | else: 32 | implement_workers.append(self.get_class(process_name)) 33 | 34 | for lib in self.import_libs: 35 | str_processor += lib + '\n' 36 | str_processor += '\n' 37 | 38 | for implement in self.implement_classes: 39 | str_processor += implement + '\n' 40 | 41 | for worker in implement_workers: 42 | if isinstance(worker, list): 43 | str_processor += 'chain_processor.add_processor([%s])\n' % ', '.join(worker) 44 | else: 45 | str_processor += 'chain_processor.add_processor(%s)\n' % worker 46 | 47 | str_processor += '%s.run(chain_processor.process)' % implement_workers[0] 48 | 49 | with open('_processor.py', 'w') as of: 50 | of.write(str_processor) 51 | 52 | return str_processor 53 | 54 | def get_object(self, argument): 55 | """ 56 | Detect object(dict, variable, object) and return string of object 57 | Args: 58 | argument: list: [..., data_type] 59 | """ 60 | argument = naming_services.normalize_service_argument(argument) 61 | if argument[-1] == 'dict': 62 | dict_argument = {} 63 | for key, value in argument[0].items(): 64 | if value[-1] == 'object': 65 | dict_argument[key] = self.get_class(value[0]) 66 | elif value[-1] == 'variable': 67 | dict_argument[key] = self.setting_loader[value[0]] 68 | 69 | return str(dict_argument) 70 | 71 | elif argument[-1] == 'object': 72 | return str(self.get_class(argument[0])) 73 | 74 | elif argument[-1] == 'variable': 75 | variable = self.setting_loader[argument[0]] 76 | 77 | if hasattr(variable, '__iter__'): 78 | variable = ["'" + x + "'" if type(x) is str else str(x) for x in variable] 79 | return str(variable) 80 | elif type(variable) is str: 81 | return "'" + variable + "'" 82 | else: 83 | return str(variable) 84 | 85 | return None 86 | 87 | def get_class(self, str_class): 88 | """ 89 | Create coding from class: _class = Class(*parameters) 90 | """ 91 | object_name = self.service_loader[str_class]['class'] 92 | class_dir, class_name = naming_services.class_name_extract(object_name) 93 | 94 | str_import = 'from {} import {}'.format(class_dir, class_name) 95 | if str_import not in self.import_libs: 96 | self.import_libs.append(str_import) 97 | 98 | object_arguments = self.service_loader[str_class]['arguments'] 99 | object_arguments = object_arguments if (type(object_arguments) is list) else [object_arguments] 100 | object_arguments = [self.get_object(x) for x in object_arguments] 101 | if object_arguments[0] is None: 102 | object_arguments = [''] 103 | 104 | str_deliver = '{} = {}({})'.format(self.generate_class_name(class_name), class_name, ', '.join(object_arguments)) 105 | 106 | if str_deliver not in self.implement_classes: 107 | self.implement_classes.append(str_deliver) 108 | 109 | return self.generate_class_name(class_name) 110 | 111 | @staticmethod 112 | def generate_class_name(class_name): 113 | return naming_services.get_deliver_from_class(class_name) 114 | 115 | 116 | def main(file_setting='settings.yml'): 117 | generate_processor = GenerateProcessor() 118 | print(generate_processor.process(file_setting)) 119 | 120 | 121 | if __name__ == '__main__': 122 | main() 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ANK - Python Streaming system ## 2 | 3 | 4 | ### Overview: ### 5 | Python Streaming system, REST-API and Schedule task using queue message(rabbitMQ, zeroMQ, kafka) 6 | Each processor is a chain, put it together to make a pipeline. 7 | 8 | 9 | ### Support chain model: 10 | * 1 - 1: 11 | ``` 12 | processor - processor 13 | ``` 14 | * 1 - n: 15 | ``` 16 | / processor 17 | processor - processor 18 | \ processor 19 | ``` 20 | * n - 1: 21 | ``` 22 | processor \ 23 | processor - processor 24 | processor / 25 | ``` 26 | * n - n: 27 | ``` 28 | processor - processor processor - processor processor - processor 29 | processor - processor processor / processor < 30 | processor - processor processor - processor processor - processor 31 | ``` 32 | * join message: 33 | ``` 34 | message1 \ 35 | message2 - [message1, message2, message3] 36 | message3 / 37 | ``` 38 | * split message: 39 | ``` 40 | / message1 41 | [message1, message2, message3] - message2 42 | \ message3 43 | ``` 44 | 45 | 46 | ### Requirements: ### 47 | * Python 2.x 48 | * Python 3.x 49 | 50 | 51 | ### How to use: ### 52 | * **Install:** 53 | * From github: 54 | - clone this project: `git clone git@github.com:sunary/ank.git` 55 | - install: `python setup.py install` 56 | * From pypi: 57 | - `pip install ank` 58 | 59 | * **Create new Service:** 60 | ```shell 61 | $ ank -c TestService 62 | $ cd TestService 63 | ``` 64 | then continue using below commands 65 | 66 | * **Edit app (processor.py):** 67 | * *Example:* 68 | ```python 69 | from ank.components.pipe_app import PipeApp 70 | 71 | class ExampleApp(PipeApp): 72 | 73 | def start(self): 74 | for i in range(100): 75 | self.chain_process(i) 76 | 77 | def process(self, message=None): 78 | ''' 79 | Args: 80 | message: {'content': (*) 'content of message', 81 | 'flags': (list|tuple) 'define next process will be use'} 82 | raise TypeError if you don't declare this in return of before branching-processor 83 | if 'flags' == [True, True]: process both in next branching-processors 84 | if 'flags' == [True, False]: process 1st processor in next branching-processors 85 | if 'flags' == [False, True]: process 2nd processor in next branching-processors 86 | if 'flags' == [False, False]: no processor, stop chain 87 | is None: stop chain 88 | ''' 89 | return message['content'] + 1 90 | ``` 91 | 92 | * **Edit services and chains (services.yml):** 93 | * *Syntax:* 94 | ```yaml 95 | services: 96 | Object1: 97 | - class: module.name.ClassName 98 | - arguments: [$Object, %variable%] 99 | 100 | AnkChain2: 101 | - class: ank.components.module_name.XXXApps 102 | - arguments: ~ 103 | 104 | chains: 105 | - Object1 106 | - AnkChain2 107 | ``` 108 | * *Example:* 109 | ```yaml 110 | services: 111 | StartApp: 112 | class: processor.StartApp 113 | arguments: [$MongoClient, $Redis, '%batch_size%'] 114 | 115 | Mongodb: 116 | class: pymongo.MongoClient 117 | arguments: ['%mongo_host%', '%mongo_port%'] 118 | 119 | Redis: 120 | class: redis.client.StrictRedis 121 | arguments: ['%redis_host%', '%redis_port%'] 122 | 123 | OtherApp: 124 | class: processor.OtherApp 125 | arguments: ~ 126 | 127 | LogApp: 128 | class: components.log_app.LogApp 129 | arguments: ~ 130 | 131 | chains: 132 | - StartApp 133 | - LogApp 134 | - OtherApp 135 | - [OddApp, EvenApp] # will be processed depend on key `flags` 136 | - OtherApp 137 | ``` 138 | ANK will read top-down `chains`, find correspond `services` and get parameters from `settings.yml`. 139 | 140 | * **Generate and edit setting (settings.yml):** 141 | 142 | ```shell 143 | $ ank create -c BaseApp 144 | ``` 145 | 146 | * *Example:* 147 | 148 | ```yaml 149 | parameters: 150 | mongo_host: localhost 151 | mongo_port: 27017 152 | 153 | redis_host: localhost 154 | redis_port: 6379 155 | 156 | queue_uri: 'amqp://admin:admin@localhost:5672/' 157 | exchange_name: InputExchange 158 | routing_key: ExchangeToQueue 159 | 160 | batch_size: 100 161 | ``` 162 | 163 | Help you create `settings` template file. Just rename from `_settings.yml` to `settings.yml` and fill in values. 164 | 165 | * **Build Service (create docker image) and run:** 166 | 167 | ```shell 168 | $ ank build 169 | $ docker run --entrypoint /bin/sh $docker_image_id 170 | ``` 171 | 172 | * **Generate processor (_processor.py):** 173 | 174 | ```shell 175 | $ ank gen_processor -fs settings.yml 176 | ``` 177 | * **Generate setting (_setting.yml):** 178 | 179 | ```shell 180 | $ ank gen_setting -fs _setting.yml 181 | ``` 182 | * **Run Service:** 183 | 184 | ```python 185 | from ank import program_loader 186 | program_loader.main(file_setting='settings.yml') 187 | ``` 188 | 189 | or 190 | 191 | ```shell 192 | $ ank run -fs settings.yml 193 | ``` 194 | 195 | ### Base Apps: ### 196 | * **PipeApp:** Pipeline App. 197 | * **APIApp:** REST-API interface using flask. 198 | * **ScheduleApp:** Using crondtab-time format to set schedule. 199 | 200 | 201 | ### Component Apps: ### 202 | * **LogApp:** Log every messages. 203 | * **JoinApp:** Join messages into one. 204 | * **SplitApp:** Split message. 205 | * **---Consumer:** Get message from queue. 206 | * **---Producer:** Push message to queue. 207 | 208 | 209 | ### TODO 210 | 211 | - [x] base apps: Base, API, Schedule 212 | - [x] supported chain: join, split 213 | - [x] supported consumer/producer: redis, kafka, zmq, rabbitmq 214 | - [ ] supported callback success/fail 215 | - [x] generate: processor.py, settings.yml 216 | - [ ] management/admin 217 | - [ ] build/deploy 218 | - [x] sample: PipeApp, ScheduleApp, APIApp 219 | - [ ] test 220 | -------------------------------------------------------------------------------- /ank/cli.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sunary' 2 | 3 | 4 | import sys 5 | import os 6 | import argparse 7 | import pprint 8 | import generate_processor, generate_setting, program_loader 9 | from ank import VERSION, API_DEFAULT_PORT 10 | from ank.utils import cmd_helpers 11 | 12 | 13 | CURRENT_PATH = os.path.dirname(os.path.abspath(__file__)) 14 | 15 | 16 | def create(prj_name, baseapp): 17 | if not os.path.exists(prj_name): 18 | os.makedirs(prj_name) 19 | 20 | with open('{}/__init__.py'.format(prj_name), 'w') as create_file: 21 | create_file.write('') 22 | 23 | with open('{}/processor.py'.format(prj_name), 'w') as create_file: 24 | create_file.write(processor_template(prj_name, baseapp)) 25 | 26 | with open('{}/test_service.py'.format(prj_name), 'w') as create_file: 27 | create_file.write(unittest_template(prj_name)) 28 | 29 | with open('{}/requirements.txt'.format(prj_name), 'w') as create_file: 30 | create_file.write('ank=={}'.format(VERSION)) 31 | 32 | with open('{}/Dockerfile'.format(prj_name), 'w') as create_file: 33 | create_file.write(docker_template(prj_name)) 34 | 35 | with open('{}/services.yml'.format(prj_name), 'w') as create_file: 36 | create_file.write(services_template(prj_name, baseapp)) 37 | 38 | with open('{}/settings.yml'.format(prj_name), 'w') as create_file: 39 | create_file.write(settings_template(baseapp)) 40 | 41 | with open('{}/README.md'.format(prj_name), 'w') as create_file: 42 | create_file.write(readme_template(prj_name)) 43 | 44 | 45 | def processor_template(prj_name, baseapp): 46 | if baseapp == 'BaseApp': 47 | with open(os.path.join(CURRENT_PATH, '../templates/baseapp_processor.tpy'), 'r') as of: 48 | return of.read().format(prj_name) 49 | 50 | elif baseapp == 'APIApp': 51 | with open(os.path.join(CURRENT_PATH, '../templates/apiapp_endpoint.tpy'), 'r') as of: 52 | endpoint_content = of.read().format(prj_name, API_DEFAULT_PORT) 53 | 54 | with open(os.path.join(CURRENT_PATH, '{}/endpoint.py'.format(prj_name)), 'w') as create_file: 55 | create_file.write(endpoint_content) 56 | 57 | with open(os.path.join(CURRENT_PATH, '../templates/apiapp_processor.tpy'), 'r') as of: 58 | return of.read().format(prj_name, API_DEFAULT_PORT) 59 | 60 | elif baseapp == 'ScheduleApp': 61 | with open(os.path.join(CURRENT_PATH, '../templates/scheduleapp_processor.tpy'), 'r') as of: 62 | return of.read().format(prj_name) 63 | 64 | else: 65 | raise Exception('{} not found'.format(baseapp)) 66 | 67 | 68 | def unittest_template(prj_name): 69 | with open(os.path.join(CURRENT_PATH, '../templates/unittest.tpy'), 'r') as of: 70 | return of.read().format(prj_name).format(prj_name) 71 | 72 | 73 | def docker_template(prj_name): 74 | with open(os.path.join(CURRENT_PATH, '../templates/docker.tpy'), 'r') as of: 75 | return of.read().format(prj_name).format(prj_name) 76 | 77 | 78 | def services_template(prj_name, baseapp): 79 | if baseapp == 'BaseApp': 80 | with open(os.path.join(CURRENT_PATH, '../templates/baseapp_services.tpy'), 'r') as of: 81 | return of.read().format(prj_name) 82 | 83 | elif baseapp == 'APIApp': 84 | with open(os.path.join(CURRENT_PATH, '../templates/apiapp_services.tpy'), 'r') as of: 85 | return of.read().format(prj_name) 86 | 87 | elif baseapp == 'ScheduleApp': 88 | with open(os.path.join(CURRENT_PATH, '../templates/scheduleapp_services.tpy'), 'r') as of: 89 | return of.read().format(prj_name) 90 | 91 | else: 92 | raise Exception('{} not found'.format(baseapp)) 93 | 94 | 95 | def settings_template(baseapp): 96 | if baseapp == 'BaseApp': 97 | with open(os.path.join(CURRENT_PATH, '../templates/baseapp_settings.tpy'), 'r') as of: 98 | return of.read() 99 | 100 | elif baseapp == 'APIApp': 101 | with open(os.path.join(CURRENT_PATH, '../templates/apiapp_settings.tpy'), 'r') as of: 102 | return of.read().format(API_DEFAULT_PORT) 103 | 104 | elif baseapp == 'ScheduleApp': 105 | with open(os.path.join(CURRENT_PATH, '../templates/scheduleapp_settings.tpy'), 'r') as of: 106 | return of.read() 107 | 108 | else: 109 | raise Exception('{} not found'.format(baseapp)) 110 | 111 | 112 | def readme_template(prj_name): 113 | with open(os.path.join(CURRENT_PATH, '../templates/readme.tpy'), 'r') as of: 114 | return of.read().format(prj_name) 115 | 116 | 117 | def create_setting(file_setting): 118 | generate_setting.main(file_setting) 119 | 120 | 121 | def create_processor(file_setting): 122 | generate_processor.main(file_setting) 123 | 124 | 125 | def test_service(): 126 | print(cmd_helpers.run_cmd(['python', '-m', 'unittest', 'test_service'])['message']) 127 | 128 | 129 | def run_service(file_setting): 130 | program_loader.main(file_setting=file_setting) 131 | 132 | 133 | def build_service(): 134 | print(cmd_helpers.run_cmd(['docker', 'build', '.'])['message']) 135 | 136 | 137 | def parse_args(args): 138 | parser = argparse.ArgumentParser(prog='Ank Streaming system') 139 | subparsers = parser.add_subparsers(dest='subparser') 140 | 141 | create_parser = subparsers.add_parser('create', 142 | help='Create new service') 143 | create_parser.add_argument('-c', '--class', help='App class: [default=`BaseApp`, `APIApp`, `ScheduleApp`]') 144 | 145 | gen_setting_parser = subparsers.add_parser('gen_setting', help='Generate `setting.yml` file') 146 | gen_setting_parser.add_argument('-fs', '--file_setting', help='Setting file, default `_setting.yml`') 147 | 148 | gen_processor_parser = subparsers.add_parser('gen_processor', help='Generate `_processor.py` file') 149 | gen_processor_parser.add_argument('-fs', '--file_setting', help='Setting file, default `setting.yml`') 150 | 151 | test_parser = subparsers.add_parser('test', help='Test service') 152 | 153 | run_parser = subparsers.add_parser('run', help='Run service') 154 | run_parser.add_argument('-fs', '--file_setting', help='Setting file, default `setting.yml`') 155 | 156 | build_parser = subparsers.add_parser('build', help='Build service') 157 | 158 | return parser.parse_args(args) 159 | 160 | 161 | if __name__ == '__main__': 162 | args = parse_args(sys.argv[1:]) 163 | 164 | pprint.pprint(args) 165 | if args.subparser_name == 'create': 166 | if args.c and args.c in ['BaseApp', 'APIApp', 'ScheduleApp']: 167 | baseapp = args.app 168 | else: 169 | baseapp = 'BaseApp' 170 | 171 | create(args.create, baseapp) 172 | elif args.subparser_name == 'gen_setting': 173 | create_setting(args.fs or '_settings.yml') 174 | elif args.subparser_name == 'gen_processor': 175 | create_processor(args.fs or 'setting.yml') 176 | elif args.subparser_name == 'test': 177 | test_service() 178 | elif args.subparser_name == 'run': 179 | run_service(args.fs or 'setting.yml') 180 | elif args.subparser_name == 'build': 181 | build_service() 182 | -------------------------------------------------------------------------------- /ank/daemon.py: -------------------------------------------------------------------------------- 1 | """ 2 | *** 3 | Modified generic daemon class 4 | *** 5 | 6 | Author: http://www.jejik.com/articles/2007/02/ 7 | a_simple_unix_linux_daemon_in_python/www.boxedice.com 8 | 9 | License: http://creativecommons.org/licenses/by-sa/3.0/ 10 | 11 | Changes: 23rd Jan 2009 (David Mytton ) 12 | - Replaced hard coded '/dev/null in __init__ with os.devnull 13 | - Added OS check to conditionally remove code that doesn't 14 | work on OS X 15 | - Added output to console on completion 16 | - Tidied up formatting 17 | 11th Mar 2009 (David Mytton ) 18 | - Fixed problem with daemon exiting on Python 2.4 19 | (before SystemExit was part of the Exception base) 20 | 13th Aug 2010 (David Mytton 21 | - Fixed unhandled exception if PID file is empty 22 | """ 23 | 24 | # Core modules 25 | import atexit 26 | import os 27 | import sys 28 | import time 29 | import signal 30 | 31 | 32 | class Daemon(object): 33 | """ 34 | A generic daemon class. 35 | 36 | Usage: subclass the Daemon class and override the run() method 37 | Notes: umask=022 38 | """ 39 | def __init__(self, pidfile, stdin=os.devnull, 40 | stdout=os.devnull, stderr=os.devnull, 41 | home_dir='.', umask=18, verbose=1, use_gevent=False): 42 | self.stdin = stdin 43 | self.stdout = stdout 44 | self.stderr = stderr 45 | self.pidfile = pidfile 46 | self.home_dir = home_dir 47 | self.verbose = verbose 48 | self.umask = umask 49 | self.daemon_alive = True 50 | self.use_gevent = use_gevent 51 | 52 | def daemonize(self): 53 | """ 54 | Do the UNIX double-fork magic, see Stevens' "Advanced 55 | Programming in the UNIX Environment" for details (ISBN 0201563177) 56 | http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 57 | """ 58 | try: 59 | pid = os.fork() 60 | if pid > 0: 61 | # Exit first parent 62 | sys.exit(0) 63 | except OSError as e: 64 | sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror)) 65 | sys.exit(1) 66 | 67 | # Decouple from parent environment 68 | os.chdir(self.home_dir) 69 | os.setsid() 70 | os.umask(self.umask) 71 | 72 | # Do second fork 73 | try: 74 | pid = os.fork() 75 | if pid > 0: 76 | # Exit from second parent 77 | sys.exit(0) 78 | except OSError as e: 79 | sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror)) 80 | sys.exit(1) 81 | 82 | if sys.platform != 'darwin': # This block breaks on OS X 83 | # Redirect standard file descriptors 84 | sys.stdout.flush() 85 | sys.stderr.flush() 86 | si = file(self.stdin, 'r') 87 | so = file(self.stdout, 'a+') 88 | if self.stderr: 89 | se = file(self.stderr, 'a+', 0) 90 | else: 91 | se = so 92 | os.dup2(si.fileno(), sys.stdin.fileno()) 93 | os.dup2(so.fileno(), sys.stdout.fileno()) 94 | os.dup2(se.fileno(), sys.stderr.fileno()) 95 | 96 | def sigtermhandler(signum, frame): 97 | self.daemon_alive = False 98 | sys.exit() 99 | 100 | if self.use_gevent: 101 | import gevent 102 | gevent.reinit() 103 | gevent.signal(signal.SIGTERM, sigtermhandler, signal.SIGTERM, None) 104 | gevent.signal(signal.SIGINT, sigtermhandler, signal.SIGINT, None) 105 | else: 106 | signal.signal(signal.SIGTERM, sigtermhandler) 107 | signal.signal(signal.SIGINT, sigtermhandler) 108 | 109 | if self.verbose >= 1: 110 | print('Started') 111 | 112 | # Write pidfile 113 | atexit.register( 114 | self.delpid) # Make sure pid file is removed if we quit 115 | pid = str(os.getpid()) 116 | file(self.pidfile, 'w+').write('%s\n' % pid) 117 | 118 | def delpid(self): 119 | os.remove(self.pidfile) 120 | 121 | def start(self, *args, **kwargs): 122 | """ 123 | Start the daemon 124 | """ 125 | 126 | if self.verbose >= 1: 127 | print('Starting...') 128 | 129 | # Check for a pidfile to see if the daemon already runs 130 | try: 131 | pf = file(self.pidfile, 'r') 132 | pid = int(pf.read().strip()) 133 | pf.close() 134 | except IOError: 135 | pid = None 136 | except SystemExit: 137 | pid = None 138 | 139 | if pid: 140 | message = 'pidfile %s already exists. Is it already running?\n' 141 | sys.stderr.write(message % self.pidfile) 142 | sys.exit(1) 143 | 144 | # Start the daemon 145 | self.daemonize() 146 | 147 | def stop(self): 148 | """ 149 | Stop the daemon 150 | """ 151 | 152 | if self.verbose >= 1: 153 | print('Stopping...') 154 | 155 | # Get the pid from the pidfile 156 | pid = self.get_pid() 157 | 158 | if not pid: 159 | message = 'pidfile %s does not exist. Not running?\n' 160 | sys.stderr.write(message % self.pidfile) 161 | 162 | # Just to be sure. A ValueError might occur if the PID file is 163 | # empty but does actually exist 164 | if os.path.exists(self.pidfile): 165 | os.remove(self.pidfile) 166 | 167 | return # Not an error in a restart 168 | 169 | # Try killing the daemon process 170 | try: 171 | i = 0 172 | while True: 173 | os.kill(pid, signal.SIGTERM) 174 | time.sleep(0.1) 175 | i += 1 176 | if i % 10 == 0: 177 | os.kill(pid, signal.SIGHUP) 178 | except OSError as err: 179 | err = str(err) 180 | if err.find('No such process') > 0: 181 | if os.path.exists(self.pidfile): 182 | os.remove(self.pidfile) 183 | else: 184 | print(str(err)) 185 | sys.exit(1) 186 | 187 | if self.verbose >= 1: 188 | print('Stopped') 189 | 190 | def restart(self): 191 | """ 192 | Restart the daemon 193 | """ 194 | self.stop() 195 | self.start() 196 | 197 | def get_pid(self): 198 | try: 199 | pf = file(self.pidfile, 'r') 200 | pid = int(pf.read().strip()) 201 | pf.close() 202 | except IOError: 203 | pid = None 204 | except SystemExit: 205 | pid = None 206 | return pid 207 | 208 | def is_running(self): 209 | pid = self.get_pid() 210 | 211 | if pid is None: 212 | print('Process is stopped') 213 | elif os.path.exists('/proc/%d' % pid): 214 | print('Process (pid %d) is running...' % pid) 215 | else: 216 | print('Process (pid %d) is killed' % pid) 217 | 218 | return pid and os.path.exists('/proc/%d' % pid) 219 | 220 | 221 | if __name__ == '__main__': 222 | daemon = Daemon('daemon.pid') 223 | daemon.start() 224 | print(os.getpid()) 225 | while True: 226 | time.sleep(1) 227 | --------------------------------------------------------------------------------