├── SpiderKeeper ├── app │ ├── proxy │ │ ├── __init__.py │ │ ├── contrib │ │ │ ├── __init__.py │ │ │ └── scrapy.py │ │ └── spiderctrl.py │ ├── spider │ │ ├── __init__.py │ │ ├── model.py │ │ └── controller.py │ ├── schedulers │ │ ├── __init__.py │ │ └── common.py │ ├── static │ │ ├── fonts │ │ │ ├── ionicons.eot │ │ │ ├── ionicons.ttf │ │ │ ├── FontAwesome.otf │ │ │ ├── ionicons.woff │ │ │ ├── fontawesome-webfont.eot │ │ │ ├── fontawesome-webfont.ttf │ │ │ ├── fontawesome-webfont.woff │ │ │ ├── fontawesome-webfont.woff2 │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ ├── js │ │ │ ├── pages │ │ │ │ └── base.js │ │ │ ├── html5shiv.min.js │ │ │ ├── respond.min.js │ │ │ ├── jquery.slimscroll.min.js │ │ │ ├── fastclick.min.js │ │ │ ├── AdminLTE.min.js │ │ │ └── demo.js │ │ └── css │ │ │ ├── app.css │ │ │ ├── skins │ │ │ └── skin-black-light.min.css │ │ │ └── font-awesome.min.css │ ├── util │ │ ├── __init__.py │ │ └── http.py │ ├── templates │ │ ├── job_log.html │ │ ├── server_stats.html │ │ ├── project_manage.html │ │ ├── spider_dashboard.html │ │ ├── spider_deploy.html │ │ ├── project_stats.html │ │ ├── base.html │ │ ├── job_periodic.html │ │ └── job_dashboard.html │ └── __init__.py ├── __init__.py ├── uwsgi.py ├── config.py └── run.py ├── MANIFEST.in ├── screenshot ├── screenshot_1.png ├── screenshot_2.png ├── screenshot_3.png ├── donate_wechat.png └── qqgroup_qrcode.png ├── .gitignore ├── requirements.txt ├── CHANGELOG.md ├── setup.py └── README.md /SpiderKeeper/app/proxy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /SpiderKeeper/app/spider/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /SpiderKeeper/app/proxy/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /SpiderKeeper/app/schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /SpiderKeeper/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.2.0' 2 | __author__ = 'Dormy Mo' 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include SpiderKeeper/app/templates * 2 | recursive-include SpiderKeeper/app/static * -------------------------------------------------------------------------------- /screenshot/screenshot_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/screenshot/screenshot_1.png -------------------------------------------------------------------------------- /screenshot/screenshot_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/screenshot/screenshot_2.png -------------------------------------------------------------------------------- /screenshot/screenshot_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/screenshot/screenshot_3.png -------------------------------------------------------------------------------- /screenshot/donate_wechat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/screenshot/donate_wechat.png -------------------------------------------------------------------------------- /screenshot/qqgroup_qrcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/screenshot/qqgroup_qrcode.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.pyc 3 | docs/_build 4 | *egg-info 5 | .tox 6 | .coverage 7 | venv 8 | dist 9 | build 10 | SpiderKeeper.db 11 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/ionicons.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/ionicons.eot -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/ionicons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/ionicons.ttf -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/ionicons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/ionicons.woff -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/pages/base.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by modm on 2017/4/7. 3 | */ 4 | $(document).ready() 5 | $.get("/api/projects",function (data){ 6 | 7 | }) -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaussic/SpiderKeeper/master/SpiderKeeper/app/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /SpiderKeeper/app/util/__init__.py: -------------------------------------------------------------------------------- 1 | def project_path(): 2 | import inspect, os 3 | this_file = inspect.getfile(inspect.currentframe()) 4 | return os.path.abspath(os.path.dirname(this_file)+'/../') -------------------------------------------------------------------------------- /SpiderKeeper/uwsgi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2017-09-24 14:53 4 | # @Author : modm 5 | ''' 6 | you can start the server by uwsgi 7 | like gunicorn -w 4 SpiderKeeper.uwsgi:app 8 | ''' 9 | from SpiderKeeper.app import app, initialize 10 | 11 | initialize() 12 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/css/app.css: -------------------------------------------------------------------------------- 1 | @media (max-width: 767px) { 2 | .nav-xs { 3 | margin: 0; 4 | } 5 | } 6 | 7 | .txt-args { 8 | font-size: 10px; 9 | display: block; 10 | white-space: nowrap; 11 | overflow: hidden; 12 | text-overflow: ellipsis; 13 | width: 100px; 14 | } -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/job_log.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | {% for line in log_lines %} 13 |

{{ line }}

14 | {% endfor %} 15 | 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aniso8601==1.2.0 2 | APScheduler==3.3.1 3 | click==6.7 4 | Flask==0.12.1 5 | Flask-BasicAuth==0.2.0 6 | Flask-RESTful==0.3.5 7 | flask-restful-swagger==0.19 8 | Flask-SQLAlchemy==2.2 9 | itsdangerous==0.24 10 | Jinja2==2.9.6 11 | MarkupSafe==1.0 12 | PyMySQL==0.7.11 13 | python-dateutil==2.6.0 14 | pytz==2017.2 15 | requests==2.13.0 16 | six==1.10.0 17 | SQLAlchemy==1.1.9 18 | tzlocal==1.3 19 | Werkzeug==0.12.1 20 | -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/server_stats.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Server Stats

4 | {% endblock %} 5 | {% block content_body %} 6 |
7 |
8 |

Server Stats

9 |
10 |
11 |
12 |
13 |
14 | {% for server in servers %} 15 |
16 | 17 |
18 | {{ server }} 19 |
20 |
21 |
22 |
23 |
24 | {% endfor %} 25 |
26 |
27 |
28 | {% endblock %} -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # SpiderKeeper Changelog 2 | ## 1.2.0 (2017-07-24) 3 | - support chose server manually 4 | - support set cron exp manually 5 | - fix log Chinese decode problem 6 | - fix scheduler trigger not fire problem 7 | - fix not delete project on scrapyd problem 8 | 9 | ## 1.1.0 (2017-04-25) 10 | - support basic auth 11 | - show spider crawl time info (last_runtime,avg_runtime) 12 | - optimized for mobile 13 | 14 | ## 1.0.3 (2017-04-17) 15 | - support view log 16 | 17 | ## 1.0.0 (2017-03-30) 18 | - refactor 19 | - support py3 20 | - optimized api 21 | - optimized scheduler 22 | - more scalable (can support access multiply spider service) 23 | - show running stats 24 | 25 | ## 0.2.0 (2016-04-13) 26 | - support view job of multi daemons. 27 | - support run on multi daemons. 28 | - support choice running daemon automaticaly. 29 | 30 | ## 0.1.1 (2016-02-16) 31 | - add status monitor(https://github.com/afaqurk/linux-dash) 32 | 33 | ## 0.1.0 (2016-01-18) 34 | - initial. -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/project_manage.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Project Manage

4 | {% endblock %} 5 | {% block content_body %} 6 | {% with messages = get_flashed_messages() %} 7 | {% if messages %} 8 |
9 | 14 |
15 | {% endif %} 16 | {% endwith %} 17 |
18 |
19 | {% if project %} 20 |

{{ project.project_name }}

21 | {% endif %} 22 |
23 |
24 |
25 |
26 | {% if project %} 27 | Delete Project 28 | {% else %} 29 | Create Project 31 | 32 | {% endif %} 33 |
34 |
35 | {% endblock %} -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/spider_dashboard.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Spider

4 | {% endblock %} 5 | {% block content_body %} 6 |
7 |
8 |

Periodic jobs (Spiders)

9 |
10 |
11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {% for spider_instance in spider_instance_list %} 19 | 20 | 21 | 22 | 23 | 24 | 25 | {% endfor %} 26 |
idSpider NameLast RuntimeAvg Runtime
{{ spider_instance.spider_instance_id }}{{ spider_instance.spider_name }}{{ spider_instance.spider_last_runtime }}{{ readable_time(spider_instance.spider_avg_runtime) }}
27 |
28 |
29 | {% endblock %} -------------------------------------------------------------------------------- /SpiderKeeper/config.py: -------------------------------------------------------------------------------- 1 | # Statement for enabling the development environment 2 | import os 3 | 4 | DEBUG = True 5 | 6 | # Define the application directory 7 | 8 | 9 | BASE_DIR = os.path.abspath(os.path.dirname(__file__)) 10 | 11 | SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(os.path.abspath('.'), 'SpiderKeeper.db') 12 | SQLALCHEMY_TRACK_MODIFICATIONS = False 13 | DATABASE_CONNECT_OPTIONS = {} 14 | 15 | # Application threads. A common general assumption is 16 | # using 2 per available processor cores - to handle 17 | # incoming requests using one and performing background 18 | # operations using the other. 19 | THREADS_PER_PAGE = 2 20 | 21 | # Enable protection agains *Cross-site Request Forgery (CSRF)* 22 | CSRF_ENABLED = True 23 | 24 | # Use a secure, unique and absolutely secret key for 25 | # signing the data. 26 | CSRF_SESSION_KEY = "secret" 27 | 28 | # Secret key for signing cookies 29 | SECRET_KEY = "secret" 30 | 31 | # log 32 | LOG_LEVEL = 'INFO' 33 | 34 | # spider services 35 | SERVER_TYPE = 'scrapyd' 36 | SERVERS = ['http://localhost:6800'] 37 | 38 | # basic auth 39 | NO_AUTH = False 40 | BASIC_AUTH_USERNAME = 'admin' 41 | BASIC_AUTH_PASSWORD = 'admin' 42 | BASIC_AUTH_FORCE = True 43 | -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/spider_deploy.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Deploy

4 | {% endblock %} 5 | {% block content_body %} 6 | 7 | {% with messages = get_flashed_messages() %} 8 | {% if messages %} 9 |
10 | 15 |
16 | {% endif %} 17 | {% endwith %} 18 | 19 |
20 |
21 |
22 |
23 | 24 | 25 | 26 |

Use scrapyd-client 27 | to generate egg file scrapyd-deploy --build-egg output.egg

28 |
29 |
30 |
31 | 32 |
33 |
34 |
35 | {% endblock %} -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from os import path 3 | from setuptools import setup, find_packages 4 | 5 | from SpiderKeeper import __version__, __author__ 6 | 7 | here = path.abspath(path.dirname(__file__)) 8 | with open(path.join(here, 'requirements.txt'), encoding='utf-8') as f: 9 | all_reqs = f.read().split('\n') 10 | 11 | install_requires = [x.strip() for x in all_reqs if 'git+' not in x] 12 | 13 | setup( 14 | name='SpiderKeeper', 15 | version=__version__, 16 | description='Admin ui for spider service', 17 | long_description= 18 | 'Go to https://github.com/DormyMo/SpiderKeeper/ for more information.', 19 | author=__author__, 20 | author_email='modongming91@gmail.com', 21 | url='https://github.com/DormyMo/SpiderKeeper/', 22 | license='MIT', 23 | include_package_data=True, 24 | packages=find_packages(), 25 | install_requires=install_requires, 26 | 27 | entry_points={ 28 | 'console_scripts': { 29 | 'spiderkeeper = SpiderKeeper.run:main' 30 | }, 31 | }, 32 | 33 | classifiers=[ 34 | 'Development Status :: 4 - Beta', 35 | 'License :: OSI Approved :: MIT License', 36 | 'Operating System :: OS Independent', 37 | 'Programming Language :: Python :: 2.7', 38 | 'Programming Language :: Python :: 3.5', 39 | ], 40 | ) 41 | -------------------------------------------------------------------------------- /SpiderKeeper/app/util/http.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import requests 4 | 5 | 6 | def request_get(url, retry_times=5): 7 | ''' 8 | :param url: 9 | :param retry_times: 10 | :return: response obj 11 | ''' 12 | for i in range(retry_times): 13 | try: 14 | res = requests.get(url) 15 | except Exception as e: 16 | logging.warning('request error retry %s' % url) 17 | continue 18 | return res 19 | 20 | 21 | def request_post(url, data, retry_times=5): 22 | ''' 23 | :param url: 24 | :param retry_times: 25 | :return: response obj 26 | ''' 27 | for i in range(retry_times): 28 | try: 29 | res = requests.post(url, data) 30 | except Exception as e: 31 | logging.warning('request error retry %s' % url) 32 | continue 33 | return res 34 | 35 | 36 | def request(request_type, url, data=None, retry_times=5, return_type="text"): 37 | ''' 38 | 39 | :param request_type: get/post 40 | :param url: 41 | :param data: 42 | :param retry_times: 43 | :param return_type: text/json 44 | :return: 45 | ''' 46 | if request_type == 'get': 47 | res = request_get(url, retry_times) 48 | if request_type == 'post': 49 | res = request_post(url, data, retry_times) 50 | if not res: return res 51 | if return_type == 'text': return res.text 52 | if return_type == 'json': 53 | try: 54 | res = res.json() 55 | return res 56 | except Exception as e: 57 | logging.warning('parse json error %s' % str(e)) 58 | return None 59 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/html5shiv.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | !function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document); -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/project_stats.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Running Stats

4 | {% endblock %} 5 | {% block content_body %} 6 |
7 |
8 |

Spider Running Stats (last 24 hours)

9 |
10 |
11 |
12 | 13 |
14 |
15 |
16 | 17 |
18 |
19 | {% endblock %} 20 | {% block script %} 21 | 22 | 74 | {% endblock %} -------------------------------------------------------------------------------- /SpiderKeeper/run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from optparse import OptionParser 4 | 5 | from SpiderKeeper.app import app, initialize 6 | 7 | 8 | def main(): 9 | opts, args = parse_opts(app.config) 10 | app.config.update(dict( 11 | SERVER_TYPE=opts.server_type, 12 | SERVERS=opts.servers or app.config.get('SERVERS'), 13 | SQLALCHEMY_DATABASE_URI=opts.database_url, 14 | BASIC_AUTH_USERNAME=opts.username, 15 | BASIC_AUTH_PASSWORD=opts.password, 16 | NO_AUTH=opts.no_auth 17 | )) 18 | if opts.verbose: 19 | app.logger.setLevel(logging.DEBUG) 20 | initialize() 21 | app.logger.info("SpiderKeeper startd on %s:%s username:%s/password:%s with %s servers:%s" % ( 22 | opts.host, opts.port, opts.username, opts.password, opts.server_type, ','.join(app.config.get('SERVERS', [])))) 23 | app.run(host=opts.host, port=opts.port, use_reloader=False, threaded=True) 24 | 25 | 26 | def parse_opts(config): 27 | parser = OptionParser(usage="%prog [options]", 28 | description="Admin ui for spider service") 29 | parser.add_option("--host", 30 | help="host, default:0.0.0.0", 31 | dest='host', 32 | default='0.0.0.0') 33 | parser.add_option("--port", 34 | help="port, default:5000", 35 | dest='port', 36 | type="int", 37 | default=5000) 38 | parser.add_option("--username", 39 | help="basic auth username ,default: %s" % config.get('BASIC_AUTH_USERNAME'), 40 | dest='username', 41 | default=config.get('BASIC_AUTH_USERNAME')) 42 | parser.add_option("--password", 43 | help="basic auth password ,default: %s" % config.get('BASIC_AUTH_PASSWORD'), 44 | dest='password', 45 | default=config.get('BASIC_AUTH_PASSWORD')) 46 | parser.add_option("--type", 47 | help="access spider server type, default: %s" % config.get('SERVER_TYPE'), 48 | dest='server_type', 49 | default=config.get('SERVER_TYPE')) 50 | parser.add_option("--server", 51 | help="servers, default: %s" % config.get('SERVERS'), 52 | dest='servers', 53 | action='append', 54 | default=[]) 55 | parser.add_option("--database-url", 56 | help='SpiderKeeper metadata database default: %s' % config.get('SQLALCHEMY_DATABASE_URI'), 57 | dest='database_url', 58 | default=config.get('SQLALCHEMY_DATABASE_URI')) 59 | 60 | parser.add_option("--no-auth", 61 | help="disable basic auth", 62 | dest='no_auth', 63 | action='store_true') 64 | parser.add_option("-v", "--verbose", 65 | help="log level", 66 | dest='verbose', 67 | action='store_true') 68 | return parser.parse_args() 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /SpiderKeeper/app/schedulers/common.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | from SpiderKeeper.app import scheduler, app, agent, db 5 | from SpiderKeeper.app.spider.model import Project, JobInstance, SpiderInstance 6 | 7 | 8 | def sync_job_execution_status_job(): 9 | ''' 10 | sync job execution running status 11 | :return: 12 | ''' 13 | for project in Project.query.all(): 14 | agent.sync_job_status(project) 15 | app.logger.debug('[sync_job_execution_status]') 16 | 17 | 18 | def sync_spiders(): 19 | ''' 20 | sync spiders 21 | :return: 22 | ''' 23 | for project in Project.query.all(): 24 | spider_instance_list = agent.get_spider_list(project) 25 | SpiderInstance.update_spider_instances(project.id, spider_instance_list) 26 | app.logger.debug('[sync_spiders]') 27 | 28 | 29 | def run_spider_job(job_instance_id): 30 | ''' 31 | run spider by scheduler 32 | :param job_instance: 33 | :return: 34 | ''' 35 | try: 36 | job_instance = JobInstance.find_job_instance_by_id(job_instance_id) 37 | agent.start_spider(job_instance) 38 | app.logger.info('[run_spider_job][project:%s][spider_name:%s][job_instance_id:%s]' % ( 39 | job_instance.project_id, job_instance.spider_name, job_instance.id)) 40 | except Exception as e: 41 | app.logger.error('[run_spider_job] ' + str(e)) 42 | 43 | 44 | def reload_runnable_spider_job_execution(): 45 | ''' 46 | add periodic job to scheduler 47 | :return: 48 | ''' 49 | running_job_ids = set([job.id for job in scheduler.get_jobs()]) 50 | # app.logger.debug('[running_job_ids] %s' % ','.join(running_job_ids)) 51 | available_job_ids = set() 52 | # add new job to schedule 53 | for job_instance in JobInstance.query.filter_by(enabled=0, run_type="periodic").all(): 54 | job_id = "spider_job_%s:%s" % (job_instance.id, int(time.mktime(job_instance.date_modified.timetuple()))) 55 | available_job_ids.add(job_id) 56 | if job_id not in running_job_ids: 57 | try: 58 | scheduler.add_job(run_spider_job, 59 | args=(job_instance.id,), 60 | trigger='cron', 61 | id=job_id, 62 | minute=job_instance.cron_minutes, 63 | hour=job_instance.cron_hour, 64 | day=job_instance.cron_day_of_month, 65 | day_of_week=job_instance.cron_day_of_week, 66 | month=job_instance.cron_month, 67 | second=0, 68 | max_instances=999, 69 | misfire_grace_time=60 * 60, 70 | coalesce=True) 71 | except Exception as e: 72 | app.logger.error( 73 | '[load_spider_job] failed {} {},may be cron expression format error '.format(job_id, str(e))) 74 | app.logger.info('[load_spider_job][project:%s][spider_name:%s][job_instance_id:%s][job_id:%s]' % ( 75 | job_instance.project_id, job_instance.spider_name, job_instance.id, job_id)) 76 | # remove invalid jobs 77 | for invalid_job_id in filter(lambda job_id: job_id.startswith("spider_job_"), 78 | running_job_ids.difference(available_job_ids)): 79 | scheduler.remove_job(invalid_job_id) 80 | app.logger.info('[drop_spider_job][job_id:%s]' % invalid_job_id) 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SpiderKeeper 2 | 3 | [![Latest Version](http://img.shields.io/pypi/v/SpiderKeeper.svg)](https://pypi.python.org/pypi/SpiderKeeper) 4 | [![Python Versions](http://img.shields.io/pypi/pyversions/SpiderKeeper.svg)](https://pypi.python.org/pypi/SpiderKeeper) 5 | [![The MIT License](http://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/DormyMo/SpiderKeeper/blob/master/LICENSE) 6 | 7 | A scalable admin ui for spider service 8 | 9 | ## Features 10 | 11 | - Manage your spiders from a dashboard. Schedule them to run automatically 12 | - With a single click deploy the scrapy project 13 | - Show spider running stats 14 | - Provide api 15 | 16 | 17 | Current Support spider service 18 | - [Scrapy](https://github.com/scrapy/scrapy) ( with [scrapyd](https://github.com/scrapy/scrapyd)) 19 | 20 | ## Screenshot 21 | ![job dashboard](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/screenshot_1.png) 22 | ![periodic job](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/screenshot_2.png) 23 | ![running stats](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/screenshot_3.png) 24 | 25 | ## Getting Started 26 | 27 | 28 | ### Installing 29 | 30 | 31 | ``` 32 | pip install spiderkeeper 33 | ``` 34 | 35 | ### Deployment 36 | 37 | ``` 38 | 39 | spiderkeeper [options] 40 | 41 | Options: 42 | 43 | -h, --help show this help message and exit 44 | --host=HOST host, default:0.0.0.0 45 | --port=PORT port, default:5000 46 | --username=USERNAME basic auth username ,default: admin 47 | --password=PASSWORD basic auth password ,default: admin 48 | --type=SERVER_TYPE access spider server type, default: scrapyd 49 | --server=SERVERS servers, default: ['http://localhost:6800'] 50 | --database-url=DATABASE_URL 51 | SpiderKeeper metadata database default: sqlite:////home/souche/SpiderKeeper.db 52 | --no-auth disable basic auth 53 | -v, --verbose log level 54 | 55 | 56 | example: 57 | 58 | spiderkeeper --server=http://localhost:6800 59 | 60 | ``` 61 | 62 | ## Usage 63 | 64 | ``` 65 | Visit: 66 | 67 | - web ui : http://localhost:5000 68 | 69 | 1. Create Project 70 | 71 | 2. Use [scrapyd-client](https://github.com/scrapy/scrapyd-client) to generate egg file 72 | 73 | scrapyd-deploy --build-egg output.egg 74 | 75 | 2. upload egg file (make sure you started scrapyd server) 76 | 77 | 3. Done & Enjoy it 78 | 79 | - api swagger: http://localhost:5000/api.html 80 | 81 | ``` 82 | 83 | ## TODO 84 | - [ ] Job dashboard support filter 85 | - [x] User Authentication 86 | - [ ] Collect & Show scrapy crawl stats 87 | - [ ] Optimize load balancing 88 | 89 | ## Versioning 90 | 91 | We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/DormyMo/SpiderKeeper/tags). 92 | 93 | ## Authors 94 | 95 | - *Initial work* - [DormyMo](https://github.com/DormyMo) 96 | 97 | See also the list of [contributors](https://github.com/DormyMo/SpiderKeeper/contributors) who participated in this project. 98 | 99 | ## License 100 | 101 | This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details 102 | 103 | ## Contributing 104 | 105 | Contributions are welcomed! 106 | 107 | ## 交流反馈 108 | 109 | QQ群： 110 | 111 | 1群: 389688974（已满） 112 | 113 | 2群: 285668943 114 | 115 | ## 捐赠 116 | ![Contact](https://raw.githubusercontent.com/DormyMo/SpiderKeeper/master/screenshot/donate_wechat.png) 117 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/css/skins/skin-black-light.min.css: -------------------------------------------------------------------------------- 1 | .skin-black-light .main-header{-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.05);box-shadow:0 1px 1px rgba(0,0,0,0.05)}.skin-black-light .main-header .navbar-toggle{color:#333}.skin-black-light .main-header .navbar-brand{color:#333;border-right:1px solid #eee}.skin-black-light .main-header .navbar{background-color:#fff}.skin-black-light .main-header .navbar .nav>li>a{color:#333}.skin-black-light .main-header .navbar .nav>li>a:hover,.skin-black-light .main-header .navbar .nav>li>a:active,.skin-black-light .main-header .navbar .nav>li>a:focus,.skin-black-light .main-header .navbar .nav .open>a,.skin-black-light .main-header .navbar .nav .open>a:hover,.skin-black-light .main-header .navbar .nav .open>a:focus,.skin-black-light .main-header .navbar .nav>.active>a{background:#fff;color:#999}.skin-black-light .main-header .navbar .sidebar-toggle{color:#333}.skin-black-light .main-header .navbar .sidebar-toggle:hover{color:#999;background:#fff}.skin-black-light .main-header .navbar>.sidebar-toggle{color:#333;border-right:1px solid #eee}.skin-black-light .main-header .navbar .navbar-nav>li>a{border-right:1px solid #eee}.skin-black-light .main-header .navbar .navbar-custom-menu .navbar-nav>li>a,.skin-black-light .main-header .navbar .navbar-right>li>a{border-left:1px solid #eee;border-right-width:0}.skin-black-light .main-header>.logo{background-color:#fff;color:#333;border-bottom:0 solid transparent;border-right:1px solid #eee}.skin-black-light .main-header>.logo:hover{background-color:#fcfcfc}@media (max-width:767px){.skin-black-light .main-header>.logo{background-color:#222;color:#fff;border-bottom:0 solid transparent;border-right:none}.skin-black-light .main-header>.logo:hover{background-color:#1f1f1f}}.skin-black-light .main-header li.user-header{background-color:#222}.skin-black-light .content-header{background:transparent;box-shadow:none}.skin-black-light .wrapper,.skin-black-light .main-sidebar,.skin-black-light .left-side{background-color:#f9fafc}.skin-black-light .content-wrapper,.skin-black-light .main-footer{border-left:1px solid #d2d6de}.skin-black-light .user-panel>.info,.skin-black-light .user-panel>.info>a{color:#444}.skin-black-light .sidebar-menu>li{-webkit-transition:border-left-color .3s ease;-o-transition:border-left-color .3s ease;transition:border-left-color .3s ease}.skin-black-light .sidebar-menu>li.header{color:#848484;background:#f9fafc}.skin-black-light .sidebar-menu>li>a{border-left:3px solid transparent;font-weight:600}.skin-black-light .sidebar-menu>li:hover>a,.skin-black-light .sidebar-menu>li.active>a{color:#000;background:#f4f4f5}.skin-black-light .sidebar-menu>li.active{border-left-color:#fff}.skin-black-light .sidebar-menu>li.active>a{font-weight:600}.skin-black-light .sidebar-menu>li>.treeview-menu{background:#f4f4f5}.skin-black-light .sidebar a{color:#444}.skin-black-light .sidebar a:hover{text-decoration:none}.skin-black-light .treeview-menu>li>a{color:#777}.skin-black-light .treeview-menu>li.active>a,.skin-black-light .treeview-menu>li>a:hover{color:#000}.skin-black-light .treeview-menu>li.active>a{font-weight:600}.skin-black-light .sidebar-form{border-radius:3px;border:1px solid #d2d6de;margin:10px 10px}.skin-black-light .sidebar-form input[type="text"],.skin-black-light .sidebar-form .btn{box-shadow:none;background-color:#fff;border:1px solid transparent;height:35px}.skin-black-light .sidebar-form input[type="text"]{color:#666;border-top-left-radius:2px;border-top-right-radius:0;border-bottom-right-radius:0;border-bottom-left-radius:2px}.skin-black-light .sidebar-form input[type="text"]:focus,.skin-black-light .sidebar-form input[type="text"]:focus+.input-group-btn .btn{background-color:#fff;color:#666}.skin-black-light .sidebar-form input[type="text"]:focus+.input-group-btn .btn{border-left-color:#fff}.skin-black-light .sidebar-form .btn{color:#999;border-top-left-radius:0;border-top-right-radius:2px;border-bottom-right-radius:2px;border-bottom-left-radius:0}@media (min-width:768px){.skin-black-light.sidebar-mini.sidebar-collapse .sidebar-menu>li>.treeview-menu{border-left:1px solid #d2d6de}} -------------------------------------------------------------------------------- /SpiderKeeper/app/__init__.py: -------------------------------------------------------------------------------- 1 | # Import flask and template operators 2 | import logging 3 | import traceback 4 | 5 | import apscheduler 6 | from apscheduler.schedulers.background import BackgroundScheduler 7 | from flask import Flask 8 | from flask import jsonify 9 | from flask_basicauth import BasicAuth 10 | from flask_restful import Api 11 | from flask_restful_swagger import swagger 12 | from flask_sqlalchemy import SQLAlchemy 13 | from werkzeug.exceptions import HTTPException 14 | 15 | import SpiderKeeper 16 | from SpiderKeeper import config 17 | 18 | # Define the WSGI application object 19 | app = Flask(__name__) 20 | # Configurations 21 | app.config.from_object(config) 22 | 23 | # Logging 24 | log = logging.getLogger('werkzeug') 25 | log.setLevel(logging.ERROR) 26 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 27 | handler = logging.StreamHandler() 28 | handler.setFormatter(formatter) 29 | app.logger.setLevel(app.config.get('LOG_LEVEL', "INFO")) 30 | app.logger.addHandler(handler) 31 | 32 | # swagger 33 | api = swagger.docs(Api(app), apiVersion=SpiderKeeper.__version__, api_spec_url="/api", 34 | description='SpiderKeeper') 35 | # Define the database object which is imported 36 | # by modules and controllers 37 | db = SQLAlchemy(app, session_options=dict(autocommit=False, autoflush=True)) 38 | 39 | 40 | @app.teardown_request 41 | def teardown_request(exception): 42 | if exception: 43 | db.session.rollback() 44 | db.session.remove() 45 | db.session.remove() 46 | 47 | # Define apscheduler 48 | scheduler = BackgroundScheduler() 49 | 50 | 51 | class Base(db.Model): 52 | __abstract__ = True 53 | 54 | id = db.Column(db.Integer, primary_key=True) 55 | date_created = db.Column(db.DateTime, default=db.func.current_timestamp()) 56 | date_modified = db.Column(db.DateTime, default=db.func.current_timestamp(), 57 | onupdate=db.func.current_timestamp()) 58 | 59 | 60 | # Sample HTTP error handling 61 | # @app.errorhandler(404) 62 | # def not_found(error): 63 | # abort(404) 64 | 65 | 66 | @app.errorhandler(Exception) 67 | def handle_error(e): 68 | code = 500 69 | if isinstance(e, HTTPException): 70 | code = e.code 71 | app.logger.error(traceback.print_exc()) 72 | return jsonify({ 73 | 'code': code, 74 | 'success': False, 75 | 'msg': str(e), 76 | 'data': None 77 | }) 78 | 79 | 80 | # Build the database: 81 | from SpiderKeeper.app.spider.model import * 82 | 83 | 84 | def init_database(): 85 | db.init_app(app) 86 | db.create_all() 87 | 88 | 89 | # regist spider service proxy 90 | from SpiderKeeper.app.proxy.spiderctrl import SpiderAgent 91 | from SpiderKeeper.app.proxy.contrib.scrapy import ScrapydProxy 92 | 93 | agent = SpiderAgent() 94 | 95 | 96 | def regist_server(): 97 | if app.config.get('SERVER_TYPE') == 'scrapyd': 98 | for server in app.config.get("SERVERS"): 99 | agent.regist(ScrapydProxy(server)) 100 | 101 | 102 | from SpiderKeeper.app.spider.controller import api_spider_bp 103 | 104 | # Register blueprint(s) 105 | app.register_blueprint(api_spider_bp) 106 | 107 | # start sync job status scheduler 108 | from SpiderKeeper.app.schedulers.common import sync_job_execution_status_job, sync_spiders, \ 109 | reload_runnable_spider_job_execution 110 | 111 | scheduler.add_job(sync_job_execution_status_job, 'interval', seconds=5, id='sys_sync_status') 112 | scheduler.add_job(sync_spiders, 'interval', seconds=10, id='sys_sync_spiders') 113 | scheduler.add_job(reload_runnable_spider_job_execution, 'interval', seconds=30, id='sys_reload_job') 114 | 115 | 116 | def start_scheduler(): 117 | scheduler.start() 118 | 119 | 120 | def init_basic_auth(): 121 | if not app.config.get('NO_AUTH'): 122 | basic_auth = BasicAuth(app) 123 | 124 | 125 | def initialize(): 126 | init_database() 127 | regist_server() 128 | start_scheduler() 129 | init_basic_auth() 130 | -------------------------------------------------------------------------------- /SpiderKeeper/app/proxy/contrib/scrapy.py: -------------------------------------------------------------------------------- 1 | import datetime, time 2 | 3 | import requests 4 | 5 | from SpiderKeeper.app.proxy.spiderctrl import SpiderServiceProxy 6 | from SpiderKeeper.app.spider.model import SpiderStatus, Project, SpiderInstance 7 | from SpiderKeeper.app.util.http import request 8 | 9 | 10 | class ScrapydProxy(SpiderServiceProxy): 11 | def __init__(self, server): 12 | self.spider_status_name_dict = { 13 | SpiderStatus.PENDING: 'pending', 14 | SpiderStatus.RUNNING: 'running', 15 | SpiderStatus.FINISHED: 'finished' 16 | } 17 | super(ScrapydProxy, self).__init__(server) 18 | 19 | def _scrapyd_url(self): 20 | return self.server 21 | 22 | def get_project_list(self): 23 | data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json") 24 | result = [] 25 | if data: 26 | for project_name in data['projects']: 27 | project = Project() 28 | project.project_name = project_name 29 | result.append(project) 30 | return result 31 | 32 | def delete_project(self, project_name): 33 | post_data = dict(project=project_name) 34 | data = request("post", self._scrapyd_url() + "/delproject.json", data=post_data, return_type="json") 35 | return True if data and data['status'] == 'ok' else False 36 | 37 | def get_spider_list(self, project_name): 38 | data = request("get", self._scrapyd_url() + "/listspiders.json?project=%s" % project_name, 39 | return_type="json") 40 | result = [] 41 | if data and data['status'] == 'ok': 42 | for spider_name in data['spiders']: 43 | spider_instance = SpiderInstance() 44 | spider_instance.spider_name = spider_name 45 | result.append(spider_instance) 46 | return result 47 | 48 | def get_daemon_status(self): 49 | pass 50 | 51 | def get_job_list(self, project_name, spider_status=None): 52 | data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name, 53 | return_type="json") 54 | result = {SpiderStatus.PENDING: [], SpiderStatus.RUNNING: [], SpiderStatus.FINISHED: []} 55 | if data and data['status'] == 'ok': 56 | for _status in self.spider_status_name_dict.keys(): 57 | for item in data[self.spider_status_name_dict[_status]]: 58 | start_time, end_time = None, None 59 | if item.get('start_time'): 60 | start_time = datetime.datetime.strptime(item['start_time'], '%Y-%m-%d %H:%M:%S.%f') 61 | if item.get('end_time'): 62 | end_time = datetime.datetime.strptime(item['end_time'], '%Y-%m-%d %H:%M:%S.%f') 63 | result[_status].append(dict(id=item['id'], start_time=start_time, end_time=end_time)) 64 | return result if not spider_status else result[spider_status] 65 | 66 | def start_spider(self, project_name, spider_name, arguments): 67 | post_data = dict(project=project_name, spider=spider_name) 68 | post_data.update(arguments) 69 | data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json") 70 | return data['jobid'] if data and data['status'] == 'ok' else None 71 | 72 | def cancel_spider(self, project_name, job_id): 73 | post_data = dict(project=project_name, job=job_id) 74 | data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json") 75 | return data != None 76 | 77 | def deploy(self, project_name, file_path): 78 | with open(file_path, 'rb') as f: 79 | eggdata = f.read() 80 | res = requests.post(self._scrapyd_url() + '/addversion.json', data={ 81 | 'project': project_name, 82 | 'version': int(time.time()), 83 | 'egg': eggdata, 84 | }) 85 | return res.text if res.status_code == 200 else None 86 | 87 | def log_url(self, project_name, spider_name, job_id): 88 | return self._scrapyd_url() + '/logs/%s/%s/%s.log' % (project_name, spider_name, job_id) 89 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/respond.min.js: -------------------------------------------------------------------------------- 1 | /*! Respond.js v1.4.2: min/max-width media query polyfill * Copyright 2013 Scott Jehl 2 | * Licensed under https://github.com/scottjehl/Respond/blob/master/LICENSE-MIT 3 | * */ 4 | 5 | !function(a){"use strict";a.matchMedia=a.matchMedia||function(a){var b,c=a.documentElement,d=c.firstElementChild||c.firstChild,e=a.createElement("body"),f=a.createElement("div");return f.id="mq-test-1",f.style.cssText="position:absolute;top:-100em",e.style.background="none",e.appendChild(f),function(a){return f.innerHTML='­',c.insertBefore(e,d),b=42===f.offsetWidth,c.removeChild(e),{matches:b,media:a}}}(a.document)}(this),function(a){"use strict";function b(){u(!0)}var c={};a.respond=c,c.update=function(){};var d=[],e=function(){var b=!1;try{b=new a.XMLHttpRequest}catch(c){b=new a.ActiveXObject("Microsoft.XMLHTTP")}return function(){return b}}(),f=function(a,b){var c=e();c&&(c.open("GET",a,!0),c.onreadystatechange=function(){4!==c.readyState||200!==c.status&&304!==c.status||b(c.responseText)},4!==c.readyState&&c.send(null))};if(c.ajax=f,c.queue=d,c.regex={media:/@media[^\{]+\{([^\{\}]*\{[^\}\{]*\})+/gi,keyframes:/@(?:\-(?:o|moz|webkit)\-)?keyframes[^\{]+\{(?:[^\{\}]*\{[^\}\{]*\})+[^\}]*\}/gi,urls:/(url\()['"]?([^\/\)'"][^:\)'"]+)['"]?(\))/g,findStyles:/@media *([^\{]+)\{([\S\s]+?)$/,only:/(only\s+)?([a-zA-Z]+)\s?/,minw:/\([\s]*min\-width\s*:[\s]*([\s]*[0-9\.]+)(px|em)[\s]*\)/,maxw:/\([\s]*max\-width\s*:[\s]*([\s]*[0-9\.]+)(px|em)[\s]*\)/},c.mediaQueriesSupported=a.matchMedia&&null!==a.matchMedia("only all")&&a.matchMedia("only all").matches,!c.mediaQueriesSupported){var g,h,i,j=a.document,k=j.documentElement,l=[],m=[],n=[],o={},p=30,q=j.getElementsByTagName("head")[0]||k,r=j.getElementsByTagName("base")[0],s=q.getElementsByTagName("link"),t=function(){var a,b=j.createElement("div"),c=j.body,d=k.style.fontSize,e=c&&c.style.fontSize,f=!1;return b.style.cssText="position:absolute;font-size:1em;width:1em",c||(c=f=j.createElement("body"),c.style.background="none"),k.style.fontSize="100%",c.style.fontSize="100%",c.appendChild(b),f&&k.insertBefore(c,k.firstChild),a=b.offsetWidth,f?k.removeChild(c):c.removeChild(b),k.style.fontSize=d,e&&(c.style.fontSize=e),a=i=parseFloat(a)},u=function(b){var c="clientWidth",d=k[c],e="CSS1Compat"===j.compatMode&&d||j.body[c]||d,f={},o=s[s.length-1],r=(new Date).getTime();if(b&&g&&p>r-g)return a.clearTimeout(h),h=a.setTimeout(u,p),void 0;g=r;for(var v in l)if(l.hasOwnProperty(v)){var w=l[v],x=w.minw,y=w.maxw,z=null===x,A=null===y,B="em";x&&(x=parseFloat(x)*(x.indexOf(B)>-1?i||t():1)),y&&(y=parseFloat(y)*(y.indexOf(B)>-1?i||t():1)),w.hasquery&&(z&&A||!(z||e>=x)||!(A||y>=e))||(f[w.media]||(f[w.media]=[]),f[w.media].push(m[w.rules]))}for(var C in n)n.hasOwnProperty(C)&&n[C]&&n[C].parentNode===q&&q.removeChild(n[C]);n.length=0;for(var D in f)if(f.hasOwnProperty(D)){var E=j.createElement("style"),F=f[D].join("\n");E.type="text/css",E.media=D,q.insertBefore(E,o.nextSibling),E.styleSheet?E.styleSheet.cssText=F:E.appendChild(j.createTextNode(F)),n.push(E)}},v=function(a,b,d){var e=a.replace(c.regex.keyframes,"").match(c.regex.media),f=e&&e.length||0;b=b.substring(0,b.lastIndexOf("/"));var g=function(a){return a.replace(c.regex.urls,"$1"+b+"$2$3")},h=!f&&d;b.length&&(b+="/"),h&&(f=1);for(var i=0;f>i;i++){var j,k,n,o;h?(j=d,m.push(g(a))):(j=e[i].match(c.regex.findStyles)&&RegExp.$1,m.push(RegExp.$2&&g(RegExp.$2))),n=j.split(","),o=n.length;for(var p=0;o>p;p++)k=n[p],l.push({media:k.split("(")[0].match(c.regex.only)&&RegExp.$2||"all",rules:m.length-1,hasquery:k.indexOf("(")>-1,minw:k.match(c.regex.minw)&&parseFloat(RegExp.$1)+(RegExp.$2||""),maxw:k.match(c.regex.maxw)&&parseFloat(RegExp.$1)+(RegExp.$2||"")})}u()},w=function(){if(d.length){var b=d.shift();f(b.href,function(c){v(c,b.href,b.media),o[b.href]=!0,a.setTimeout(function(){w()},0)})}},x=function(){for(var b=0;b=b.outerHeight()?k=!0:(c.stop(!0, 11 | !0).fadeIn("fast"),a.railVisible&&m.stop(!0,!0).fadeIn("fast"))}function p(){a.alwaysVisible||(B=setTimeout(function(){a.disableFadeOut&&r||y||z||(c.fadeOut("slow"),m.fadeOut("slow"))},1E3))}var r,y,z,B,A,u,l,C,k=!1,b=e(this);if(b.parent().hasClass(a.wrapperClass)){var q=b.scrollTop(),c=b.siblings("."+a.barClass),m=b.siblings("."+a.railClass);x();if(e.isPlainObject(f)){if("height"in f&&"auto"==f.height){b.parent().css("height","auto");b.css("height","auto");var h=b.parent().parent().height();b.parent().css("height", 12 | h);b.css("height",h)}else"height"in f&&(h=f.height,b.parent().css("height",h),b.css("height",h));if("scrollTo"in f)q=parseInt(a.scrollTo);else if("scrollBy"in f)q+=parseInt(a.scrollBy);else if("destroy"in f){c.remove();m.remove();b.unwrap();return}n(q,!1,!0)}}else if(!(e.isPlainObject(f)&&"destroy"in f)){a.height="auto"==a.height?b.parent().height():a.height;q=e("
").addClass(a.wrapperClass).css({position:"relative",overflow:"hidden",width:a.width,height:a.height});b.css({overflow:"hidden", 13 | width:a.width,height:a.height});var m=e("
").addClass(a.railClass).css({width:a.size,height:"100%",position:"absolute",top:0,display:a.alwaysVisible&&a.railVisible?"block":"none","border-radius":a.railBorderRadius,background:a.railColor,opacity:a.railOpacity,zIndex:90}),c=e("
").addClass(a.barClass).css({background:a.color,width:a.size,position:"absolute",top:0,opacity:a.opacity,display:a.alwaysVisible?"block":"none","border-radius":a.borderRadius,BorderRadius:a.borderRadius,MozBorderRadius:a.borderRadius, 14 | WebkitBorderRadius:a.borderRadius,zIndex:99}),h="right"==a.position?{right:a.distance}:{left:a.distance};m.css(h);c.css(h);b.wrap(q);b.parent().append(c);b.parent().append(m);a.railDraggable&&c.bind("mousedown",function(a){var b=e(document);z=!0;t=parseFloat(c.css("top"));pageY=a.pageY;b.bind("mousemove.slimscroll",function(a){currTop=t+a.pageY-pageY;c.css("top",currTop);n(0,c.position().top,!1)});b.bind("mouseup.slimscroll",function(a){z=!1;p();b.unbind(".slimscroll")});return!1}).bind("selectstart.slimscroll", 15 | function(a){a.stopPropagation();a.preventDefault();return!1});m.hover(function(){w()},function(){p()});c.hover(function(){y=!0},function(){y=!1});b.hover(function(){r=!0;w();p()},function(){r=!1;p()});b.bind("touchstart",function(a,b){a.originalEvent.touches.length&&(A=a.originalEvent.touches[0].pageY)});b.bind("touchmove",function(b){k||b.originalEvent.preventDefault();b.originalEvent.touches.length&&(n((A-b.originalEvent.touches[0].pageY)/a.touchScrollStep,!0),A=b.originalEvent.touches[0].pageY)}); 16 | x();"bottom"===a.start?(c.css({top:b.outerHeight()-c.outerHeight()}),n(0,!0)):"top"!==a.start&&(n(e(a.start).position().top,null,!0),a.alwaysVisible||c.hide());window.addEventListener?(this.addEventListener("DOMMouseScroll",v,!1),this.addEventListener("mousewheel",v,!1)):document.attachEvent("onmousewheel",v)}});return this}});e.fn.extend({slimscroll:e.fn.slimScroll})})(jQuery); -------------------------------------------------------------------------------- /SpiderKeeper/app/proxy/spiderctrl.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import random 3 | from functools import reduce 4 | 5 | from SpiderKeeper.app import db 6 | from SpiderKeeper.app.spider.model import SpiderStatus, JobExecution, JobInstance, Project, JobPriority 7 | 8 | 9 | class SpiderServiceProxy(object): 10 | def __init__(self, server): 11 | # service machine id 12 | self._server = server 13 | 14 | def get_project_list(self): 15 | ''' 16 | 17 | :return: [] 18 | ''' 19 | pass 20 | 21 | def delete_project(self, project_name): 22 | ''' 23 | 24 | :return: 25 | ''' 26 | pass 27 | 28 | def get_spider_list(self, *args, **kwargs): 29 | ''' 30 | 31 | :param args: 32 | :param kwargs: 33 | :return: [] 34 | ''' 35 | return NotImplementedError 36 | 37 | def get_daemon_status(self): 38 | return NotImplementedError 39 | 40 | def get_job_list(self, project_name, spider_status): 41 | ''' 42 | 43 | :param project_name: 44 | :param spider_status: 45 | :return: job service execution id list 46 | ''' 47 | return NotImplementedError 48 | 49 | def start_spider(self, *args, **kwargs): 50 | ''' 51 | 52 | :param args: 53 | :param kwargs: 54 | :return: {id:foo,start_time:None,end_time:None} 55 | ''' 56 | return NotImplementedError 57 | 58 | def cancel_spider(self, *args, **kwargs): 59 | return NotImplementedError 60 | 61 | def deploy(self, *args, **kwargs): 62 | pass 63 | 64 | def log_url(self, *args, **kwargs): 65 | pass 66 | 67 | @property 68 | def server(self): 69 | return self._server 70 | 71 | 72 | class SpiderAgent(): 73 | def __init__(self): 74 | self.spider_service_instances = [] 75 | 76 | def regist(self, spider_service_proxy): 77 | if isinstance(spider_service_proxy, SpiderServiceProxy): 78 | self.spider_service_instances.append(spider_service_proxy) 79 | 80 | def get_project_list(self): 81 | project_list = self.spider_service_instances[0].get_project_list() 82 | Project.load_project(project_list) 83 | return [project.to_dict() for project in Project.query.all()] 84 | 85 | def delete_project(self, project): 86 | for spider_service_instance in self.spider_service_instances: 87 | spider_service_instance.delete_project(project.project_name) 88 | 89 | def get_spider_list(self, project): 90 | spider_instance_list = self.spider_service_instances[0].get_spider_list(project.project_name) 91 | for spider_instance in spider_instance_list: 92 | spider_instance.project_id = project.id 93 | return spider_instance_list 94 | 95 | def get_daemon_status(self): 96 | pass 97 | 98 | def sync_job_status(self, project): 99 | for spider_service_instance in self.spider_service_instances: 100 | job_status = spider_service_instance.get_job_list(project.project_name) 101 | job_execution_list = JobExecution.list_uncomplete_job() 102 | job_execution_dict = dict( 103 | [(job_execution.service_job_execution_id, job_execution) for job_execution in job_execution_list]) 104 | # running 105 | for job_execution_info in job_status[SpiderStatus.RUNNING]: 106 | job_execution = job_execution_dict.get(job_execution_info['id']) 107 | if job_execution and job_execution.running_status == SpiderStatus.PENDING: 108 | job_execution.start_time = job_execution_info['start_time'] 109 | job_execution.running_status = SpiderStatus.RUNNING 110 | 111 | # finished 112 | for job_execution_info in job_status[SpiderStatus.FINISHED]: 113 | job_execution = job_execution_dict.get(job_execution_info['id']) 114 | if job_execution and job_execution.running_status != SpiderStatus.FINISHED: 115 | job_execution.start_time = job_execution_info['start_time'] 116 | job_execution.end_time = job_execution_info['end_time'] 117 | job_execution.running_status = SpiderStatus.FINISHED 118 | # commit 119 | db.session.commit() 120 | 121 | def start_spider(self, job_instance): 122 | project = Project.find_project_by_id(job_instance.project_id) 123 | spider_name = job_instance.spider_name 124 | #arguments = {} 125 | #if job_instance.spider_arguments: 126 | # arguments = dict(map(lambda x: x.split("="), job_instance.spider_arguments.split(","))) 127 | from collections import defaultdict 128 | arguments = defaultdict(list) 129 | if job_instance.spider_arguments: 130 | for k, v in list(map(lambda x: x.split('=', 1), job_instance.spider_arguments.split(','))): 131 | arguments[k].append(v) 132 | threshold = 0 133 | daemon_size = len(self.spider_service_instances) 134 | if job_instance.priority == JobPriority.HIGH: 135 | threshold = int(daemon_size / 2) 136 | if job_instance.priority == JobPriority.HIGHEST: 137 | threshold = int(daemon_size) 138 | threshold = 1 if threshold == 0 else threshold 139 | candidates = self.spider_service_instances 140 | leaders = [] 141 | if 'daemon' in arguments: 142 | for candidate in candidates: 143 | if candidate.server == arguments['daemon']: 144 | leaders = [candidate] 145 | else: 146 | # TODO optimize some better func to vote the leader 147 | for i in range(threshold): 148 | leaders.append(random.choice(candidates)) 149 | for leader in leaders: 150 | serviec_job_id = leader.start_spider(project.project_name, spider_name, arguments) 151 | job_execution = JobExecution() 152 | job_execution.project_id = job_instance.project_id 153 | job_execution.service_job_execution_id = serviec_job_id 154 | job_execution.job_instance_id = job_instance.id 155 | job_execution.create_time = datetime.datetime.now() 156 | job_execution.running_on = leader.server 157 | db.session.add(job_execution) 158 | db.session.commit() 159 | 160 | def cancel_spider(self, job_execution): 161 | job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id) 162 | project = Project.find_project_by_id(job_instance.project_id) 163 | for spider_service_instance in self.spider_service_instances: 164 | if spider_service_instance.server == job_execution.running_on: 165 | if spider_service_instance.cancel_spider(project.project_name, job_execution.service_job_execution_id): 166 | job_execution.end_time = datetime.datetime.now() 167 | job_execution.running_status = SpiderStatus.CANCELED 168 | db.session.commit() 169 | break 170 | 171 | def deploy(self, project, file_path): 172 | for spider_service_instance in self.spider_service_instances: 173 | if not spider_service_instance.deploy(project.project_name, file_path): 174 | return False 175 | return True 176 | 177 | def log_url(self, job_execution): 178 | job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id) 179 | project = Project.find_project_by_id(job_instance.project_id) 180 | for spider_service_instance in self.spider_service_instances: 181 | if spider_service_instance.server == job_execution.running_on: 182 | return spider_service_instance.log_url(project.project_name, job_instance.spider_name, 183 | job_execution.service_job_execution_id) 184 | 185 | @property 186 | def servers(self): 187 | return [self.spider_service_instance.server for self.spider_service_instance in 188 | self.spider_service_instances] 189 | 190 | 191 | if __name__ == '__main__': 192 | pass 193 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/fastclick.min.js: -------------------------------------------------------------------------------- 1 | !function e(t,n,r){function i(s,a){if(!n[s]){if(!t[s]){var c="function"==typeof require&&require;if(!a&&c)return c(s,!0);if(o)return o(s,!0);var l=new Error("Cannot find module '"+s+"'");throw l.code="MODULE_NOT_FOUND",l}var u=n[s]={exports:{}};t[s][0].call(u.exports,function(e){var n=t[s][1][e];return i(n?n:e)},u,u.exports,e,t,n,r)}return n[s].exports}for(var o="function"==typeof require&&require,s=0;sc;c++)a[s[c]]=i(a[s[c]],a);r&&(t.addEventListener("mouseover",this.onMouse,!0),t.addEventListener("mousedown",this.onMouse,!0),t.addEventListener("mouseup",this.onMouse,!0)),t.addEventListener("click",this.onClick,!0),t.addEventListener("touchstart",this.onTouchStart,!1),t.addEventListener("touchmove",this.onTouchMove,!1),t.addEventListener("touchend",this.onTouchEnd,!1),t.addEventListener("touchcancel",this.onTouchCancel,!1),Event.prototype.stopImmediatePropagation||(t.removeEventListener=function(e,n,r){var i=Node.prototype.removeEventListener;"click"===e?i.call(t,e,n.hijacked||n,r):i.call(t,e,n,r)},t.addEventListener=function(e,n,r){var i=Node.prototype.addEventListener;"click"===e?i.call(t,e,n.hijacked||(n.hijacked=function(e){e.propagationStopped||n(e)}),r):i.call(t,e,n,r)}),"function"==typeof t.onclick&&(o=t.onclick,t.addEventListener("click",function(e){o(e)},!1),t.onclick=null)}}var n=navigator.userAgent.indexOf("Windows Phone")>=0,r=navigator.userAgent.indexOf("Android")>0&&!n,i=/iP(ad|hone|od)/.test(navigator.userAgent)&&!n,o=i&&/OS 4_\d(_\d)?/.test(navigator.userAgent),s=i&&/OS [6-7]_\d/.test(navigator.userAgent),a=navigator.userAgent.indexOf("BB10")>0;e.prototype.needsClick=function(e){switch(e.nodeName.toLowerCase()){case"button":case"select":case"textarea":if(e.disabled)return!0;break;case"input":if(i&&"file"===e.type||e.disabled)return!0;break;case"label":case"iframe":case"video":return!0}return/\bneedsclick\b/.test(e.className)},e.prototype.needsFocus=function(e){switch(e.nodeName.toLowerCase()){case"textarea":return!0;case"select":return!r;case"input":switch(e.type){case"button":case"checkbox":case"file":case"image":case"radio":case"submit":return!1}return!e.disabled&&!e.readOnly;default:return/\bneedsfocus\b/.test(e.className)}},e.prototype.sendClick=function(e,t){var n,r;document.activeElement&&document.activeElement!==e&&document.activeElement.blur(),r=t.changedTouches[0],n=document.createEvent("MouseEvents"),n.initMouseEvent(this.determineEventType(e),!0,!0,window,1,r.screenX,r.screenY,r.clientX,r.clientY,!1,!1,!1,!1,0,null),n.forwardedTouchEvent=!0,e.dispatchEvent(n)},e.prototype.determineEventType=function(e){return r&&"select"===e.tagName.toLowerCase()?"mousedown":"click"},e.prototype.focus=function(e){var t;i&&e.setSelectionRange&&0!==e.type.indexOf("date")&&"time"!==e.type&&"month"!==e.type?(t=e.value.length,e.setSelectionRange(t,t)):e.focus()},e.prototype.updateScrollParent=function(e){var t,n;if(t=e.fastClickScrollParent,!t||!t.contains(e)){n=e;do{if(n.scrollHeight>n.offsetHeight){t=n,e.fastClickScrollParent=n;break}n=n.parentElement}while(n)}t&&(t.fastClickLastScrollTop=t.scrollTop)},e.prototype.getTargetElementFromEventTarget=function(e){return e.nodeType===Node.TEXT_NODE?e.parentNode:e},e.prototype.onTouchStart=function(e){var t,n,r;if(e.targetTouches.length>1)return!0;if(t=this.getTargetElementFromEventTarget(e.target),n=e.targetTouches[0],i){if(r=window.getSelection(),r.rangeCount&&!r.isCollapsed)return!0;if(!o){if(n.identifier&&n.identifier===this.lastTouchIdentifier)return e.preventDefault(),!1;this.lastTouchIdentifier=n.identifier,this.updateScrollParent(t)}}return this.trackingClick=!0,this.trackingClickStart=e.timeStamp,this.targetElement=t,this.touchStartX=n.pageX,this.touchStartY=n.pageY,e.timeStamp-this.lastClickTimen||Math.abs(t.pageY-this.touchStartY)>n?!0:!1},e.prototype.onTouchMove=function(e){return this.trackingClick?((this.targetElement!==this.getTargetElementFromEventTarget(e.target)||this.touchHasMoved(e))&&(this.trackingClick=!1,this.targetElement=null),!0):!0},e.prototype.findControl=function(e){return void 0!==e.control?e.control:e.htmlFor?document.getElementById(e.htmlFor):e.querySelector("button, input:not([type=hidden]), keygen, meter, output, progress, select, textarea")},e.prototype.onTouchEnd=function(e){var t,n,a,c,l,u=this.targetElement;if(!this.trackingClick)return!0;if(e.timeStamp-this.lastClickTimethis.tapTimeout)return!0;if(this.cancelNextClick=!1,this.lastClickTime=e.timeStamp,n=this.trackingClickStart,this.trackingClick=!1,this.trackingClickStart=0,s&&(l=e.changedTouches[0],u=document.elementFromPoint(l.pageX-window.pageXOffset,l.pageY-window.pageYOffset)||u,u.fastClickScrollParent=this.targetElement.fastClickScrollParent),a=u.tagName.toLowerCase(),"label"===a){if(t=this.findControl(u)){if(this.focus(u),r)return!1;u=t}}else if(this.needsFocus(u))return e.timeStamp-n>100||i&&window.top!==window&&"input"===a?(this.targetElement=null,!1):(this.focus(u),this.sendClick(u,e),i&&"select"===a||(this.targetElement=null,e.preventDefault()),!1);return i&&!o&&(c=u.fastClickScrollParent,c&&c.fastClickLastScrollTop!==c.scrollTop)?!0:(this.needsClick(u)||(e.preventDefault(),this.sendClick(u,e)),!1)},e.prototype.onTouchCancel=function(){this.trackingClick=!1,this.targetElement=null},e.prototype.onMouse=function(e){return this.targetElement?e.forwardedTouchEvent?!0:e.cancelable&&(!this.needsClick(this.targetElement)||this.cancelNextClick)?(e.stopImmediatePropagation?e.stopImmediatePropagation():e.propagationStopped=!0,e.stopPropagation(),e.preventDefault(),!1):!0:!0},e.prototype.onClick=function(e){var t;return this.trackingClick?(this.targetElement=null,this.trackingClick=!1,!0):"submit"===e.target.type&&0===e.detail?!0:(t=this.onMouse(e),t||(this.targetElement=null),t)},e.prototype.destroy=function(){var e=this.layer;r&&(e.removeEventListener("mouseover",this.onMouse,!0),e.removeEventListener("mousedown",this.onMouse,!0),e.removeEventListener("mouseup",this.onMouse,!0)),e.removeEventListener("click",this.onClick,!0),e.removeEventListener("touchstart",this.onTouchStart,!1),e.removeEventListener("touchmove",this.onTouchMove,!1),e.removeEventListener("touchend",this.onTouchEnd,!1),e.removeEventListener("touchcancel",this.onTouchCancel,!1)},e.notNeeded=function(e){var t,n,i,o;if("undefined"==typeof window.ontouchstart)return!0;if(n=+(/Chrome\/([0-9]+)/.exec(navigator.userAgent)||[,0])[1]){if(!r)return!0;if(t=document.querySelector("meta[name=viewport]")){if(-1!==t.content.indexOf("user-scalable=no"))return!0;if(n>31&&document.documentElement.scrollWidth<=window.outerWidth)return!0}}if(a&&(i=navigator.userAgent.match(/Version\/([0-9]*)\.([0-9]*)/),i[1]>=10&&i[2]>=3&&(t=document.querySelector("meta[name=viewport]")))){if(-1!==t.content.indexOf("user-scalable=no"))return!0;if(document.documentElement.scrollWidth<=window.outerWidth)return!0}return"none"===e.style.msTouchAction||"manipulation"===e.style.touchAction?!0:(o=+(/Firefox\/([0-9]+)/.exec(navigator.userAgent)||[,0])[1],o>=27&&(t=document.querySelector("meta[name=viewport]"),t&&(-1!==t.content.indexOf("user-scalable=no")||document.documentElement.scrollWidth<=window.outerWidth))?!0:"none"===e.style.touchAction||"manipulation"===e.style.touchAction?!0:!1)},e.attach=function(t,n){return new e(t,n)},"function"==typeof define&&"object"==typeof define.amd&&define.amd?define(function(){return e}):"undefined"!=typeof t&&t.exports?(t.exports=e.attach,t.exports.FastClick=e):window.FastClick=e}()},{}],2:[function(e){window.Origami={fastclick:e("./bower_components/fastclick/lib/fastclick.js")}},{"./bower_components/fastclick/lib/fastclick.js":1}]},{},[2]);;(function() {function trigger(){document.dispatchEvent(new CustomEvent('o.load'))};document.addEventListener('load',trigger);if (document.readyState==='ready') trigger();}());(function() {function trigger(){document.dispatchEvent(new CustomEvent('o.DOMContentLoaded'))};document.addEventListener('DOMContentLoaded',trigger);if (document.readyState==='interactive') trigger();}()) -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | SpiderKeeper 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 29 | 30 | 31 | 32 |
33 |
34 | 35 | 36 | 41 | 42 | 71 |
72 | 73 | 74 | 75 | 76 | 111 | 112 | 113 | 114 | 115 |
116 | 117 |
118 | {% block content_header %}{% endblock %} 119 |
120 | 121 | 122 |
123 | {% block content_body %}{% endblock %} 124 |
125 | 126 |
127 | 128 | 129 | 135 | 136 | 162 | 163 |
164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | {% block script %}{% endblock %} 179 | 180 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/AdminLTE.min.js: -------------------------------------------------------------------------------- 1 | /*! AdminLTE app.js 2 | * ================ 3 | * Main JS application file for AdminLTE v2. This file 4 | * should be included in all pages. It controls some layout 5 | * options and implements exclusive AdminLTE plugins. 6 | * 7 | * @Author Almsaeed Studio 8 | * @Support 9 | * @Email 10 | * @version 2.3.12 11 | * @license MIT 12 | */ 13 | function _init(){"use strict";$.AdminLTE.layout={activate:function(){var a=this;a.fix(),a.fixSidebar(),$("body, html, .wrapper").css("height","auto"),$(window,".wrapper").resize(function(){a.fix(),a.fixSidebar()})},fix:function(){$(".layout-boxed > .wrapper").css("overflow","hidden");var a=$(".main-footer").outerHeight()||0,b=$(".main-header").outerHeight()+a,c=$(window).height(),d=$(".sidebar").height()||0;if($("body").hasClass("fixed"))$(".content-wrapper, .right-side").css("min-height",c-a);else{var e;c>=d?($(".content-wrapper, .right-side").css("min-height",c-b),e=c-b):($(".content-wrapper, .right-side").css("min-height",d),e=d);var f=$($.AdminLTE.options.controlSidebarOptions.selector);"undefined"!=typeof f&&f.height()>e&&$(".content-wrapper, .right-side").css("min-height",f.height())}},fixSidebar:function(){return $("body").hasClass("fixed")?("undefined"==typeof $.fn.slimScroll&&window.console&&window.console.error("Error: the fixed layout requires the slimscroll plugin!"),void($.AdminLTE.options.sidebarSlimScroll&&"undefined"!=typeof $.fn.slimScroll&&($(".sidebar").slimScroll({destroy:!0}).height("auto"),$(".sidebar").slimScroll({height:$(window).height()-$(".main-header").height()+"px",color:"rgba(0,0,0,0.2)",size:"3px"})))):void("undefined"!=typeof $.fn.slimScroll&&$(".sidebar").slimScroll({destroy:!0}).height("auto"))}},$.AdminLTE.pushMenu={activate:function(a){var b=$.AdminLTE.options.screenSizes;$(document).on("click",a,function(a){a.preventDefault(),$(window).width()>b.sm-1?$("body").hasClass("sidebar-collapse")?$("body").removeClass("sidebar-collapse").trigger("expanded.pushMenu"):$("body").addClass("sidebar-collapse").trigger("collapsed.pushMenu"):$("body").hasClass("sidebar-open")?$("body").removeClass("sidebar-open").removeClass("sidebar-collapse").trigger("collapsed.pushMenu"):$("body").addClass("sidebar-open").trigger("expanded.pushMenu")}),$(".content-wrapper").click(function(){$(window).width()<=b.sm-1&&$("body").hasClass("sidebar-open")&&$("body").removeClass("sidebar-open")}),($.AdminLTE.options.sidebarExpandOnHover||$("body").hasClass("fixed")&&$("body").hasClass("sidebar-mini"))&&this.expandOnHover()},expandOnHover:function(){var a=this,b=$.AdminLTE.options.screenSizes.sm-1;$(".main-sidebar").hover(function(){$("body").hasClass("sidebar-mini")&&$("body").hasClass("sidebar-collapse")&&$(window).width()>b&&a.expand()},function(){$("body").hasClass("sidebar-mini")&&$("body").hasClass("sidebar-expanded-on-hover")&&$(window).width()>b&&a.collapse()})},expand:function(){$("body").removeClass("sidebar-collapse").addClass("sidebar-expanded-on-hover")},collapse:function(){$("body").hasClass("sidebar-expanded-on-hover")&&$("body").removeClass("sidebar-expanded-on-hover").addClass("sidebar-collapse")}},$.AdminLTE.tree=function(a){var b=this,c=$.AdminLTE.options.animationSpeed;$(document).off("click",a+" li a").on("click",a+" li a",function(a){var d=$(this),e=d.next();if(e.is(".treeview-menu")&&e.is(":visible")&&!$("body").hasClass("sidebar-collapse"))e.slideUp(c,function(){e.removeClass("menu-open")}),e.parent("li").removeClass("active");else if(e.is(".treeview-menu")&&!e.is(":visible")){var f=d.parents("ul").first(),g=f.find("ul:visible").slideUp(c);g.removeClass("menu-open");var h=d.parent("li");e.slideDown(c,function(){e.addClass("menu-open"),f.find("li.active").removeClass("active"),h.addClass("active"),b.layout.fix()})}e.is(".treeview-menu")&&a.preventDefault()})},$.AdminLTE.controlSidebar={activate:function(){var a=this,b=$.AdminLTE.options.controlSidebarOptions,c=$(b.selector),d=$(b.toggleBtnSelector);d.on("click",function(d){d.preventDefault(),c.hasClass("control-sidebar-open")||$("body").hasClass("control-sidebar-open")?a.close(c,b.slide):a.open(c,b.slide)});var e=$(".control-sidebar-bg");a._fix(e),$("body").hasClass("fixed")?a._fixForFixed(c):$(".content-wrapper, .right-side").height() .box-body, > .box-footer, > form >.box-body, > form > .box-footer");c.hasClass("collapsed-box")?(a.children(":first").removeClass(b.icons.open).addClass(b.icons.collapse),d.slideDown(b.animationSpeed,function(){c.removeClass("collapsed-box")})):(a.children(":first").removeClass(b.icons.collapse).addClass(b.icons.open),d.slideUp(b.animationSpeed,function(){c.addClass("collapsed-box")}))},remove:function(a){var b=a.parents(".box").first();b.slideUp(this.animationSpeed)}}}if("undefined"==typeof jQuery)throw new Error("AdminLTE requires jQuery");$.AdminLTE={},$.AdminLTE.options={navbarMenuSlimscroll:!0,navbarMenuSlimscrollWidth:"3px",navbarMenuHeight:"200px",animationSpeed:500,sidebarToggleSelector:"[data-toggle='offcanvas']",sidebarPushMenu:!0,sidebarSlimScroll:!0,sidebarExpandOnHover:!1,enableBoxRefresh:!0,enableBSToppltip:!0,BSTooltipSelector:"[data-toggle='tooltip']",enableFastclick:!1,enableControlTreeView:!0,enableControlSidebar:!0,controlSidebarOptions:{toggleBtnSelector:"[data-toggle='control-sidebar']",selector:".control-sidebar",slide:!0},enableBoxWidget:!0,boxWidgetOptions:{boxWidgetIcons:{collapse:"fa-minus",open:"fa-plus",remove:"fa-times"},boxWidgetSelectors:{remove:'[data-widget="remove"]',collapse:'[data-widget="collapse"]'}},directChat:{enable:!0,contactToggleSelector:'[data-widget="chat-pane-toggle"]'},colors:{lightBlue:"#3c8dbc",red:"#f56954",green:"#00a65a",aqua:"#00c0ef",yellow:"#f39c12",blue:"#0073b7",navy:"#001F3F",teal:"#39CCCC",olive:"#3D9970",lime:"#01FF70",orange:"#FF851B",fuchsia:"#F012BE",purple:"#8E24AA",maroon:"#D81B60",black:"#222222",gray:"#d2d6de"},screenSizes:{xs:480,sm:768,md:992,lg:1200}},$(function(){"use strict";$("body").removeClass("hold-transition"),"undefined"!=typeof AdminLTEOptions&&$.extend(!0,$.AdminLTE.options,AdminLTEOptions);var a=$.AdminLTE.options;_init(),$.AdminLTE.layout.activate(),a.enableControlTreeView&&$.AdminLTE.tree(".sidebar"),a.enableControlSidebar&&$.AdminLTE.controlSidebar.activate(),a.navbarMenuSlimscroll&&"undefined"!=typeof $.fn.slimscroll&&$(".navbar .menu").slimscroll({height:a.navbarMenuHeight,alwaysVisible:!1,size:a.navbarMenuSlimscrollWidth}).css("width","100%"),a.sidebarPushMenu&&$.AdminLTE.pushMenu.activate(a.sidebarToggleSelector),a.enableBSToppltip&&$("body").tooltip({selector:a.BSTooltipSelector,container:"body"}),a.enableBoxWidget&&$.AdminLTE.boxWidget.activate(),a.enableFastclick&&"undefined"!=typeof FastClick&&FastClick.attach(document.body),a.directChat.enable&&$(document).on("click",a.directChat.contactToggleSelector,function(){var a=$(this).parents(".direct-chat").first();a.toggleClass("direct-chat-contacts-open")}),$('.btn-group[data-toggle="btn-toggle"]').each(function(){var a=$(this);$(this).find(".btn").on("click",function(b){a.find(".btn.active").removeClass("active"),$(this).addClass("active"),b.preventDefault()})})}),function(a){"use strict";a.fn.boxRefresh=function(b){function c(a){a.append(f),e.onLoadStart.call(a)}function d(a){a.find(f).remove(),e.onLoadDone.call(a)}var e=a.extend({trigger:".refresh-btn",source:"",onLoadStart:function(a){return a},onLoadDone:function(a){return a}},b),f=a('
');return this.each(function(){if(""===e.source)return void(window.console&&window.console.log("Please specify a source first - boxRefresh()"));var b=a(this),f=b.find(e.trigger).first();f.on("click",function(a){a.preventDefault(),c(b),b.find(".box-body").load(e.source,function(){d(b)})})})}}(jQuery),function(a){"use strict";a.fn.activateBox=function(){a.AdminLTE.boxWidget.activate(this)},a.fn.toggleBox=function(){var b=a(a.AdminLTE.boxWidget.selectors.collapse,this);a.AdminLTE.boxWidget.collapse(b)},a.fn.removeBox=function(){var b=a(a.AdminLTE.boxWidget.selectors.remove,this);a.AdminLTE.boxWidget.remove(b)}}(jQuery),function(a){"use strict";a.fn.todolist=function(b){var c=a.extend({onCheck:function(a){return a},onUncheck:function(a){return a}},b);return this.each(function(){"undefined"!=typeof a.fn.iCheck?(a("input",this).on("ifChecked",function(){var b=a(this).parents("li").first();b.toggleClass("done"),c.onCheck.call(b)}),a("input",this).on("ifUnchecked",function(){var b=a(this).parents("li").first();b.toggleClass("done"),c.onUncheck.call(b)})):a("input",this).on("change",function(){var b=a(this).parents("li").first();b.toggleClass("done"),a("input",b).is(":checked")?c.onCheck.call(b):c.onUncheck.call(b)})})}}(jQuery); -------------------------------------------------------------------------------- /SpiderKeeper/app/spider/model.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy import desc 3 | from SpiderKeeper.app import db, Base 4 | 5 | 6 | class Project(Base): 7 | __tablename__ = 'sk_project' 8 | 9 | project_name = db.Column(db.String(50)) 10 | 11 | @classmethod 12 | def load_project(cls, project_list): 13 | for project in project_list: 14 | existed_project = cls.query.filter_by(project_name=project.project_name).first() 15 | if not existed_project: 16 | db.session.add(project) 17 | db.session.commit() 18 | 19 | @classmethod 20 | def find_project_by_id(cls, project_id): 21 | return Project.query.filter_by(id=project_id).first() 22 | 23 | def to_dict(self): 24 | return { 25 | "project_id": self.id, 26 | "project_name": self.project_name 27 | } 28 | 29 | 30 | class SpiderInstance(Base): 31 | __tablename__ = 'sk_spider' 32 | 33 | spider_name = db.Column(db.String(100)) 34 | project_id = db.Column(db.INTEGER, nullable=False, index=True) 35 | 36 | @classmethod 37 | def update_spider_instances(cls, project_id, spider_instance_list): 38 | for spider_instance in spider_instance_list: 39 | existed_spider_instance = cls.query.filter_by(project_id=project_id, 40 | spider_name=spider_instance.spider_name).first() 41 | if not existed_spider_instance: 42 | db.session.add(spider_instance) 43 | db.session.commit() 44 | 45 | for spider in cls.query.filter_by(project_id=project_id).all(): 46 | existed_spider = any( 47 | spider.spider_name == s.spider_name 48 | for s in spider_instance_list 49 | ) 50 | if not existed_spider: 51 | db.session.delete(spider) 52 | db.session.commit() 53 | 54 | @classmethod 55 | def list_spider_by_project_id(cls, project_id): 56 | return cls.query.filter_by(project_id=project_id).all() 57 | 58 | def to_dict(self): 59 | return dict(spider_instance_id=self.id, 60 | spider_name=self.spider_name, 61 | project_id=self.project_id) 62 | 63 | @classmethod 64 | def list_spiders(cls, project_id): 65 | sql_last_runtime = ''' 66 | select * from (select a.spider_name,b.date_created from sk_job_instance as a 67 | left join sk_job_execution as b 68 | on a.id = b.job_instance_id 69 | order by b.date_created desc) as c 70 | group by c.spider_name 71 | ''' 72 | sql_avg_runtime = ''' 73 | select a.spider_name,avg(end_time-start_time) from sk_job_instance as a 74 | left join sk_job_execution as b 75 | on a.id = b.job_instance_id 76 | where b.end_time is not null 77 | group by a.spider_name 78 | ''' 79 | last_runtime_list = dict( 80 | (spider_name, last_run_time) for spider_name, last_run_time in db.engine.execute(sql_last_runtime)) 81 | avg_runtime_list = dict( 82 | (spider_name, avg_run_time) for spider_name, avg_run_time in db.engine.execute(sql_avg_runtime)) 83 | res = [] 84 | for spider in cls.query.filter_by(project_id=project_id).all(): 85 | last_runtime = last_runtime_list.get(spider.spider_name) 86 | res.append(dict(spider.to_dict(), 87 | **{'spider_last_runtime': last_runtime if last_runtime else '-', 88 | 'spider_avg_runtime': avg_runtime_list.get(spider.spider_name) 89 | })) 90 | return res 91 | 92 | 93 | class JobPriority(): 94 | LOW, NORMAL, HIGH, HIGHEST = range(-1, 3) 95 | 96 | 97 | class JobRunType(): 98 | ONETIME = 'onetime' 99 | PERIODIC = 'periodic' 100 | 101 | 102 | class JobInstance(Base): 103 | __tablename__ = 'sk_job_instance' 104 | 105 | spider_name = db.Column(db.String(100), nullable=False, index=True) 106 | project_id = db.Column(db.INTEGER, nullable=False, index=True) 107 | tags = db.Column(db.Text) # job tag(split by , ) 108 | spider_arguments = db.Column(db.Text) # job execute arguments(split by , ex.: arg1=foo,arg2=bar) 109 | priority = db.Column(db.INTEGER) 110 | desc = db.Column(db.Text) 111 | cron_minutes = db.Column(db.String(20), default="0") 112 | cron_hour = db.Column(db.String(20), default="*") 113 | cron_day_of_month = db.Column(db.String(20), default="*") 114 | cron_day_of_week = db.Column(db.String(20), default="*") 115 | cron_month = db.Column(db.String(20), default="*") 116 | enabled = db.Column(db.INTEGER, default=0) # 0/-1 117 | run_type = db.Column(db.String(20)) # periodic/onetime 118 | 119 | def to_dict(self): 120 | return dict( 121 | job_instance_id=self.id, 122 | spider_name=self.spider_name, 123 | tags=self.tags.split(',') if self.tags else None, 124 | spider_arguments=self.spider_arguments, 125 | priority=self.priority, 126 | desc=self.desc, 127 | cron_minutes=self.cron_minutes, 128 | cron_hour=self.cron_hour, 129 | cron_day_of_month=self.cron_day_of_month, 130 | cron_day_of_week=self.cron_day_of_week, 131 | cron_month=self.cron_month, 132 | enabled=self.enabled == 0, 133 | run_type=self.run_type 134 | 135 | ) 136 | 137 | @classmethod 138 | def list_job_instance_by_project_id(cls, project_id): 139 | return cls.query.filter_by(project_id=project_id).all() 140 | 141 | @classmethod 142 | def find_job_instance_by_id(cls, job_instance_id): 143 | return cls.query.filter_by(id=job_instance_id).first() 144 | 145 | 146 | class SpiderStatus(): 147 | PENDING, RUNNING, FINISHED, CANCELED = range(4) 148 | 149 | 150 | class JobExecution(Base): 151 | __tablename__ = 'sk_job_execution' 152 | 153 | project_id = db.Column(db.INTEGER, nullable=False, index=True) 154 | service_job_execution_id = db.Column(db.String(50), nullable=False, index=True) 155 | job_instance_id = db.Column(db.INTEGER, nullable=False, index=True) 156 | create_time = db.Column(db.DATETIME) 157 | start_time = db.Column(db.DATETIME) 158 | end_time = db.Column(db.DATETIME) 159 | running_status = db.Column(db.INTEGER, default=SpiderStatus.PENDING) 160 | running_on = db.Column(db.Text) 161 | 162 | def to_dict(self): 163 | job_instance = JobInstance.query.filter_by(id=self.job_instance_id).first() 164 | return { 165 | 'project_id': self.project_id, 166 | 'job_execution_id': self.id, 167 | 'job_instance_id': self.job_instance_id, 168 | 'service_job_execution_id': self.service_job_execution_id, 169 | 'create_time': self.create_time.strftime('%Y-%m-%d %H:%M:%S') if self.create_time else None, 170 | 'start_time': self.start_time.strftime('%Y-%m-%d %H:%M:%S') if self.start_time else None, 171 | 'end_time': self.end_time.strftime('%Y-%m-%d %H:%M:%S') if self.end_time else None, 172 | 'running_status': self.running_status, 173 | 'running_on': self.running_on, 174 | 'job_instance': job_instance.to_dict() if job_instance else {} 175 | } 176 | 177 | @classmethod 178 | def find_job_by_service_id(cls, service_job_execution_id): 179 | return cls.query.filter_by(service_job_execution_id=service_job_execution_id).first() 180 | 181 | @classmethod 182 | def list_job_by_service_ids(cls, service_job_execution_ids): 183 | return cls.query.filter(cls.service_job_execution_id.in_(service_job_execution_ids)).all() 184 | 185 | @classmethod 186 | def list_uncomplete_job(cls): 187 | return cls.query.filter(cls.running_status != SpiderStatus.FINISHED, 188 | cls.running_status != SpiderStatus.CANCELED).all() 189 | 190 | @classmethod 191 | def list_jobs(cls, project_id, each_status_limit=100): 192 | result = {} 193 | result['PENDING'] = [job_execution.to_dict() for job_execution in 194 | JobExecution.query.filter_by(project_id=project_id, 195 | running_status=SpiderStatus.PENDING).order_by( 196 | desc(JobExecution.date_modified)).limit(each_status_limit)] 197 | result['RUNNING'] = [job_execution.to_dict() for job_execution in 198 | JobExecution.query.filter_by(project_id=project_id, 199 | running_status=SpiderStatus.RUNNING).order_by( 200 | desc(JobExecution.date_modified)).limit(each_status_limit)] 201 | result['COMPLETED'] = [job_execution.to_dict() for job_execution in 202 | JobExecution.query.filter(JobExecution.project_id == project_id).filter( 203 | (JobExecution.running_status == SpiderStatus.FINISHED) | ( 204 | JobExecution.running_status == SpiderStatus.CANCELED)).order_by( 205 | desc(JobExecution.date_modified)).limit(each_status_limit)] 206 | return result 207 | 208 | @classmethod 209 | def list_run_stats_by_hours(cls, project_id): 210 | result = {} 211 | hour_keys = [] 212 | last_time = datetime.datetime.now() - datetime.timedelta(hours=23) 213 | last_time = datetime.datetime(last_time.year, last_time.month, last_time.day, last_time.hour) 214 | for hour in range(23, -1, -1): 215 | time_tmp = datetime.datetime.now() - datetime.timedelta(hours=hour) 216 | hour_key = time_tmp.strftime('%Y-%m-%d %H:00:00') 217 | hour_keys.append(hour_key) 218 | result[hour_key] = 0 # init 219 | for job_execution in JobExecution.query.filter(JobExecution.project_id == project_id, 220 | JobExecution.date_created >= last_time).all(): 221 | hour_key = job_execution.create_time.strftime('%Y-%m-%d %H:00:00') 222 | result[hour_key] += 1 223 | return [dict(key=hour_key, value=result[hour_key]) for hour_key in hour_keys] 224 | -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/job_periodic.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Periodic jobs

4 |
    12 | 15 |
16 | {% endblock %} 17 | {% block content_body %} 18 |
19 |
20 |

Periodic jobs (Spiders)

21 |
22 |
23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | {% for job_instance in job_instance_list %} 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | {% if job_instance.priority == -1 %} 48 | 51 | {% elif job_instance.priority == 0 %} 52 | 55 | {% elif job_instance.priority == 1 %} 56 | 59 | {% elif job_instance.priority == 2 %} 60 | 63 | {% endif %} 64 | 67 | 68 | {% if job_instance.enabled %} 69 | 73 | {% else %} 74 | 78 | {% endif %} 79 | 85 | 86 | {% endfor %} 87 |
#MonthDay of MonthDay of WeekHourMinutesSpiderPriorityArgsTagsEnabledAction
{{ job_instance.job_instance_id }}{{ job_instance.cron_month }}{{ job_instance.cron_day_of_month }}{{ job_instance.cron_day_of_week }}{{ job_instance.cron_hour }}{{ job_instance.cron_minutes }}{{ job_instance.spider_name }} 49 | LOW 50 | 53 | NORMAL 54 | 57 | HIGH 58 | 61 | HIGHEST 62 | {{ job_instance.spider_arguments }} 66 | {{ job_instance.tags }} 70 | Enabled 72 | 75 | Disabled 77 | 80 | Run 82 | Remove 84 |
88 |
89 |
90 | 91 | 220 | 221 | {% endblock %} -------------------------------------------------------------------------------- /SpiderKeeper/app/templates/job_dashboard.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content_header %} 3 |

Job Dashboard

4 |
    12 | 15 |
16 | {% endblock %} 17 | {% block content_body %} 18 | 19 |
20 |
21 |

Next Jobs

22 |
23 | 25 |
26 |
27 |
28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | {% for job in job_status.PENDING %} 38 | {% if job.job_instance %} 39 | 40 | 41 | 42 | 43 | 46 | {% if job.job_instance.priority == -1 %} 47 | 50 | {% elif job.job_instance.priority == 0 %} 51 | 54 | {% elif job.job_instance.priority == 1 %} 55 | 58 | {% elif job.job_instance.priority == 2 %} 59 | 62 | {% endif %} 63 | 64 | 65 | {% endif %} 66 | {% endfor %} 67 |
#JobSpiderArgsPriorityWait
{{ job.job_execution_id }}{{ job.job_instance_id }}{{ job.job_instance.spider_name }}{{ job.job_instance.spider_arguments }} 45 | 48 | LOW 49 | 52 | NORMAL 53 | 56 | HIGH 57 | 60 | HIGHEST 61 | {{ timedelta(now,job.create_time) }}
68 |
69 |
70 | 71 | 72 |
73 |
74 |

Running Jobs

75 |
76 | 78 |
79 |
80 |
81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | {% for job in job_status.RUNNING %} 95 | {% if job.job_instance %} 96 | 97 | 98 | 99 | 100 | 103 | {% if job.job_instance.priority == -1 %} 104 | 107 | {% elif job.job_instance.priority == 0 %} 108 | 111 | {% elif job.job_instance.priority == 1 %} 112 | 115 | {% elif job.job_instance.priority == 2 %} 116 | 119 | {% endif %} 120 | 121 | 122 | 125 | 126 | 130 | 131 | {% endif %} 132 | {% endfor %} 133 |
#JobSpiderArgsPriorityRuntimeStartedLogRunning OnAction
{{ job.job_execution_id }}{{ job.job_instance_id }}{{ job.job_instance.spider_name }}{{ job.job_instance.spider_arguments }} 102 | 105 | LOW 106 | 109 | NORMAL 110 | 113 | HIGH 114 | 117 | HIGHEST 118 | {{ timedelta(now,job.start_time) }}{{ job.start_time }}Log 124 | {{ job.running_on }} 127 | Stop 129 |
134 |
135 |
136 | 137 | 138 |
139 |
140 |

Completed Jobs

141 |
142 | 144 |
145 |
146 |
147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | {% for job in job_status.COMPLETED %} 160 | {% if job.job_instance %} 161 | 162 | 163 | 164 | 165 | 168 | {% if job.job_instance.priority == -1 %} 169 | 172 | {% elif job.job_instance.priority == 0 %} 173 | 176 | {% elif job.job_instance.priority == 1 %} 177 | 180 | {% elif job.job_instance.priority == 2 %} 181 | 184 | {% endif %} 185 | 186 | 187 | 190 | {% if job.running_status == 2 %} 191 | 194 | {% else %} 195 | 198 | {% endif %} 199 | 200 | {% endif %} 201 | {% endfor %} 202 |
#JobSpiderArgsPriorityRuntimeStartedLogStatus
{{ job.job_execution_id }}{{ job.job_instance_id }}{{ job.job_instance.spider_name }}{{ job.job_instance.spider_arguments }} 167 | 170 | LOW 171 | 174 | NORMAL 175 | 178 | HIGH 179 | 182 | HIGHEST 183 | {{ timedelta(job.end_time,job.start_time) }}{{ job.start_time }}Log 189 | 192 | FINISHED 193 | 196 | CANCELED 197 |
203 |
204 |
205 | 206 | 258 | 259 | {% endblock %} -------------------------------------------------------------------------------- /SpiderKeeper/app/static/js/demo.js: -------------------------------------------------------------------------------- 1 | /** 2 | * AdminLTE Demo Menu 3 | * ------------------ 4 | * You should not use this file in production. 5 | * This file is for demo purposes only. 6 | */ 7 | (function ($, AdminLTE) { 8 | 9 | "use strict"; 10 | 11 | /** 12 | * List of all the available skins 13 | * 14 | * @type Array 15 | */ 16 | var my_skins = [ 17 | "skin-blue", 18 | "skin-black", 19 | "skin-red", 20 | "skin-yellow", 21 | "skin-purple", 22 | "skin-green", 23 | "skin-blue-light", 24 | "skin-black-light", 25 | "skin-red-light", 26 | "skin-yellow-light", 27 | "skin-purple-light", 28 | "skin-green-light" 29 | ]; 30 | 31 | //Create the new tab 32 | var tab_pane = $("
", { 33 | "id": "control-sidebar-theme-demo-options-tab", 34 | "class": "tab-pane active" 35 | }); 36 | 37 | //Create the tab button 38 | var tab_button = $("
  • ", {"class": "active"}) 39 | .html("" 40 | + "" 41 | + ""); 42 | 43 | //Add the tab button to the right sidebar tabs 44 | $("[href='#control-sidebar-home-tab']") 45 | .parent() 46 | .before(tab_button); 47 | 48 | //Create the menu 49 | var demo_settings = $("
    "); 50 | 51 | //Layout options 52 | demo_settings.append( 53 | "

    " 54 | + "Layout Options" 55 | + "

    " 56 | //Fixed layout 57 | + "
    " 58 | + "" 62 | + "

    Activate the fixed layout. You can't use fixed and boxed layouts together

    " 63 | + "
    " 64 | //Boxed layout 65 | + "
    " 66 | + "" 70 | + "

    Activate the boxed layout

    " 71 | + "
    " 72 | //Sidebar Toggle 73 | + "
    " 74 | + "" 78 | + "

    Toggle the left sidebar's state (open or collapse)

    " 79 | + "
    " 80 | //Sidebar mini expand on hover toggle 81 | + "
    " 82 | + "" 86 | + "

    Let the sidebar mini expand on hover

    " 87 | + "
    " 88 | //Control Sidebar Toggle 89 | + "
    " 90 | + "" 94 | + "

    Toggle between slide over content and push content effects

    " 95 | + "
    " 96 | //Control Sidebar Skin Toggle 97 | + "
    " 98 | + "" 102 | + "

    Toggle between dark and light skins for the right sidebar

    " 103 | + "
    " 104 | ); 105 | var skins_list = $("
      ", {"class": 'list-unstyled clearfix'}); 106 | 107 | //Dark sidebar skins 108 | var skin_blue = 109 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 110 | .append("" 111 | + "
      " 112 | + "
      " 113 | + "      " 114 | + "

      Blue

      "); 115 | skins_list.append(skin_blue); 116 | var skin_black = 117 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 118 | .append("" 119 | + "
      " 120 | + "
      " 121 | + "      " 122 | + "

      Black

      "); 123 | skins_list.append(skin_black); 124 | var skin_purple = 125 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 126 | .append("" 127 | + "
      " 128 | + "
      " 129 | + "      " 130 | + "

      Purple

      "); 131 | skins_list.append(skin_purple); 132 | var skin_green = 133 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 134 | .append("" 135 | + "
      " 136 | + "
      " 137 | + "      " 138 | + "

      Green

      "); 139 | skins_list.append(skin_green); 140 | var skin_red = 141 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 142 | .append("" 143 | + "
      " 144 | + "
      " 145 | + "      " 146 | + "

      Red

      "); 147 | skins_list.append(skin_red); 148 | var skin_yellow = 149 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 150 | .append("" 151 | + "
      " 152 | + "
      " 153 | + "      " 154 | + "

      Yellow

      "); 155 | skins_list.append(skin_yellow); 156 | 157 | //Light sidebar skins 158 | var skin_blue_light = 159 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 160 | .append("" 161 | + "
      " 162 | + "
      " 163 | + "      " 164 | + "

      Blue Light

      "); 165 | skins_list.append(skin_blue_light); 166 | var skin_black_light = 167 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 168 | .append("" 169 | + "
      " 170 | + "
      " 171 | + "      " 172 | + "

      Black Light

      "); 173 | skins_list.append(skin_black_light); 174 | var skin_purple_light = 175 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 176 | .append("" 177 | + "
      " 178 | + "
      " 179 | + "      " 180 | + "

      Purple Light

      "); 181 | skins_list.append(skin_purple_light); 182 | var skin_green_light = 183 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 184 | .append("" 185 | + "
      " 186 | + "
      " 187 | + "      " 188 | + "

      Green Light

      "); 189 | skins_list.append(skin_green_light); 190 | var skin_red_light = 191 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 192 | .append("" 193 | + "
      " 194 | + "
      " 195 | + "      " 196 | + "

      Red Light

      "); 197 | skins_list.append(skin_red_light); 198 | var skin_yellow_light = 199 | $("
    • ", {style: "float:left; width: 33.33333%; padding: 5px;"}) 200 | .append("" 201 | + "
      " 202 | + "
      " 203 | + "      " 204 | + "

      Yellow Light

      "); 205 | skins_list.append(skin_yellow_light); 206 | 207 | demo_settings.append("

      Skins

      "); 208 | demo_settings.append(skins_list); 209 | 210 | tab_pane.append(demo_settings); 211 | $("#control-sidebar-home-tab").after(tab_pane); 212 | 213 | setup(); 214 | 215 | /** 216 | * Toggles layout classes 217 | * 218 | * @param String cls the layout class to toggle 219 | * @returns void 220 | */ 221 | function change_layout(cls) { 222 | $("body").toggleClass(cls); 223 | AdminLTE.layout.fixSidebar(); 224 | //Fix the problem with right sidebar and layout boxed 225 | if (cls == "layout-boxed") 226 | AdminLTE.controlSidebar._fix($(".control-sidebar-bg")); 227 | if ($('body').hasClass('fixed') && cls == 'fixed') { 228 | AdminLTE.pushMenu.expandOnHover(); 229 | AdminLTE.layout.activate(); 230 | } 231 | AdminLTE.controlSidebar._fix($(".control-sidebar-bg")); 232 | AdminLTE.controlSidebar._fix($(".control-sidebar")); 233 | } 234 | 235 | /** 236 | * Replaces the old skin with the new skin 237 | * @param String cls the new skin class 238 | * @returns Boolean false to prevent link's default action 239 | */ 240 | function change_skin(cls) { 241 | $.each(my_skins, function (i) { 242 | $("body").removeClass(my_skins[i]); 243 | }); 244 | 245 | $("body").addClass(cls); 246 | store('skin', cls); 247 | return false; 248 | } 249 | 250 | /** 251 | * Store a new settings in the browser 252 | * 253 | * @param String name Name of the setting 254 | * @param String val Value of the setting 255 | * @returns void 256 | */ 257 | function store(name, val) { 258 | if (typeof (Storage) !== "undefined") { 259 | localStorage.setItem(name, val); 260 | } else { 261 | window.alert('Please use a modern browser to properly view this template!'); 262 | } 263 | } 264 | 265 | /** 266 | * Get a prestored setting 267 | * 268 | * @param String name Name of of the setting 269 | * @returns String The value of the setting | null 270 | */ 271 | function get(name) { 272 | if (typeof (Storage) !== "undefined") { 273 | return localStorage.getItem(name); 274 | } else { 275 | window.alert('Please use a modern browser to properly view this template!'); 276 | } 277 | } 278 | 279 | /** 280 | * Retrieve default settings and apply them to the template 281 | * 282 | * @returns void 283 | */ 284 | function setup() { 285 | var tmp = get('skin'); 286 | if (tmp && $.inArray(tmp, my_skins)) 287 | change_skin(tmp); 288 | 289 | //Add the change skin listener 290 | $("[data-skin]").on('click', function (e) { 291 | if($(this).hasClass('knob')) 292 | return; 293 | e.preventDefault(); 294 | change_skin($(this).data('skin')); 295 | }); 296 | 297 | //Add the layout manager 298 | $("[data-layout]").on('click', function () { 299 | change_layout($(this).data('layout')); 300 | }); 301 | 302 | $("[data-controlsidebar]").on('click', function () { 303 | change_layout($(this).data('controlsidebar')); 304 | var slide = !AdminLTE.options.controlSidebarOptions.slide; 305 | AdminLTE.options.controlSidebarOptions.slide = slide; 306 | if (!slide) 307 | $('.control-sidebar').removeClass('control-sidebar-open'); 308 | }); 309 | 310 | $("[data-sidebarskin='toggle']").on('click', function () { 311 | var sidebar = $(".control-sidebar"); 312 | if (sidebar.hasClass("control-sidebar-dark")) { 313 | sidebar.removeClass("control-sidebar-dark") 314 | sidebar.addClass("control-sidebar-light") 315 | } else { 316 | sidebar.removeClass("control-sidebar-light") 317 | sidebar.addClass("control-sidebar-dark") 318 | } 319 | }); 320 | 321 | $("[data-enable='expandOnHover']").on('click', function () { 322 | $(this).attr('disabled', true); 323 | AdminLTE.pushMenu.expandOnHover(); 324 | if (!$('body').hasClass('sidebar-collapse')) 325 | $("[data-layout='sidebar-collapse']").click(); 326 | }); 327 | 328 | // Reset options 329 | if ($('body').hasClass('fixed')) { 330 | $("[data-layout='fixed']").attr('checked', 'checked'); 331 | } 332 | if ($('body').hasClass('layout-boxed')) { 333 | $("[data-layout='layout-boxed']").attr('checked', 'checked'); 334 | } 335 | if ($('body').hasClass('sidebar-collapse')) { 336 | $("[data-layout='sidebar-collapse']").attr('checked', 'checked'); 337 | } 338 | 339 | } 340 | })(jQuery, $.AdminLTE); 341 | -------------------------------------------------------------------------------- /SpiderKeeper/app/spider/controller.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import tempfile 4 | 5 | import flask_restful 6 | import requests 7 | from flask import Blueprint, request 8 | from flask import abort 9 | from flask import flash 10 | from flask import redirect 11 | from flask import render_template 12 | from flask import session 13 | from flask_restful_swagger import swagger 14 | from werkzeug.utils import secure_filename 15 | 16 | from SpiderKeeper.app import db, api, agent, app 17 | from SpiderKeeper.app.spider.model import JobInstance, Project, JobExecution, SpiderInstance, JobRunType 18 | 19 | api_spider_bp = Blueprint('spider', __name__) 20 | 21 | ''' 22 | ========= api ========= 23 | ''' 24 | 25 | 26 | class ProjectCtrl(flask_restful.Resource): 27 | @swagger.operation( 28 | summary='list projects', 29 | parameters=[]) 30 | def get(self): 31 | return [project.to_dict() for project in Project.query.all()] 32 | 33 | @swagger.operation( 34 | summary='add project', 35 | parameters=[{ 36 | "name": "project_name", 37 | "description": "project name", 38 | "required": True, 39 | "paramType": "form", 40 | "dataType": 'string' 41 | }]) 42 | def post(self): 43 | project_name = request.form['project_name'] 44 | project = Project() 45 | project.project_name = project_name 46 | db.session.add(project) 47 | db.session.commit() 48 | return project.to_dict() 49 | 50 | 51 | class SpiderCtrl(flask_restful.Resource): 52 | @swagger.operation( 53 | summary='list spiders', 54 | parameters=[{ 55 | "name": "project_id", 56 | "description": "project id", 57 | "required": True, 58 | "paramType": "path", 59 | "dataType": 'int' 60 | }]) 61 | def get(self, project_id): 62 | project = Project.find_project_by_id(project_id) 63 | return [spider_instance.to_dict() for spider_instance in 64 | SpiderInstance.query.filter_by(project_id=project_id).all()] 65 | 66 | 67 | class SpiderDetailCtrl(flask_restful.Resource): 68 | @swagger.operation( 69 | summary='spider detail', 70 | parameters=[{ 71 | "name": "project_id", 72 | "description": "project id", 73 | "required": True, 74 | "paramType": "path", 75 | "dataType": 'int' 76 | }, { 77 | "name": "spider_id", 78 | "description": "spider instance id", 79 | "required": True, 80 | "paramType": "path", 81 | "dataType": 'int' 82 | }]) 83 | def get(self, project_id, spider_id): 84 | spider_instance = SpiderInstance.query.filter_by(project_id=project_id, id=spider_id).first() 85 | return spider_instance.to_dict() if spider_instance else abort(404) 86 | 87 | @swagger.operation( 88 | summary='run spider', 89 | parameters=[{ 90 | "name": "project_id", 91 | "description": "project id", 92 | "required": True, 93 | "paramType": "path", 94 | "dataType": 'int' 95 | }, { 96 | "name": "spider_id", 97 | "description": "spider instance id", 98 | "required": True, 99 | "paramType": "path", 100 | "dataType": 'int' 101 | }, { 102 | "name": "spider_arguments", 103 | "description": "spider arguments", 104 | "required": False, 105 | "paramType": "form", 106 | "dataType": 'string' 107 | }, { 108 | "name": "priority", 109 | "description": "LOW: -1, NORMAL: 0, HIGH: 1, HIGHEST: 2", 110 | "required": False, 111 | "paramType": "form", 112 | "dataType": 'int' 113 | }, { 114 | "name": "tags", 115 | "description": "spider tags", 116 | "required": False, 117 | "paramType": "form", 118 | "dataType": 'string' 119 | }, { 120 | "name": "desc", 121 | "description": "spider desc", 122 | "required": False, 123 | "paramType": "form", 124 | "dataType": 'string' 125 | }]) 126 | def put(self, project_id, spider_id): 127 | spider_instance = SpiderInstance.query.filter_by(project_id=project_id, id=spider_id).first() 128 | if not spider_instance: abort(404) 129 | job_instance = JobInstance() 130 | job_instance.spider_name = spider_instance.spider_name 131 | job_instance.project_id = project_id 132 | job_instance.spider_arguments = request.form.get('spider_arguments') 133 | job_instance.desc = request.form.get('desc') 134 | job_instance.tags = request.form.get('tags') 135 | job_instance.run_type = JobRunType.ONETIME 136 | job_instance.priority = request.form.get('priority', 0) 137 | job_instance.enabled = -1 138 | db.session.add(job_instance) 139 | db.session.commit() 140 | agent.start_spider(job_instance) 141 | return True 142 | 143 | 144 | JOB_INSTANCE_FIELDS = [column.name for column in JobInstance.__table__.columns] 145 | JOB_INSTANCE_FIELDS.remove('id') 146 | JOB_INSTANCE_FIELDS.remove('date_created') 147 | JOB_INSTANCE_FIELDS.remove('date_modified') 148 | 149 | 150 | class JobCtrl(flask_restful.Resource): 151 | @swagger.operation( 152 | summary='list job instance', 153 | parameters=[{ 154 | "name": "project_id", 155 | "description": "project id", 156 | "required": True, 157 | "paramType": "path", 158 | "dataType": 'int' 159 | }]) 160 | def get(self, project_id): 161 | return [job_instance.to_dict() for job_instance in 162 | JobInstance.query.filter_by(run_type="periodic", project_id=project_id).all()] 163 | 164 | @swagger.operation( 165 | summary='add job instance', 166 | notes="json keys:
      " + "
      ".join(JOB_INSTANCE_FIELDS), 167 | parameters=[{ 168 | "name": "project_id", 169 | "description": "project id", 170 | "required": True, 171 | "paramType": "path", 172 | "dataType": 'int' 173 | }, { 174 | "name": "spider_name", 175 | "description": "spider_name", 176 | "required": True, 177 | "paramType": "form", 178 | "dataType": 'string' 179 | }, { 180 | "name": "spider_arguments", 181 | "description": "spider_arguments, split by ','", 182 | "required": False, 183 | "paramType": "form", 184 | "dataType": 'string' 185 | }, { 186 | "name": "desc", 187 | "description": "desc", 188 | "required": False, 189 | "paramType": "form", 190 | "dataType": 'string' 191 | }, { 192 | "name": "tags", 193 | "description": "tags , split by ','", 194 | "required": False, 195 | "paramType": "form", 196 | "dataType": 'string' 197 | }, { 198 | "name": "run_type", 199 | "description": "onetime/periodic", 200 | "required": False, 201 | "paramType": "form", 202 | "dataType": 'string' 203 | }, { 204 | "name": "priority", 205 | "description": "LOW: -1, NORMAL: 0, HIGH: 1, HIGHEST: 2", 206 | "required": False, 207 | "paramType": "form", 208 | "dataType": 'int' 209 | }, { 210 | "name": "cron_minutes", 211 | "description": "@see http://apscheduler.readthedocs.io/en/latest/modules/triggers/cron.html", 212 | "required": False, 213 | "paramType": "form", 214 | "dataType": 'string' 215 | }, { 216 | "name": "cron_hour", 217 | "description": "", 218 | "required": False, 219 | "paramType": "form", 220 | "dataType": 'string' 221 | }, { 222 | "name": "cron_day_of_month", 223 | "description": "", 224 | "required": False, 225 | "paramType": "form", 226 | "dataType": 'string' 227 | }, { 228 | "name": "cron_day_of_week", 229 | "description": "", 230 | "required": False, 231 | "paramType": "form", 232 | "dataType": 'string' 233 | }, { 234 | "name": "cron_month", 235 | "description": "", 236 | "required": False, 237 | "paramType": "form", 238 | "dataType": 'string' 239 | }]) 240 | def post(self, project_id): 241 | post_data = request.form 242 | if post_data: 243 | job_instance = JobInstance() 244 | job_instance.spider_name = post_data['spider_name'] 245 | job_instance.project_id = project_id 246 | job_instance.spider_arguments = post_data.get('spider_arguments') 247 | job_instance.desc = post_data.get('desc') 248 | job_instance.tags = post_data.get('tags') 249 | job_instance.run_type = post_data['run_type'] 250 | job_instance.priority = post_data.get('priority', 0) 251 | if job_instance.run_type == "periodic": 252 | job_instance.cron_minutes = post_data.get('cron_minutes') or '0' 253 | job_instance.cron_hour = post_data.get('cron_hour') or '*' 254 | job_instance.cron_day_of_month = post_data.get('cron_day_of_month') or '*' 255 | job_instance.cron_day_of_week = post_data.get('cron_day_of_week') or '*' 256 | job_instance.cron_month = post_data.get('cron_month') or '*' 257 | db.session.add(job_instance) 258 | db.session.commit() 259 | return True 260 | 261 | 262 | class JobDetailCtrl(flask_restful.Resource): 263 | @swagger.operation( 264 | summary='update job instance', 265 | notes="json keys:
      " + "
      ".join(JOB_INSTANCE_FIELDS), 266 | parameters=[{ 267 | "name": "project_id", 268 | "description": "project id", 269 | "required": True, 270 | "paramType": "path", 271 | "dataType": 'int' 272 | }, { 273 | "name": "job_id", 274 | "description": "job instance id", 275 | "required": True, 276 | "paramType": "path", 277 | "dataType": 'int' 278 | }, { 279 | "name": "spider_name", 280 | "description": "spider_name", 281 | "required": False, 282 | "paramType": "form", 283 | "dataType": 'string' 284 | }, { 285 | "name": "spider_arguments", 286 | "description": "spider_arguments, split by ','", 287 | "required": False, 288 | "paramType": "form", 289 | "dataType": 'string' 290 | }, { 291 | "name": "desc", 292 | "description": "desc", 293 | "required": False, 294 | "paramType": "form", 295 | "dataType": 'string' 296 | }, { 297 | "name": "tags", 298 | "description": "tags , split by ','", 299 | "required": False, 300 | "paramType": "form", 301 | "dataType": 'string' 302 | }, { 303 | "name": "run_type", 304 | "description": "onetime/periodic", 305 | "required": False, 306 | "paramType": "form", 307 | "dataType": 'string' 308 | }, { 309 | "name": "priority", 310 | "description": "LOW: -1, NORMAL: 0, HIGH: 1, HIGHEST: 2", 311 | "required": False, 312 | "paramType": "form", 313 | "dataType": 'int' 314 | }, { 315 | "name": "cron_minutes", 316 | "description": "@see http://apscheduler.readthedocs.io/en/latest/modules/triggers/cron.html", 317 | "required": False, 318 | "paramType": "form", 319 | "dataType": 'string' 320 | }, { 321 | "name": "cron_hour", 322 | "description": "", 323 | "required": False, 324 | "paramType": "form", 325 | "dataType": 'string' 326 | }, { 327 | "name": "cron_day_of_month", 328 | "description": "", 329 | "required": False, 330 | "paramType": "form", 331 | "dataType": 'string' 332 | }, { 333 | "name": "cron_day_of_week", 334 | "description": "", 335 | "required": False, 336 | "paramType": "form", 337 | "dataType": 'string' 338 | }, { 339 | "name": "cron_month", 340 | "description": "", 341 | "required": False, 342 | "paramType": "form", 343 | "dataType": 'string' 344 | }, { 345 | "name": "enabled", 346 | "description": "-1 / 0, default: 0", 347 | "required": False, 348 | "paramType": "form", 349 | "dataType": 'int' 350 | }, { 351 | "name": "status", 352 | "description": "if set to 'run' will run the job", 353 | "required": False, 354 | "paramType": "form", 355 | "dataType": 'int' 356 | } 357 | 358 | ]) 359 | def put(self, project_id, job_id): 360 | post_data = request.form 361 | if post_data: 362 | job_instance = JobInstance.query.filter_by(project_id=project_id, id=job_id).first() 363 | if not job_instance: abort(404) 364 | job_instance.spider_arguments = post_data.get('spider_arguments') or job_instance.spider_arguments 365 | job_instance.priority = post_data.get('priority') or job_instance.priority 366 | job_instance.enabled = post_data.get('enabled', 0) 367 | job_instance.cron_minutes = post_data.get('cron_minutes') or job_instance.cron_minutes 368 | job_instance.cron_hour = post_data.get('cron_hour') or job_instance.cron_hour 369 | job_instance.cron_day_of_month = post_data.get('cron_day_of_month') or job_instance.cron_day_of_month 370 | job_instance.cron_day_of_week = post_data.get('cron_day_of_week') or job_instance.cron_day_of_week 371 | job_instance.cron_month = post_data.get('cron_month') or job_instance.cron_month 372 | job_instance.desc = post_data.get('desc', 0) or job_instance.desc 373 | job_instance.tags = post_data.get('tags', 0) or job_instance.tags 374 | db.session.commit() 375 | if post_data.get('status') == 'run': 376 | agent.start_spider(job_instance) 377 | return True 378 | 379 | 380 | class JobExecutionCtrl(flask_restful.Resource): 381 | @swagger.operation( 382 | summary='list job execution status', 383 | parameters=[{ 384 | "name": "project_id", 385 | "description": "project id", 386 | "required": True, 387 | "paramType": "path", 388 | "dataType": 'int' 389 | }]) 390 | def get(self, project_id): 391 | return JobExecution.list_jobs(project_id) 392 | 393 | 394 | class JobExecutionDetailCtrl(flask_restful.Resource): 395 | @swagger.operation( 396 | summary='stop job', 397 | notes='', 398 | parameters=[ 399 | { 400 | "name": "project_id", 401 | "description": "project id", 402 | "required": True, 403 | "paramType": "path", 404 | "dataType": 'int' 405 | }, 406 | { 407 | "name": "job_exec_id", 408 | "description": "job_execution_id", 409 | "required": True, 410 | "paramType": "path", 411 | "dataType": 'string' 412 | } 413 | ]) 414 | def put(self, project_id, job_exec_id): 415 | job_execution = JobExecution.query.filter_by(project_id=project_id, id=job_exec_id).first() 416 | if job_execution: 417 | agent.cancel_spider(job_execution) 418 | return True 419 | 420 | 421 | api.add_resource(ProjectCtrl, "/api/projects") 422 | api.add_resource(SpiderCtrl, "/api/projects//spiders") 423 | api.add_resource(SpiderDetailCtrl, "/api/projects//spiders/") 424 | api.add_resource(JobCtrl, "/api/projects//jobs") 425 | api.add_resource(JobDetailCtrl, "/api/projects//jobs/") 426 | api.add_resource(JobExecutionCtrl, "/api/projects//jobexecs") 427 | api.add_resource(JobExecutionDetailCtrl, "/api/projects//jobexecs/") 428 | 429 | ''' 430 | ========= Router ========= 431 | ''' 432 | 433 | 434 | @app.before_request 435 | def intercept_no_project(): 436 | if request.path.find('/project//') > -1: 437 | flash("create project first") 438 | return redirect("/project/manage", code=302) 439 | 440 | 441 | @app.context_processor 442 | def inject_common(): 443 | return dict(now=datetime.datetime.now(), 444 | servers=agent.servers) 445 | 446 | 447 | @app.context_processor 448 | def inject_project(): 449 | project_context = {} 450 | project_context['project_list'] = Project.query.all() 451 | if project_context['project_list'] and (not session.get('project_id')): 452 | project = Project.query.first() 453 | session['project_id'] = project.id 454 | if session.get('project_id'): 455 | project_context['project'] = Project.find_project_by_id(session['project_id']) 456 | project_context['spider_list'] = [spider_instance.to_dict() for spider_instance in 457 | SpiderInstance.query.filter_by(project_id=session['project_id']).all()] 458 | else: 459 | project_context['project'] = {} 460 | return project_context 461 | 462 | 463 | @app.context_processor 464 | def utility_processor(): 465 | def timedelta(end_time, start_time): 466 | ''' 467 | 468 | :param end_time: 469 | :param start_time: 470 | :param unit: s m h 471 | :return: 472 | ''' 473 | if not end_time or not start_time: 474 | return '' 475 | if type(end_time) == str: 476 | end_time = datetime.datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S') 477 | if type(start_time) == str: 478 | start_time = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') 479 | total_seconds = (end_time - start_time).total_seconds() 480 | return readable_time(total_seconds) 481 | 482 | def readable_time(total_seconds): 483 | if not total_seconds: 484 | return '-' 485 | if total_seconds < 60: 486 | return '%s s' % total_seconds 487 | if total_seconds < 3600: 488 | return '%s m' % int(total_seconds / 60) 489 | return '%s h %s m' % (int(total_seconds / 3600), int((total_seconds % 3600) / 60)) 490 | 491 | return dict(timedelta=timedelta, readable_time=readable_time) 492 | 493 | 494 | @app.route("/") 495 | def index(): 496 | project = Project.query.first() 497 | if project: 498 | return redirect("/project/%s/job/dashboard" % project.id, code=302) 499 | return redirect("/project/manage", code=302) 500 | 501 | 502 | @app.route("/project/") 503 | def project_index(project_id): 504 | session['project_id'] = project_id 505 | return redirect("/project/%s/job/dashboard" % project_id, code=302) 506 | 507 | 508 | @app.route("/project/create", methods=['post']) 509 | def project_create(): 510 | project_name = request.form['project_name'] 511 | project = Project() 512 | project.project_name = project_name 513 | db.session.add(project) 514 | db.session.commit() 515 | return redirect("/project/%s/spider/deploy" % project.id, code=302) 516 | 517 | 518 | @app.route("/project//delete") 519 | def project_delete(project_id): 520 | project = Project.find_project_by_id(project_id) 521 | agent.delete_project(project) 522 | db.session.delete(project) 523 | db.session.commit() 524 | return redirect("/project/manage", code=302) 525 | 526 | 527 | @app.route("/project/manage") 528 | def project_manage(): 529 | return render_template("project_manage.html") 530 | 531 | 532 | @app.route("/project//job/dashboard") 533 | def job_dashboard(project_id): 534 | return render_template("job_dashboard.html", job_status=JobExecution.list_jobs(project_id)) 535 | 536 | 537 | @app.route("/project//job/periodic") 538 | def job_periodic(project_id): 539 | project = Project.find_project_by_id(project_id) 540 | job_instance_list = [job_instance.to_dict() for job_instance in 541 | JobInstance.query.filter_by(run_type="periodic", project_id=project_id).all()] 542 | return render_template("job_periodic.html", 543 | job_instance_list=job_instance_list) 544 | 545 | 546 | @app.route("/project//job/add", methods=['post']) 547 | def job_add(project_id): 548 | project = Project.find_project_by_id(project_id) 549 | job_instance = JobInstance() 550 | job_instance.spider_name = request.form['spider_name'] 551 | job_instance.project_id = project_id 552 | job_instance.spider_arguments = request.form['spider_arguments'] 553 | job_instance.priority = request.form.get('priority', 0) 554 | job_instance.run_type = request.form['run_type'] 555 | # chose daemon manually 556 | if request.form['daemon'] != 'auto': 557 | spider_args = [] 558 | if request.form['spider_arguments']: 559 | spider_args = request.form['spider_arguments'].split(",") 560 | spider_args.append("daemon={}".format(request.form['daemon'])) 561 | job_instance.spider_arguments = ','.join(spider_args) 562 | if job_instance.run_type == JobRunType.ONETIME: 563 | job_instance.enabled = -1 564 | db.session.add(job_instance) 565 | db.session.commit() 566 | agent.start_spider(job_instance) 567 | if job_instance.run_type == JobRunType.PERIODIC: 568 | job_instance.cron_minutes = request.form.get('cron_minutes') or '0' 569 | job_instance.cron_hour = request.form.get('cron_hour') or '*' 570 | job_instance.cron_day_of_month = request.form.get('cron_day_of_month') or '*' 571 | job_instance.cron_day_of_week = request.form.get('cron_day_of_week') or '*' 572 | job_instance.cron_month = request.form.get('cron_month') or '*' 573 | # set cron exp manually 574 | if request.form.get('cron_exp'): 575 | job_instance.cron_minutes, job_instance.cron_hour, job_instance.cron_day_of_month, job_instance.cron_day_of_week, job_instance.cron_month = \ 576 | request.form['cron_exp'].split(' ') 577 | db.session.add(job_instance) 578 | db.session.commit() 579 | return redirect(request.referrer, code=302) 580 | 581 | 582 | @app.route("/project//jobexecs//stop") 583 | def job_stop(project_id, job_exec_id): 584 | job_execution = JobExecution.query.filter_by(project_id=project_id, id=job_exec_id).first() 585 | agent.cancel_spider(job_execution) 586 | return redirect(request.referrer, code=302) 587 | 588 | 589 | @app.route("/project//jobexecs//log") 590 | def job_log(project_id, job_exec_id): 591 | job_execution = JobExecution.query.filter_by(project_id=project_id, id=job_exec_id).first() 592 | res = requests.get(agent.log_url(job_execution)) 593 | res.encoding = 'utf8' 594 | raw = res.text 595 | return render_template("job_log.html", log_lines=raw.split('\n')) 596 | 597 | 598 | @app.route("/project//job//run") 599 | def job_run(project_id, job_instance_id): 600 | job_instance = JobInstance.query.filter_by(project_id=project_id, id=job_instance_id).first() 601 | agent.start_spider(job_instance) 602 | return redirect(request.referrer, code=302) 603 | 604 | 605 | @app.route("/project//job//remove") 606 | def job_remove(project_id, job_instance_id): 607 | job_instance = JobInstance.query.filter_by(project_id=project_id, id=job_instance_id).first() 608 | db.session.delete(job_instance) 609 | db.session.commit() 610 | return redirect(request.referrer, code=302) 611 | 612 | 613 | @app.route("/project//job//switch") 614 | def job_switch(project_id, job_instance_id): 615 | job_instance = JobInstance.query.filter_by(project_id=project_id, id=job_instance_id).first() 616 | job_instance.enabled = -1 if job_instance.enabled == 0 else 0 617 | db.session.commit() 618 | return redirect(request.referrer, code=302) 619 | 620 | 621 | @app.route("/project//spider/dashboard") 622 | def spider_dashboard(project_id): 623 | spider_instance_list = SpiderInstance.list_spiders(project_id) 624 | return render_template("spider_dashboard.html", 625 | spider_instance_list=spider_instance_list) 626 | 627 | 628 | @app.route("/project//spider/deploy") 629 | def spider_deploy(project_id): 630 | project = Project.find_project_by_id(project_id) 631 | return render_template("spider_deploy.html") 632 | 633 | 634 | @app.route("/project//spider/upload", methods=['post']) 635 | def spider_egg_upload(project_id): 636 | project = Project.find_project_by_id(project_id) 637 | if 'file' not in request.files: 638 | flash('No file part') 639 | return redirect(request.referrer) 640 | file = request.files['file'] 641 | # if user does not select file, browser also 642 | # submit a empty part without filename 643 | if file.filename == '': 644 | flash('No selected file') 645 | return redirect(request.referrer) 646 | if file: 647 | filename = secure_filename(file.filename) 648 | dst = os.path.join(tempfile.gettempdir(), filename) 649 | file.save(dst) 650 | agent.deploy(project, dst) 651 | flash('deploy success!') 652 | return redirect(request.referrer) 653 | 654 | 655 | @app.route("/project//project/stats") 656 | def project_stats(project_id): 657 | project = Project.find_project_by_id(project_id) 658 | run_stats = JobExecution.list_run_stats_by_hours(project_id) 659 | return render_template("project_stats.html", run_stats=run_stats) 660 | 661 | 662 | @app.route("/project//server/stats") 663 | def service_stats(project_id): 664 | project = Project.find_project_by_id(project_id) 665 | run_stats = JobExecution.list_run_stats_by_hours(project_id) 666 | return render_template("server_stats.html", run_stats=run_stats) 667 | -------------------------------------------------------------------------------- /SpiderKeeper/app/static/css/font-awesome.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.7.0');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'),url('../fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} 5 | --------------------------------------------------------------------------------