├── .gitignore ├── README.md ├── __init__.py ├── scrapy.cfg ├── static ├── css │ ├── bootstrap-rtl.css │ ├── bootstrap-rtl.min.css │ ├── bootstrap.css │ ├── bootstrap.min.css │ ├── plugins │ │ └── morris.css │ ├── sb-admin-rtl.css │ └── sb-admin.css ├── font-awesome │ ├── css │ │ ├── font-awesome.css │ │ └── font-awesome.min.css │ ├── fonts │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ └── fontawesome-webfont.woff │ ├── less │ │ ├── bordered-pulled.less │ │ ├── core.less │ │ ├── fixed-width.less │ │ ├── font-awesome.less │ │ ├── icons.less │ │ ├── larger.less │ │ ├── list.less │ │ ├── mixins.less │ │ ├── path.less │ │ ├── rotated-flipped.less │ │ ├── spinning.less │ │ ├── stacked.less │ │ └── variables.less │ └── scss │ │ ├── _bordered-pulled.scss │ │ ├── _core.scss │ │ ├── _fixed-width.scss │ │ ├── _icons.scss │ │ ├── _larger.scss │ │ ├── _list.scss │ │ ├── _mixins.scss │ │ ├── _path.scss │ │ ├── _rotated-flipped.scss │ │ ├── _spinning.scss │ │ ├── _stacked.scss │ │ ├── _variables.scss │ │ └── font-awesome.scss ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 └── js │ ├── bootstrap.js │ ├── bootstrap.min.js │ ├── jquery.js │ └── plugins │ ├── flot │ ├── excanvas.min.js │ ├── flot-data.js │ ├── jquery.flot.js │ ├── jquery.flot.pie.js │ ├── jquery.flot.resize.js │ └── jquery.flot.tooltip.min.js │ └── morris │ ├── morris-data.js │ ├── morris.js │ ├── morris.min.js │ └── raphael.min.js ├── templates ├── block_page.html ├── charts.html ├── forms.html ├── index.html ├── list.html ├── list.html_bak ├── public_css.html ├── public_js.html ├── public_left.html ├── public_nav.html ├── search_list.html_bak ├── tables.html └── test.html ├── web.py └── webspider ├── QiniuStorage.py ├── QiniuStore.py ├── SearchApi.py ├── __init__.py ├── addSearch.py ├── conf ├── __init__.py └── config.py ├── delProxy.py ├── downloaderMiddlewareSet.py ├── getProxy.py ├── model ├── Article.py ├── Classify.py ├── Page.py ├── Proxy.py ├── Rule.py └── __init__.py ├── pipelines.py ├── run.py ├── search ├── __init__.py └── search._py ├── settings.py ├── spiders ├── ImgSpider.py ├── ProxySpider.py ├── RuleSpider.py ├── __init__.py └── item │ ├── ArticleItem.py │ ├── ImgItem.py │ ├── ProxyItem.py │ └── __init__.py └── util ├── UserAgent.py ├── WebProxy.py ├── __init__.py └── common.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | # IDE相关，不再同步 3 | /.idea -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ScrapyDemo 2 | ScrapyDemo : Redis MySQLdb logging IngoreHttpRequestMiddleware UserAgentMiddleware HttpProxyMiddleware rules 3 | 4 | 项目原名叫:webspider,如果不能使用请下载项目后修改为webspider再运行. 5 | 6 | 该项目是2015年底完成的,当时创建了github代码仓库忘了上传了.现在上传上来.只提供代码供大家参考,不保证代码正常运行. 7 | 8 | 项目使用了以下库,不完整,后续的有些忘了添加进来了. 9 | 10 | - Scrapy [ 如果提示没有lxml,就安装lxml,一般来说会自动安装lxml的 ] 11 | - Redis reids 的python操作库 12 | - Sqlalchemy orm 数据操作 13 | - Pymssql mssql的python操作库 14 | - Whoosh 全文搜索 15 | - Jieba 结巴分词 全文搜索中用到 16 | - Flask python 的web框架,提供搜索接口的时候使用 17 | - Qiniu 当初想用七牛云存储的保存下载的图片 18 | - Requests python 发送request的库 19 | 20 | ## 项目说明 21 | 项目附带代理采集,代理使用,爬虫,保存数据库,结巴分词,whoosh索引,flask开放web搜索接口,qiniu自动图片下载. 22 | 23 | ### 注:由于项目在实际交接的过程中采用了mssql,但是用Sqlalchemy过渡,所以如果想用mysql的话,只需要少量的代码改动即可实现,我还是主张代码仅供参考,毕竟时间太长了,我也不想修改了. 24 | 25 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.org/en/latest/deploy.html 5 | 6 | [settings] 7 | default = webspider.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = webspider 12 | -------------------------------------------------------------------------------- /static/css/plugins/morris.css: -------------------------------------------------------------------------------- 1 | .morris-hover{position:absolute;z-index:1000}.morris-hover.morris-default-style{border-radius:10px;padding:6px;color:#666;background:rgba(255,255,255,0.8);border:solid 2px rgba(230,230,230,0.8);font-family:sans-serif;font-size:12px;text-align:center}.morris-hover.morris-default-style .morris-hover-row-label{font-weight:bold;margin:0.25em 0} 2 | .morris-hover.morris-default-style .morris-hover-point{white-space:nowrap;margin:0.1em 0} 3 | -------------------------------------------------------------------------------- /static/css/sb-admin-rtl.css: -------------------------------------------------------------------------------- 1 | 2 | @media (min-width: 768px){ 3 | #wrapper {padding-right: 225px; padding-left: 0;} 4 | .side-nav{right: 0;left: auto;} 5 | } -------------------------------------------------------------------------------- /static/css/sb-admin.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Start Bootstrap - SB Admin Bootstrap Admin Template (http://startbootstrap.com) 3 | * Code licensed under the Apache License v2.0. 4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0. 5 | */ 6 | 7 | /* Global Styles */ 8 | 9 | body { 10 | margin-top: 100px; 11 | background-color: #222; 12 | } 13 | 14 | @media(min-width:768px) { 15 | body { 16 | margin-top: 50px; 17 | } 18 | } 19 | 20 | #wrapper { 21 | padding-left: 0; 22 | } 23 | 24 | #page-wrapper { 25 | width: 100%; 26 | padding: 0; 27 | background-color: #fff; 28 | } 29 | #page-wrapper .container-fluid{ 30 | min-height: 800px; 31 | } 32 | 33 | .huge { 34 | font-size: 50px; 35 | line-height: normal; 36 | } 37 | 38 | @media(min-width:768px) { 39 | #wrapper { 40 | padding-left: 225px; 41 | } 42 | 43 | #page-wrapper { 44 | padding: 10px; 45 | } 46 | } 47 | 48 | /* Top Navigation */ 49 | 50 | .top-nav { 51 | padding: 0 15px; 52 | } 53 | 54 | .top-nav>li { 55 | display: inline-block; 56 | float: left; 57 | } 58 | 59 | .top-nav>li>a { 60 | padding-top: 15px; 61 | padding-bottom: 15px; 62 | line-height: 20px; 63 | color: #999; 64 | } 65 | 66 | .top-nav>li>a:hover, 67 | .top-nav>li>a:focus, 68 | .top-nav>.open>a, 69 | .top-nav>.open>a:hover, 70 | .top-nav>.open>a:focus { 71 | color: #fff; 72 | background-color: #000; 73 | } 74 | 75 | .top-nav>.open>.dropdown-menu { 76 | float: left; 77 | position: absolute; 78 | margin-top: 0; 79 | border: 1px solid rgba(0,0,0,.15); 80 | border-top-left-radius: 0; 81 | border-top-right-radius: 0; 82 | background-color: #fff; 83 | -webkit-box-shadow: 0 6px 12px rgba(0,0,0,.175); 84 | box-shadow: 0 6px 12px rgba(0,0,0,.175); 85 | } 86 | 87 | .top-nav>.open>.dropdown-menu>li>a { 88 | white-space: normal; 89 | } 90 | 91 | ul.message-dropdown { 92 | padding: 0; 93 | max-height: 250px; 94 | overflow-x: hidden; 95 | overflow-y: auto; 96 | } 97 | 98 | li.message-preview { 99 | width: 275px; 100 | border-bottom: 1px solid rgba(0,0,0,.15); 101 | } 102 | 103 | li.message-preview>a { 104 | padding-top: 15px; 105 | padding-bottom: 15px; 106 | } 107 | 108 | li.message-footer { 109 | margin: 5px 0; 110 | } 111 | 112 | ul.alert-dropdown { 113 | width: 200px; 114 | } 115 | 116 | /* Side Navigation */ 117 | 118 | @media(min-width:768px) { 119 | .side-nav { 120 | position: fixed; 121 | top: 51px; 122 | left: 225px; 123 | width: 225px; 124 | margin-left: -225px; 125 | border: none; 126 | border-radius: 0; 127 | overflow-y: auto; 128 | background-color: #222; 129 | bottom: 0; 130 | overflow-x: hidden; 131 | padding-bottom: 40px; 132 | } 133 | 134 | .side-nav>li>a { 135 | width: 225px; 136 | } 137 | 138 | .side-nav li a:hover, 139 | .side-nav li a:focus { 140 | outline: none; 141 | background-color: #000 !important; 142 | } 143 | } 144 | 145 | .side-nav>li>ul { 146 | padding: 0; 147 | } 148 | 149 | .side-nav>li>ul>li>a { 150 | display: block; 151 | padding: 10px 15px 10px 38px; 152 | text-decoration: none; 153 | color: #999; 154 | } 155 | 156 | .side-nav>li>ul>li>a:hover { 157 | color: #fff; 158 | } 159 | 160 | /* Flot Chart Containers */ 161 | 162 | .flot-chart { 163 | display: block; 164 | height: 400px; 165 | } 166 | 167 | .flot-chart-content { 168 | width: 100%; 169 | height: 100%; 170 | } 171 | 172 | /* Custom Colored Panels */ 173 | 174 | .huge { 175 | font-size: 40px; 176 | } 177 | 178 | .panel-green { 179 | border-color: #5cb85c; 180 | } 181 | 182 | .panel-green > .panel-heading { 183 | border-color: #5cb85c; 184 | color: #fff; 185 | background-color: #5cb85c; 186 | } 187 | 188 | .panel-green > a { 189 | color: #5cb85c; 190 | } 191 | 192 | .panel-green > a:hover { 193 | color: #3d8b3d; 194 | } 195 | 196 | .panel-red { 197 | border-color: #d9534f; 198 | } 199 | 200 | .panel-red > .panel-heading { 201 | border-color: #d9534f; 202 | color: #fff; 203 | background-color: #d9534f; 204 | } 205 | 206 | .panel-red > a { 207 | color: #d9534f; 208 | } 209 | 210 | .panel-red > a:hover { 211 | color: #b52b27; 212 | } 213 | 214 | .panel-yellow { 215 | border-color: #f0ad4e; 216 | } 217 | 218 | .panel-yellow > .panel-heading { 219 | border-color: #f0ad4e; 220 | color: #fff; 221 | background-color: #f0ad4e; 222 | } 223 | 224 | .panel-yellow > a { 225 | color: #f0ad4e; 226 | } 227 | 228 | .panel-yellow > a:hover { 229 | color: #df8a13; 230 | } 231 | -------------------------------------------------------------------------------- /static/font-awesome/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/font-awesome/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /static/font-awesome/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/font-awesome/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /static/font-awesome/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/font-awesome/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /static/font-awesome/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/font-awesome/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /static/font-awesome/less/bordered-pulled.less: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em @fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .@{fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /static/font-awesome/less/core.less: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | -------------------------------------------------------------------------------- /static/font-awesome/less/fixed-width.less: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .@{fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /static/font-awesome/less/font-awesome.less: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables.less"; 7 | @import "mixins.less"; 8 | @import "path.less"; 9 | @import "core.less"; 10 | @import "larger.less"; 11 | @import "fixed-width.less"; 12 | @import "list.less"; 13 | @import "bordered-pulled.less"; 14 | @import "spinning.less"; 15 | @import "rotated-flipped.less"; 16 | @import "stacked.less"; 17 | @import "icons.less"; 18 | -------------------------------------------------------------------------------- /static/font-awesome/less/larger.less: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .@{fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .@{fa-css-prefix}-2x { font-size: 2em; } 11 | .@{fa-css-prefix}-3x { font-size: 3em; } 12 | .@{fa-css-prefix}-4x { font-size: 4em; } 13 | .@{fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /static/font-awesome/less/list.less: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: @fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .@{fa-css-prefix}-li { 11 | position: absolute; 12 | left: -@fa-li-width; 13 | width: @fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.@{fa-css-prefix}-lg { 17 | left: (-@fa-li-width + (4em / 14)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /static/font-awesome/less/mixins.less: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | .fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | 13 | .fa-icon-rotate(@degrees, @rotation) { 14 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation); 15 | -webkit-transform: rotate(@degrees); 16 | -ms-transform: rotate(@degrees); 17 | transform: rotate(@degrees); 18 | } 19 | 20 | .fa-icon-flip(@horiz, @vert, @rotation) { 21 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation, mirror=1); 22 | -webkit-transform: scale(@horiz, @vert); 23 | -ms-transform: scale(@horiz, @vert); 24 | transform: scale(@horiz, @vert); 25 | } 26 | -------------------------------------------------------------------------------- /static/font-awesome/less/path.less: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('@{fa-font-path}/fontawesome-webfont.eot?v=@{fa-version}'); 7 | src: url('@{fa-font-path}/fontawesome-webfont.eot?#iefix&v=@{fa-version}') format('embedded-opentype'), 8 | url('@{fa-font-path}/fontawesome-webfont.woff?v=@{fa-version}') format('woff'), 9 | url('@{fa-font-path}/fontawesome-webfont.ttf?v=@{fa-version}') format('truetype'), 10 | url('@{fa-font-path}/fontawesome-webfont.svg?v=@{fa-version}#fontawesomeregular') format('svg'); 11 | // src: url('@{fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 12 | font-weight: normal; 13 | font-style: normal; 14 | } 15 | -------------------------------------------------------------------------------- /static/font-awesome/less/rotated-flipped.less: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-rotate-90 { .fa-icon-rotate(90deg, 1); } 5 | .@{fa-css-prefix}-rotate-180 { .fa-icon-rotate(180deg, 2); } 6 | .@{fa-css-prefix}-rotate-270 { .fa-icon-rotate(270deg, 3); } 7 | 8 | .@{fa-css-prefix}-flip-horizontal { .fa-icon-flip(-1, 1, 0); } 9 | .@{fa-css-prefix}-flip-vertical { .fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .@{fa-css-prefix}-rotate-90, 15 | :root .@{fa-css-prefix}-rotate-180, 16 | :root .@{fa-css-prefix}-rotate-270, 17 | :root .@{fa-css-prefix}-flip-horizontal, 18 | :root .@{fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /static/font-awesome/less/spinning.less: -------------------------------------------------------------------------------- 1 | // Spinning Icons 2 | // -------------------------- 3 | 4 | .@{fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | @-webkit-keyframes fa-spin { 10 | 0% { 11 | -webkit-transform: rotate(0deg); 12 | transform: rotate(0deg); 13 | } 14 | 100% { 15 | -webkit-transform: rotate(359deg); 16 | transform: rotate(359deg); 17 | } 18 | } 19 | 20 | @keyframes fa-spin { 21 | 0% { 22 | -webkit-transform: rotate(0deg); 23 | transform: rotate(0deg); 24 | } 25 | 100% { 26 | -webkit-transform: rotate(359deg); 27 | transform: rotate(359deg); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /static/font-awesome/less/stacked.less: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .@{fa-css-prefix}-stack-1x, .@{fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .@{fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .@{fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .@{fa-css-prefix}-inverse { color: @fa-inverse; } 21 | -------------------------------------------------------------------------------- /static/font-awesome/less/variables.less: -------------------------------------------------------------------------------- 1 | // Variables 2 | // -------------------------- 3 | 4 | @fa-font-path: "../fonts"; 5 | //@fa-font-path: "//netdna.bootstrapcdn.com/font-awesome/4.2.0/fonts"; // for referencing Bootstrap CDN font files directly 6 | @fa-css-prefix: fa; 7 | @fa-version: "4.2.0"; 8 | @fa-border-color: #eee; 9 | @fa-inverse: #fff; 10 | @fa-li-width: (30em / 14); 11 | 12 | @fa-var-adjust: "\f042"; 13 | @fa-var-adn: "\f170"; 14 | @fa-var-align-center: "\f037"; 15 | @fa-var-align-justify: "\f039"; 16 | @fa-var-align-left: "\f036"; 17 | @fa-var-align-right: "\f038"; 18 | @fa-var-ambulance: "\f0f9"; 19 | @fa-var-anchor: "\f13d"; 20 | @fa-var-android: "\f17b"; 21 | @fa-var-angellist: "\f209"; 22 | @fa-var-angle-double-down: "\f103"; 23 | @fa-var-angle-double-left: "\f100"; 24 | @fa-var-angle-double-right: "\f101"; 25 | @fa-var-angle-double-up: "\f102"; 26 | @fa-var-angle-down: "\f107"; 27 | @fa-var-angle-left: "\f104"; 28 | @fa-var-angle-right: "\f105"; 29 | @fa-var-angle-up: "\f106"; 30 | @fa-var-apple: "\f179"; 31 | @fa-var-archive: "\f187"; 32 | @fa-var-area-chart: "\f1fe"; 33 | @fa-var-arrow-circle-down: "\f0ab"; 34 | @fa-var-arrow-circle-left: "\f0a8"; 35 | @fa-var-arrow-circle-o-down: "\f01a"; 36 | @fa-var-arrow-circle-o-left: "\f190"; 37 | @fa-var-arrow-circle-o-right: "\f18e"; 38 | @fa-var-arrow-circle-o-up: "\f01b"; 39 | @fa-var-arrow-circle-right: "\f0a9"; 40 | @fa-var-arrow-circle-up: "\f0aa"; 41 | @fa-var-arrow-down: "\f063"; 42 | @fa-var-arrow-left: "\f060"; 43 | @fa-var-arrow-right: "\f061"; 44 | @fa-var-arrow-up: "\f062"; 45 | @fa-var-arrows: "\f047"; 46 | @fa-var-arrows-alt: "\f0b2"; 47 | @fa-var-arrows-h: "\f07e"; 48 | @fa-var-arrows-v: "\f07d"; 49 | @fa-var-asterisk: "\f069"; 50 | @fa-var-at: "\f1fa"; 51 | @fa-var-automobile: "\f1b9"; 52 | @fa-var-backward: "\f04a"; 53 | @fa-var-ban: "\f05e"; 54 | @fa-var-bank: "\f19c"; 55 | @fa-var-bar-chart: "\f080"; 56 | @fa-var-bar-chart-o: "\f080"; 57 | @fa-var-barcode: "\f02a"; 58 | @fa-var-bars: "\f0c9"; 59 | @fa-var-beer: "\f0fc"; 60 | @fa-var-behance: "\f1b4"; 61 | @fa-var-behance-square: "\f1b5"; 62 | @fa-var-bell: "\f0f3"; 63 | @fa-var-bell-o: "\f0a2"; 64 | @fa-var-bell-slash: "\f1f6"; 65 | @fa-var-bell-slash-o: "\f1f7"; 66 | @fa-var-bicycle: "\f206"; 67 | @fa-var-binoculars: "\f1e5"; 68 | @fa-var-birthday-cake: "\f1fd"; 69 | @fa-var-bitbucket: "\f171"; 70 | @fa-var-bitbucket-square: "\f172"; 71 | @fa-var-bitcoin: "\f15a"; 72 | @fa-var-bold: "\f032"; 73 | @fa-var-bolt: "\f0e7"; 74 | @fa-var-bomb: "\f1e2"; 75 | @fa-var-book: "\f02d"; 76 | @fa-var-bookmark: "\f02e"; 77 | @fa-var-bookmark-o: "\f097"; 78 | @fa-var-briefcase: "\f0b1"; 79 | @fa-var-btc: "\f15a"; 80 | @fa-var-bug: "\f188"; 81 | @fa-var-building: "\f1ad"; 82 | @fa-var-building-o: "\f0f7"; 83 | @fa-var-bullhorn: "\f0a1"; 84 | @fa-var-bullseye: "\f140"; 85 | @fa-var-bus: "\f207"; 86 | @fa-var-cab: "\f1ba"; 87 | @fa-var-calculator: "\f1ec"; 88 | @fa-var-calendar: "\f073"; 89 | @fa-var-calendar-o: "\f133"; 90 | @fa-var-camera: "\f030"; 91 | @fa-var-camera-retro: "\f083"; 92 | @fa-var-car: "\f1b9"; 93 | @fa-var-caret-down: "\f0d7"; 94 | @fa-var-caret-left: "\f0d9"; 95 | @fa-var-caret-right: "\f0da"; 96 | @fa-var-caret-square-o-down: "\f150"; 97 | @fa-var-caret-square-o-left: "\f191"; 98 | @fa-var-caret-square-o-right: "\f152"; 99 | @fa-var-caret-square-o-up: "\f151"; 100 | @fa-var-caret-up: "\f0d8"; 101 | @fa-var-cc: "\f20a"; 102 | @fa-var-cc-amex: "\f1f3"; 103 | @fa-var-cc-discover: "\f1f2"; 104 | @fa-var-cc-mastercard: "\f1f1"; 105 | @fa-var-cc-paypal: "\f1f4"; 106 | @fa-var-cc-stripe: "\f1f5"; 107 | @fa-var-cc-visa: "\f1f0"; 108 | @fa-var-certificate: "\f0a3"; 109 | @fa-var-chain: "\f0c1"; 110 | @fa-var-chain-broken: "\f127"; 111 | @fa-var-check: "\f00c"; 112 | @fa-var-check-circle: "\f058"; 113 | @fa-var-check-circle-o: "\f05d"; 114 | @fa-var-check-square: "\f14a"; 115 | @fa-var-check-square-o: "\f046"; 116 | @fa-var-chevron-circle-down: "\f13a"; 117 | @fa-var-chevron-circle-left: "\f137"; 118 | @fa-var-chevron-circle-right: "\f138"; 119 | @fa-var-chevron-circle-up: "\f139"; 120 | @fa-var-chevron-down: "\f078"; 121 | @fa-var-chevron-left: "\f053"; 122 | @fa-var-chevron-right: "\f054"; 123 | @fa-var-chevron-up: "\f077"; 124 | @fa-var-child: "\f1ae"; 125 | @fa-var-circle: "\f111"; 126 | @fa-var-circle-o: "\f10c"; 127 | @fa-var-circle-o-notch: "\f1ce"; 128 | @fa-var-circle-thin: "\f1db"; 129 | @fa-var-clipboard: "\f0ea"; 130 | @fa-var-clock-o: "\f017"; 131 | @fa-var-close: "\f00d"; 132 | @fa-var-cloud: "\f0c2"; 133 | @fa-var-cloud-download: "\f0ed"; 134 | @fa-var-cloud-upload: "\f0ee"; 135 | @fa-var-cny: "\f157"; 136 | @fa-var-code: "\f121"; 137 | @fa-var-code-fork: "\f126"; 138 | @fa-var-codepen: "\f1cb"; 139 | @fa-var-coffee: "\f0f4"; 140 | @fa-var-cog: "\f013"; 141 | @fa-var-cogs: "\f085"; 142 | @fa-var-columns: "\f0db"; 143 | @fa-var-comment: "\f075"; 144 | @fa-var-comment-o: "\f0e5"; 145 | @fa-var-comments: "\f086"; 146 | @fa-var-comments-o: "\f0e6"; 147 | @fa-var-compass: "\f14e"; 148 | @fa-var-compress: "\f066"; 149 | @fa-var-copy: "\f0c5"; 150 | @fa-var-copyright: "\f1f9"; 151 | @fa-var-credit-card: "\f09d"; 152 | @fa-var-crop: "\f125"; 153 | @fa-var-crosshairs: "\f05b"; 154 | @fa-var-css3: "\f13c"; 155 | @fa-var-cube: "\f1b2"; 156 | @fa-var-cubes: "\f1b3"; 157 | @fa-var-cut: "\f0c4"; 158 | @fa-var-cutlery: "\f0f5"; 159 | @fa-var-dashboard: "\f0e4"; 160 | @fa-var-database: "\f1c0"; 161 | @fa-var-dedent: "\f03b"; 162 | @fa-var-delicious: "\f1a5"; 163 | @fa-var-desktop: "\f108"; 164 | @fa-var-deviantart: "\f1bd"; 165 | @fa-var-digg: "\f1a6"; 166 | @fa-var-dollar: "\f155"; 167 | @fa-var-dot-circle-o: "\f192"; 168 | @fa-var-download: "\f019"; 169 | @fa-var-dribbble: "\f17d"; 170 | @fa-var-dropbox: "\f16b"; 171 | @fa-var-drupal: "\f1a9"; 172 | @fa-var-edit: "\f044"; 173 | @fa-var-eject: "\f052"; 174 | @fa-var-ellipsis-h: "\f141"; 175 | @fa-var-ellipsis-v: "\f142"; 176 | @fa-var-empire: "\f1d1"; 177 | @fa-var-envelope: "\f0e0"; 178 | @fa-var-envelope-o: "\f003"; 179 | @fa-var-envelope-square: "\f199"; 180 | @fa-var-eraser: "\f12d"; 181 | @fa-var-eur: "\f153"; 182 | @fa-var-euro: "\f153"; 183 | @fa-var-exchange: "\f0ec"; 184 | @fa-var-exclamation: "\f12a"; 185 | @fa-var-exclamation-circle: "\f06a"; 186 | @fa-var-exclamation-triangle: "\f071"; 187 | @fa-var-expand: "\f065"; 188 | @fa-var-external-link: "\f08e"; 189 | @fa-var-external-link-square: "\f14c"; 190 | @fa-var-eye: "\f06e"; 191 | @fa-var-eye-slash: "\f070"; 192 | @fa-var-eyedropper: "\f1fb"; 193 | @fa-var-facebook: "\f09a"; 194 | @fa-var-facebook-square: "\f082"; 195 | @fa-var-fast-backward: "\f049"; 196 | @fa-var-fast-forward: "\f050"; 197 | @fa-var-fax: "\f1ac"; 198 | @fa-var-female: "\f182"; 199 | @fa-var-fighter-jet: "\f0fb"; 200 | @fa-var-file: "\f15b"; 201 | @fa-var-file-archive-o: "\f1c6"; 202 | @fa-var-file-audio-o: "\f1c7"; 203 | @fa-var-file-code-o: "\f1c9"; 204 | @fa-var-file-excel-o: "\f1c3"; 205 | @fa-var-file-image-o: "\f1c5"; 206 | @fa-var-file-movie-o: "\f1c8"; 207 | @fa-var-file-o: "\f016"; 208 | @fa-var-file-pdf-o: "\f1c1"; 209 | @fa-var-file-photo-o: "\f1c5"; 210 | @fa-var-file-picture-o: "\f1c5"; 211 | @fa-var-file-powerpoint-o: "\f1c4"; 212 | @fa-var-file-sound-o: "\f1c7"; 213 | @fa-var-file-text: "\f15c"; 214 | @fa-var-file-text-o: "\f0f6"; 215 | @fa-var-file-video-o: "\f1c8"; 216 | @fa-var-file-word-o: "\f1c2"; 217 | @fa-var-file-zip-o: "\f1c6"; 218 | @fa-var-files-o: "\f0c5"; 219 | @fa-var-film: "\f008"; 220 | @fa-var-filter: "\f0b0"; 221 | @fa-var-fire: "\f06d"; 222 | @fa-var-fire-extinguisher: "\f134"; 223 | @fa-var-flag: "\f024"; 224 | @fa-var-flag-checkered: "\f11e"; 225 | @fa-var-flag-o: "\f11d"; 226 | @fa-var-flash: "\f0e7"; 227 | @fa-var-flask: "\f0c3"; 228 | @fa-var-flickr: "\f16e"; 229 | @fa-var-floppy-o: "\f0c7"; 230 | @fa-var-folder: "\f07b"; 231 | @fa-var-folder-o: "\f114"; 232 | @fa-var-folder-open: "\f07c"; 233 | @fa-var-folder-open-o: "\f115"; 234 | @fa-var-font: "\f031"; 235 | @fa-var-forward: "\f04e"; 236 | @fa-var-foursquare: "\f180"; 237 | @fa-var-frown-o: "\f119"; 238 | @fa-var-futbol-o: "\f1e3"; 239 | @fa-var-gamepad: "\f11b"; 240 | @fa-var-gavel: "\f0e3"; 241 | @fa-var-gbp: "\f154"; 242 | @fa-var-ge: "\f1d1"; 243 | @fa-var-gear: "\f013"; 244 | @fa-var-gears: "\f085"; 245 | @fa-var-gift: "\f06b"; 246 | @fa-var-git: "\f1d3"; 247 | @fa-var-git-square: "\f1d2"; 248 | @fa-var-github: "\f09b"; 249 | @fa-var-github-alt: "\f113"; 250 | @fa-var-github-square: "\f092"; 251 | @fa-var-gittip: "\f184"; 252 | @fa-var-glass: "\f000"; 253 | @fa-var-globe: "\f0ac"; 254 | @fa-var-google: "\f1a0"; 255 | @fa-var-google-plus: "\f0d5"; 256 | @fa-var-google-plus-square: "\f0d4"; 257 | @fa-var-google-wallet: "\f1ee"; 258 | @fa-var-graduation-cap: "\f19d"; 259 | @fa-var-group: "\f0c0"; 260 | @fa-var-h-square: "\f0fd"; 261 | @fa-var-hacker-news: "\f1d4"; 262 | @fa-var-hand-o-down: "\f0a7"; 263 | @fa-var-hand-o-left: "\f0a5"; 264 | @fa-var-hand-o-right: "\f0a4"; 265 | @fa-var-hand-o-up: "\f0a6"; 266 | @fa-var-hdd-o: "\f0a0"; 267 | @fa-var-header: "\f1dc"; 268 | @fa-var-headphones: "\f025"; 269 | @fa-var-heart: "\f004"; 270 | @fa-var-heart-o: "\f08a"; 271 | @fa-var-history: "\f1da"; 272 | @fa-var-home: "\f015"; 273 | @fa-var-hospital-o: "\f0f8"; 274 | @fa-var-html5: "\f13b"; 275 | @fa-var-ils: "\f20b"; 276 | @fa-var-image: "\f03e"; 277 | @fa-var-inbox: "\f01c"; 278 | @fa-var-indent: "\f03c"; 279 | @fa-var-info: "\f129"; 280 | @fa-var-info-circle: "\f05a"; 281 | @fa-var-inr: "\f156"; 282 | @fa-var-instagram: "\f16d"; 283 | @fa-var-institution: "\f19c"; 284 | @fa-var-ioxhost: "\f208"; 285 | @fa-var-italic: "\f033"; 286 | @fa-var-joomla: "\f1aa"; 287 | @fa-var-jpy: "\f157"; 288 | @fa-var-jsfiddle: "\f1cc"; 289 | @fa-var-key: "\f084"; 290 | @fa-var-keyboard-o: "\f11c"; 291 | @fa-var-krw: "\f159"; 292 | @fa-var-language: "\f1ab"; 293 | @fa-var-laptop: "\f109"; 294 | @fa-var-lastfm: "\f202"; 295 | @fa-var-lastfm-square: "\f203"; 296 | @fa-var-leaf: "\f06c"; 297 | @fa-var-legal: "\f0e3"; 298 | @fa-var-lemon-o: "\f094"; 299 | @fa-var-level-down: "\f149"; 300 | @fa-var-level-up: "\f148"; 301 | @fa-var-life-bouy: "\f1cd"; 302 | @fa-var-life-buoy: "\f1cd"; 303 | @fa-var-life-ring: "\f1cd"; 304 | @fa-var-life-saver: "\f1cd"; 305 | @fa-var-lightbulb-o: "\f0eb"; 306 | @fa-var-line-chart: "\f201"; 307 | @fa-var-link: "\f0c1"; 308 | @fa-var-linkedin: "\f0e1"; 309 | @fa-var-linkedin-square: "\f08c"; 310 | @fa-var-linux: "\f17c"; 311 | @fa-var-list: "\f03a"; 312 | @fa-var-list-alt: "\f022"; 313 | @fa-var-list-ol: "\f0cb"; 314 | @fa-var-list-ul: "\f0ca"; 315 | @fa-var-location-arrow: "\f124"; 316 | @fa-var-lock: "\f023"; 317 | @fa-var-long-arrow-down: "\f175"; 318 | @fa-var-long-arrow-left: "\f177"; 319 | @fa-var-long-arrow-right: "\f178"; 320 | @fa-var-long-arrow-up: "\f176"; 321 | @fa-var-magic: "\f0d0"; 322 | @fa-var-magnet: "\f076"; 323 | @fa-var-mail-forward: "\f064"; 324 | @fa-var-mail-reply: "\f112"; 325 | @fa-var-mail-reply-all: "\f122"; 326 | @fa-var-male: "\f183"; 327 | @fa-var-map-marker: "\f041"; 328 | @fa-var-maxcdn: "\f136"; 329 | @fa-var-meanpath: "\f20c"; 330 | @fa-var-medkit: "\f0fa"; 331 | @fa-var-meh-o: "\f11a"; 332 | @fa-var-microphone: "\f130"; 333 | @fa-var-microphone-slash: "\f131"; 334 | @fa-var-minus: "\f068"; 335 | @fa-var-minus-circle: "\f056"; 336 | @fa-var-minus-square: "\f146"; 337 | @fa-var-minus-square-o: "\f147"; 338 | @fa-var-mobile: "\f10b"; 339 | @fa-var-mobile-phone: "\f10b"; 340 | @fa-var-money: "\f0d6"; 341 | @fa-var-moon-o: "\f186"; 342 | @fa-var-mortar-board: "\f19d"; 343 | @fa-var-music: "\f001"; 344 | @fa-var-navicon: "\f0c9"; 345 | @fa-var-newspaper-o: "\f1ea"; 346 | @fa-var-openid: "\f19b"; 347 | @fa-var-outdent: "\f03b"; 348 | @fa-var-pagelines: "\f18c"; 349 | @fa-var-paint-brush: "\f1fc"; 350 | @fa-var-paper-plane: "\f1d8"; 351 | @fa-var-paper-plane-o: "\f1d9"; 352 | @fa-var-paperclip: "\f0c6"; 353 | @fa-var-paragraph: "\f1dd"; 354 | @fa-var-paste: "\f0ea"; 355 | @fa-var-pause: "\f04c"; 356 | @fa-var-paw: "\f1b0"; 357 | @fa-var-paypal: "\f1ed"; 358 | @fa-var-pencil: "\f040"; 359 | @fa-var-pencil-square: "\f14b"; 360 | @fa-var-pencil-square-o: "\f044"; 361 | @fa-var-phone: "\f095"; 362 | @fa-var-phone-square: "\f098"; 363 | @fa-var-photo: "\f03e"; 364 | @fa-var-picture-o: "\f03e"; 365 | @fa-var-pie-chart: "\f200"; 366 | @fa-var-pied-piper: "\f1a7"; 367 | @fa-var-pied-piper-alt: "\f1a8"; 368 | @fa-var-pinterest: "\f0d2"; 369 | @fa-var-pinterest-square: "\f0d3"; 370 | @fa-var-plane: "\f072"; 371 | @fa-var-play: "\f04b"; 372 | @fa-var-play-circle: "\f144"; 373 | @fa-var-play-circle-o: "\f01d"; 374 | @fa-var-plug: "\f1e6"; 375 | @fa-var-plus: "\f067"; 376 | @fa-var-plus-circle: "\f055"; 377 | @fa-var-plus-square: "\f0fe"; 378 | @fa-var-plus-square-o: "\f196"; 379 | @fa-var-power-off: "\f011"; 380 | @fa-var-print: "\f02f"; 381 | @fa-var-puzzle-piece: "\f12e"; 382 | @fa-var-qq: "\f1d6"; 383 | @fa-var-qrcode: "\f029"; 384 | @fa-var-question: "\f128"; 385 | @fa-var-question-circle: "\f059"; 386 | @fa-var-quote-left: "\f10d"; 387 | @fa-var-quote-right: "\f10e"; 388 | @fa-var-ra: "\f1d0"; 389 | @fa-var-random: "\f074"; 390 | @fa-var-rebel: "\f1d0"; 391 | @fa-var-recycle: "\f1b8"; 392 | @fa-var-reddit: "\f1a1"; 393 | @fa-var-reddit-square: "\f1a2"; 394 | @fa-var-refresh: "\f021"; 395 | @fa-var-remove: "\f00d"; 396 | @fa-var-renren: "\f18b"; 397 | @fa-var-reorder: "\f0c9"; 398 | @fa-var-repeat: "\f01e"; 399 | @fa-var-reply: "\f112"; 400 | @fa-var-reply-all: "\f122"; 401 | @fa-var-retweet: "\f079"; 402 | @fa-var-rmb: "\f157"; 403 | @fa-var-road: "\f018"; 404 | @fa-var-rocket: "\f135"; 405 | @fa-var-rotate-left: "\f0e2"; 406 | @fa-var-rotate-right: "\f01e"; 407 | @fa-var-rouble: "\f158"; 408 | @fa-var-rss: "\f09e"; 409 | @fa-var-rss-square: "\f143"; 410 | @fa-var-rub: "\f158"; 411 | @fa-var-ruble: "\f158"; 412 | @fa-var-rupee: "\f156"; 413 | @fa-var-save: "\f0c7"; 414 | @fa-var-scissors: "\f0c4"; 415 | @fa-var-search: "\f002"; 416 | @fa-var-search-minus: "\f010"; 417 | @fa-var-search-plus: "\f00e"; 418 | @fa-var-send: "\f1d8"; 419 | @fa-var-send-o: "\f1d9"; 420 | @fa-var-share: "\f064"; 421 | @fa-var-share-alt: "\f1e0"; 422 | @fa-var-share-alt-square: "\f1e1"; 423 | @fa-var-share-square: "\f14d"; 424 | @fa-var-share-square-o: "\f045"; 425 | @fa-var-shekel: "\f20b"; 426 | @fa-var-sheqel: "\f20b"; 427 | @fa-var-shield: "\f132"; 428 | @fa-var-shopping-cart: "\f07a"; 429 | @fa-var-sign-in: "\f090"; 430 | @fa-var-sign-out: "\f08b"; 431 | @fa-var-signal: "\f012"; 432 | @fa-var-sitemap: "\f0e8"; 433 | @fa-var-skype: "\f17e"; 434 | @fa-var-slack: "\f198"; 435 | @fa-var-sliders: "\f1de"; 436 | @fa-var-slideshare: "\f1e7"; 437 | @fa-var-smile-o: "\f118"; 438 | @fa-var-soccer-ball-o: "\f1e3"; 439 | @fa-var-sort: "\f0dc"; 440 | @fa-var-sort-alpha-asc: "\f15d"; 441 | @fa-var-sort-alpha-desc: "\f15e"; 442 | @fa-var-sort-amount-asc: "\f160"; 443 | @fa-var-sort-amount-desc: "\f161"; 444 | @fa-var-sort-asc: "\f0de"; 445 | @fa-var-sort-desc: "\f0dd"; 446 | @fa-var-sort-down: "\f0dd"; 447 | @fa-var-sort-numeric-asc: "\f162"; 448 | @fa-var-sort-numeric-desc: "\f163"; 449 | @fa-var-sort-up: "\f0de"; 450 | @fa-var-soundcloud: "\f1be"; 451 | @fa-var-space-shuttle: "\f197"; 452 | @fa-var-spinner: "\f110"; 453 | @fa-var-spoon: "\f1b1"; 454 | @fa-var-spotify: "\f1bc"; 455 | @fa-var-square: "\f0c8"; 456 | @fa-var-square-o: "\f096"; 457 | @fa-var-stack-exchange: "\f18d"; 458 | @fa-var-stack-overflow: "\f16c"; 459 | @fa-var-star: "\f005"; 460 | @fa-var-star-half: "\f089"; 461 | @fa-var-star-half-empty: "\f123"; 462 | @fa-var-star-half-full: "\f123"; 463 | @fa-var-star-half-o: "\f123"; 464 | @fa-var-star-o: "\f006"; 465 | @fa-var-steam: "\f1b6"; 466 | @fa-var-steam-square: "\f1b7"; 467 | @fa-var-step-backward: "\f048"; 468 | @fa-var-step-forward: "\f051"; 469 | @fa-var-stethoscope: "\f0f1"; 470 | @fa-var-stop: "\f04d"; 471 | @fa-var-strikethrough: "\f0cc"; 472 | @fa-var-stumbleupon: "\f1a4"; 473 | @fa-var-stumbleupon-circle: "\f1a3"; 474 | @fa-var-subscript: "\f12c"; 475 | @fa-var-suitcase: "\f0f2"; 476 | @fa-var-sun-o: "\f185"; 477 | @fa-var-superscript: "\f12b"; 478 | @fa-var-support: "\f1cd"; 479 | @fa-var-table: "\f0ce"; 480 | @fa-var-tablet: "\f10a"; 481 | @fa-var-tachometer: "\f0e4"; 482 | @fa-var-tag: "\f02b"; 483 | @fa-var-tags: "\f02c"; 484 | @fa-var-tasks: "\f0ae"; 485 | @fa-var-taxi: "\f1ba"; 486 | @fa-var-tencent-weibo: "\f1d5"; 487 | @fa-var-terminal: "\f120"; 488 | @fa-var-text-height: "\f034"; 489 | @fa-var-text-width: "\f035"; 490 | @fa-var-th: "\f00a"; 491 | @fa-var-th-large: "\f009"; 492 | @fa-var-th-list: "\f00b"; 493 | @fa-var-thumb-tack: "\f08d"; 494 | @fa-var-thumbs-down: "\f165"; 495 | @fa-var-thumbs-o-down: "\f088"; 496 | @fa-var-thumbs-o-up: "\f087"; 497 | @fa-var-thumbs-up: "\f164"; 498 | @fa-var-ticket: "\f145"; 499 | @fa-var-times: "\f00d"; 500 | @fa-var-times-circle: "\f057"; 501 | @fa-var-times-circle-o: "\f05c"; 502 | @fa-var-tint: "\f043"; 503 | @fa-var-toggle-down: "\f150"; 504 | @fa-var-toggle-left: "\f191"; 505 | @fa-var-toggle-off: "\f204"; 506 | @fa-var-toggle-on: "\f205"; 507 | @fa-var-toggle-right: "\f152"; 508 | @fa-var-toggle-up: "\f151"; 509 | @fa-var-trash: "\f1f8"; 510 | @fa-var-trash-o: "\f014"; 511 | @fa-var-tree: "\f1bb"; 512 | @fa-var-trello: "\f181"; 513 | @fa-var-trophy: "\f091"; 514 | @fa-var-truck: "\f0d1"; 515 | @fa-var-try: "\f195"; 516 | @fa-var-tty: "\f1e4"; 517 | @fa-var-tumblr: "\f173"; 518 | @fa-var-tumblr-square: "\f174"; 519 | @fa-var-turkish-lira: "\f195"; 520 | @fa-var-twitch: "\f1e8"; 521 | @fa-var-twitter: "\f099"; 522 | @fa-var-twitter-square: "\f081"; 523 | @fa-var-umbrella: "\f0e9"; 524 | @fa-var-underline: "\f0cd"; 525 | @fa-var-undo: "\f0e2"; 526 | @fa-var-university: "\f19c"; 527 | @fa-var-unlink: "\f127"; 528 | @fa-var-unlock: "\f09c"; 529 | @fa-var-unlock-alt: "\f13e"; 530 | @fa-var-unsorted: "\f0dc"; 531 | @fa-var-upload: "\f093"; 532 | @fa-var-usd: "\f155"; 533 | @fa-var-user: "\f007"; 534 | @fa-var-user-md: "\f0f0"; 535 | @fa-var-users: "\f0c0"; 536 | @fa-var-video-camera: "\f03d"; 537 | @fa-var-vimeo-square: "\f194"; 538 | @fa-var-vine: "\f1ca"; 539 | @fa-var-vk: "\f189"; 540 | @fa-var-volume-down: "\f027"; 541 | @fa-var-volume-off: "\f026"; 542 | @fa-var-volume-up: "\f028"; 543 | @fa-var-warning: "\f071"; 544 | @fa-var-wechat: "\f1d7"; 545 | @fa-var-weibo: "\f18a"; 546 | @fa-var-weixin: "\f1d7"; 547 | @fa-var-wheelchair: "\f193"; 548 | @fa-var-wifi: "\f1eb"; 549 | @fa-var-windows: "\f17a"; 550 | @fa-var-won: "\f159"; 551 | @fa-var-wordpress: "\f19a"; 552 | @fa-var-wrench: "\f0ad"; 553 | @fa-var-xing: "\f168"; 554 | @fa-var-xing-square: "\f169"; 555 | @fa-var-yahoo: "\f19e"; 556 | @fa-var-yelp: "\f1e9"; 557 | @fa-var-yen: "\f157"; 558 | @fa-var-youtube: "\f167"; 559 | @fa-var-youtube-play: "\f16a"; 560 | @fa-var-youtube-square: "\f166"; 561 | 562 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_bordered-pulled.scss: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em $fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .#{$fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_core.scss: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_fixed-width.scss: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .#{$fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_larger.scss: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .#{$fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .#{$fa-css-prefix}-2x { font-size: 2em; } 11 | .#{$fa-css-prefix}-3x { font-size: 3em; } 12 | .#{$fa-css-prefix}-4x { font-size: 4em; } 13 | .#{$fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_list.scss: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: $fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .#{$fa-css-prefix}-li { 11 | position: absolute; 12 | left: -$fa-li-width; 13 | width: $fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.#{$fa-css-prefix}-lg { 17 | left: -$fa-li-width + (4em / 14); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_mixins.scss: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | @mixin fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal 14px/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | } 12 | 13 | @mixin fa-icon-rotate($degrees, $rotation) { 14 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 15 | -webkit-transform: rotate($degrees); 16 | -ms-transform: rotate($degrees); 17 | transform: rotate($degrees); 18 | } 19 | 20 | @mixin fa-icon-flip($horiz, $vert, $rotation) { 21 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 22 | -webkit-transform: scale($horiz, $vert); 23 | -ms-transform: scale($horiz, $vert); 24 | transform: scale($horiz, $vert); 25 | } 26 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_path.scss: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?v=#{$fa-version}'); 7 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?#iefix&v=#{$fa-version}') format('embedded-opentype'), 8 | url('#{$fa-font-path}/fontawesome-webfont.woff?v=#{$fa-version}') format('woff'), 9 | url('#{$fa-font-path}/fontawesome-webfont.ttf?v=#{$fa-version}') format('truetype'), 10 | url('#{$fa-font-path}/fontawesome-webfont.svg?v=#{$fa-version}#fontawesomeregular') format('svg'); 11 | //src: url('#{$fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 12 | font-weight: normal; 13 | font-style: normal; 14 | } 15 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_rotated-flipped.scss: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-rotate-90 { @include fa-icon-rotate(90deg, 1); } 5 | .#{$fa-css-prefix}-rotate-180 { @include fa-icon-rotate(180deg, 2); } 6 | .#{$fa-css-prefix}-rotate-270 { @include fa-icon-rotate(270deg, 3); } 7 | 8 | .#{$fa-css-prefix}-flip-horizontal { @include fa-icon-flip(-1, 1, 0); } 9 | .#{$fa-css-prefix}-flip-vertical { @include fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .#{$fa-css-prefix}-rotate-90, 15 | :root .#{$fa-css-prefix}-rotate-180, 16 | :root .#{$fa-css-prefix}-rotate-270, 17 | :root .#{$fa-css-prefix}-flip-horizontal, 18 | :root .#{$fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_spinning.scss: -------------------------------------------------------------------------------- 1 | // Spinning Icons 2 | // -------------------------- 3 | 4 | .#{$fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | @-webkit-keyframes fa-spin { 10 | 0% { 11 | -webkit-transform: rotate(0deg); 12 | transform: rotate(0deg); 13 | } 14 | 100% { 15 | -webkit-transform: rotate(359deg); 16 | transform: rotate(359deg); 17 | } 18 | } 19 | 20 | @keyframes fa-spin { 21 | 0% { 22 | -webkit-transform: rotate(0deg); 23 | transform: rotate(0deg); 24 | } 25 | 100% { 26 | -webkit-transform: rotate(359deg); 27 | transform: rotate(359deg); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_stacked.scss: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .#{$fa-css-prefix}-stack-1x, .#{$fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .#{$fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .#{$fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .#{$fa-css-prefix}-inverse { color: $fa-inverse; } 21 | -------------------------------------------------------------------------------- /static/font-awesome/scss/_variables.scss: -------------------------------------------------------------------------------- 1 | // Variables 2 | // -------------------------- 3 | 4 | $fa-font-path: "../fonts" !default; 5 | //$fa-font-path: "//netdna.bootstrapcdn.com/font-awesome/4.2.0/fonts" !default; // for referencing Bootstrap CDN font files directly 6 | $fa-css-prefix: fa !default; 7 | $fa-version: "4.2.0" !default; 8 | $fa-border-color: #eee !default; 9 | $fa-inverse: #fff !default; 10 | $fa-li-width: (30em / 14) !default; 11 | 12 | $fa-var-adjust: "\f042"; 13 | $fa-var-adn: "\f170"; 14 | $fa-var-align-center: "\f037"; 15 | $fa-var-align-justify: "\f039"; 16 | $fa-var-align-left: "\f036"; 17 | $fa-var-align-right: "\f038"; 18 | $fa-var-ambulance: "\f0f9"; 19 | $fa-var-anchor: "\f13d"; 20 | $fa-var-android: "\f17b"; 21 | $fa-var-angellist: "\f209"; 22 | $fa-var-angle-double-down: "\f103"; 23 | $fa-var-angle-double-left: "\f100"; 24 | $fa-var-angle-double-right: "\f101"; 25 | $fa-var-angle-double-up: "\f102"; 26 | $fa-var-angle-down: "\f107"; 27 | $fa-var-angle-left: "\f104"; 28 | $fa-var-angle-right: "\f105"; 29 | $fa-var-angle-up: "\f106"; 30 | $fa-var-apple: "\f179"; 31 | $fa-var-archive: "\f187"; 32 | $fa-var-area-chart: "\f1fe"; 33 | $fa-var-arrow-circle-down: "\f0ab"; 34 | $fa-var-arrow-circle-left: "\f0a8"; 35 | $fa-var-arrow-circle-o-down: "\f01a"; 36 | $fa-var-arrow-circle-o-left: "\f190"; 37 | $fa-var-arrow-circle-o-right: "\f18e"; 38 | $fa-var-arrow-circle-o-up: "\f01b"; 39 | $fa-var-arrow-circle-right: "\f0a9"; 40 | $fa-var-arrow-circle-up: "\f0aa"; 41 | $fa-var-arrow-down: "\f063"; 42 | $fa-var-arrow-left: "\f060"; 43 | $fa-var-arrow-right: "\f061"; 44 | $fa-var-arrow-up: "\f062"; 45 | $fa-var-arrows: "\f047"; 46 | $fa-var-arrows-alt: "\f0b2"; 47 | $fa-var-arrows-h: "\f07e"; 48 | $fa-var-arrows-v: "\f07d"; 49 | $fa-var-asterisk: "\f069"; 50 | $fa-var-at: "\f1fa"; 51 | $fa-var-automobile: "\f1b9"; 52 | $fa-var-backward: "\f04a"; 53 | $fa-var-ban: "\f05e"; 54 | $fa-var-bank: "\f19c"; 55 | $fa-var-bar-chart: "\f080"; 56 | $fa-var-bar-chart-o: "\f080"; 57 | $fa-var-barcode: "\f02a"; 58 | $fa-var-bars: "\f0c9"; 59 | $fa-var-beer: "\f0fc"; 60 | $fa-var-behance: "\f1b4"; 61 | $fa-var-behance-square: "\f1b5"; 62 | $fa-var-bell: "\f0f3"; 63 | $fa-var-bell-o: "\f0a2"; 64 | $fa-var-bell-slash: "\f1f6"; 65 | $fa-var-bell-slash-o: "\f1f7"; 66 | $fa-var-bicycle: "\f206"; 67 | $fa-var-binoculars: "\f1e5"; 68 | $fa-var-birthday-cake: "\f1fd"; 69 | $fa-var-bitbucket: "\f171"; 70 | $fa-var-bitbucket-square: "\f172"; 71 | $fa-var-bitcoin: "\f15a"; 72 | $fa-var-bold: "\f032"; 73 | $fa-var-bolt: "\f0e7"; 74 | $fa-var-bomb: "\f1e2"; 75 | $fa-var-book: "\f02d"; 76 | $fa-var-bookmark: "\f02e"; 77 | $fa-var-bookmark-o: "\f097"; 78 | $fa-var-briefcase: "\f0b1"; 79 | $fa-var-btc: "\f15a"; 80 | $fa-var-bug: "\f188"; 81 | $fa-var-building: "\f1ad"; 82 | $fa-var-building-o: "\f0f7"; 83 | $fa-var-bullhorn: "\f0a1"; 84 | $fa-var-bullseye: "\f140"; 85 | $fa-var-bus: "\f207"; 86 | $fa-var-cab: "\f1ba"; 87 | $fa-var-calculator: "\f1ec"; 88 | $fa-var-calendar: "\f073"; 89 | $fa-var-calendar-o: "\f133"; 90 | $fa-var-camera: "\f030"; 91 | $fa-var-camera-retro: "\f083"; 92 | $fa-var-car: "\f1b9"; 93 | $fa-var-caret-down: "\f0d7"; 94 | $fa-var-caret-left: "\f0d9"; 95 | $fa-var-caret-right: "\f0da"; 96 | $fa-var-caret-square-o-down: "\f150"; 97 | $fa-var-caret-square-o-left: "\f191"; 98 | $fa-var-caret-square-o-right: "\f152"; 99 | $fa-var-caret-square-o-up: "\f151"; 100 | $fa-var-caret-up: "\f0d8"; 101 | $fa-var-cc: "\f20a"; 102 | $fa-var-cc-amex: "\f1f3"; 103 | $fa-var-cc-discover: "\f1f2"; 104 | $fa-var-cc-mastercard: "\f1f1"; 105 | $fa-var-cc-paypal: "\f1f4"; 106 | $fa-var-cc-stripe: "\f1f5"; 107 | $fa-var-cc-visa: "\f1f0"; 108 | $fa-var-certificate: "\f0a3"; 109 | $fa-var-chain: "\f0c1"; 110 | $fa-var-chain-broken: "\f127"; 111 | $fa-var-check: "\f00c"; 112 | $fa-var-check-circle: "\f058"; 113 | $fa-var-check-circle-o: "\f05d"; 114 | $fa-var-check-square: "\f14a"; 115 | $fa-var-check-square-o: "\f046"; 116 | $fa-var-chevron-circle-down: "\f13a"; 117 | $fa-var-chevron-circle-left: "\f137"; 118 | $fa-var-chevron-circle-right: "\f138"; 119 | $fa-var-chevron-circle-up: "\f139"; 120 | $fa-var-chevron-down: "\f078"; 121 | $fa-var-chevron-left: "\f053"; 122 | $fa-var-chevron-right: "\f054"; 123 | $fa-var-chevron-up: "\f077"; 124 | $fa-var-child: "\f1ae"; 125 | $fa-var-circle: "\f111"; 126 | $fa-var-circle-o: "\f10c"; 127 | $fa-var-circle-o-notch: "\f1ce"; 128 | $fa-var-circle-thin: "\f1db"; 129 | $fa-var-clipboard: "\f0ea"; 130 | $fa-var-clock-o: "\f017"; 131 | $fa-var-close: "\f00d"; 132 | $fa-var-cloud: "\f0c2"; 133 | $fa-var-cloud-download: "\f0ed"; 134 | $fa-var-cloud-upload: "\f0ee"; 135 | $fa-var-cny: "\f157"; 136 | $fa-var-code: "\f121"; 137 | $fa-var-code-fork: "\f126"; 138 | $fa-var-codepen: "\f1cb"; 139 | $fa-var-coffee: "\f0f4"; 140 | $fa-var-cog: "\f013"; 141 | $fa-var-cogs: "\f085"; 142 | $fa-var-columns: "\f0db"; 143 | $fa-var-comment: "\f075"; 144 | $fa-var-comment-o: "\f0e5"; 145 | $fa-var-comments: "\f086"; 146 | $fa-var-comments-o: "\f0e6"; 147 | $fa-var-compass: "\f14e"; 148 | $fa-var-compress: "\f066"; 149 | $fa-var-copy: "\f0c5"; 150 | $fa-var-copyright: "\f1f9"; 151 | $fa-var-credit-card: "\f09d"; 152 | $fa-var-crop: "\f125"; 153 | $fa-var-crosshairs: "\f05b"; 154 | $fa-var-css3: "\f13c"; 155 | $fa-var-cube: "\f1b2"; 156 | $fa-var-cubes: "\f1b3"; 157 | $fa-var-cut: "\f0c4"; 158 | $fa-var-cutlery: "\f0f5"; 159 | $fa-var-dashboard: "\f0e4"; 160 | $fa-var-database: "\f1c0"; 161 | $fa-var-dedent: "\f03b"; 162 | $fa-var-delicious: "\f1a5"; 163 | $fa-var-desktop: "\f108"; 164 | $fa-var-deviantart: "\f1bd"; 165 | $fa-var-digg: "\f1a6"; 166 | $fa-var-dollar: "\f155"; 167 | $fa-var-dot-circle-o: "\f192"; 168 | $fa-var-download: "\f019"; 169 | $fa-var-dribbble: "\f17d"; 170 | $fa-var-dropbox: "\f16b"; 171 | $fa-var-drupal: "\f1a9"; 172 | $fa-var-edit: "\f044"; 173 | $fa-var-eject: "\f052"; 174 | $fa-var-ellipsis-h: "\f141"; 175 | $fa-var-ellipsis-v: "\f142"; 176 | $fa-var-empire: "\f1d1"; 177 | $fa-var-envelope: "\f0e0"; 178 | $fa-var-envelope-o: "\f003"; 179 | $fa-var-envelope-square: "\f199"; 180 | $fa-var-eraser: "\f12d"; 181 | $fa-var-eur: "\f153"; 182 | $fa-var-euro: "\f153"; 183 | $fa-var-exchange: "\f0ec"; 184 | $fa-var-exclamation: "\f12a"; 185 | $fa-var-exclamation-circle: "\f06a"; 186 | $fa-var-exclamation-triangle: "\f071"; 187 | $fa-var-expand: "\f065"; 188 | $fa-var-external-link: "\f08e"; 189 | $fa-var-external-link-square: "\f14c"; 190 | $fa-var-eye: "\f06e"; 191 | $fa-var-eye-slash: "\f070"; 192 | $fa-var-eyedropper: "\f1fb"; 193 | $fa-var-facebook: "\f09a"; 194 | $fa-var-facebook-square: "\f082"; 195 | $fa-var-fast-backward: "\f049"; 196 | $fa-var-fast-forward: "\f050"; 197 | $fa-var-fax: "\f1ac"; 198 | $fa-var-female: "\f182"; 199 | $fa-var-fighter-jet: "\f0fb"; 200 | $fa-var-file: "\f15b"; 201 | $fa-var-file-archive-o: "\f1c6"; 202 | $fa-var-file-audio-o: "\f1c7"; 203 | $fa-var-file-code-o: "\f1c9"; 204 | $fa-var-file-excel-o: "\f1c3"; 205 | $fa-var-file-image-o: "\f1c5"; 206 | $fa-var-file-movie-o: "\f1c8"; 207 | $fa-var-file-o: "\f016"; 208 | $fa-var-file-pdf-o: "\f1c1"; 209 | $fa-var-file-photo-o: "\f1c5"; 210 | $fa-var-file-picture-o: "\f1c5"; 211 | $fa-var-file-powerpoint-o: "\f1c4"; 212 | $fa-var-file-sound-o: "\f1c7"; 213 | $fa-var-file-text: "\f15c"; 214 | $fa-var-file-text-o: "\f0f6"; 215 | $fa-var-file-video-o: "\f1c8"; 216 | $fa-var-file-word-o: "\f1c2"; 217 | $fa-var-file-zip-o: "\f1c6"; 218 | $fa-var-files-o: "\f0c5"; 219 | $fa-var-film: "\f008"; 220 | $fa-var-filter: "\f0b0"; 221 | $fa-var-fire: "\f06d"; 222 | $fa-var-fire-extinguisher: "\f134"; 223 | $fa-var-flag: "\f024"; 224 | $fa-var-flag-checkered: "\f11e"; 225 | $fa-var-flag-o: "\f11d"; 226 | $fa-var-flash: "\f0e7"; 227 | $fa-var-flask: "\f0c3"; 228 | $fa-var-flickr: "\f16e"; 229 | $fa-var-floppy-o: "\f0c7"; 230 | $fa-var-folder: "\f07b"; 231 | $fa-var-folder-o: "\f114"; 232 | $fa-var-folder-open: "\f07c"; 233 | $fa-var-folder-open-o: "\f115"; 234 | $fa-var-font: "\f031"; 235 | $fa-var-forward: "\f04e"; 236 | $fa-var-foursquare: "\f180"; 237 | $fa-var-frown-o: "\f119"; 238 | $fa-var-futbol-o: "\f1e3"; 239 | $fa-var-gamepad: "\f11b"; 240 | $fa-var-gavel: "\f0e3"; 241 | $fa-var-gbp: "\f154"; 242 | $fa-var-ge: "\f1d1"; 243 | $fa-var-gear: "\f013"; 244 | $fa-var-gears: "\f085"; 245 | $fa-var-gift: "\f06b"; 246 | $fa-var-git: "\f1d3"; 247 | $fa-var-git-square: "\f1d2"; 248 | $fa-var-github: "\f09b"; 249 | $fa-var-github-alt: "\f113"; 250 | $fa-var-github-square: "\f092"; 251 | $fa-var-gittip: "\f184"; 252 | $fa-var-glass: "\f000"; 253 | $fa-var-globe: "\f0ac"; 254 | $fa-var-google: "\f1a0"; 255 | $fa-var-google-plus: "\f0d5"; 256 | $fa-var-google-plus-square: "\f0d4"; 257 | $fa-var-google-wallet: "\f1ee"; 258 | $fa-var-graduation-cap: "\f19d"; 259 | $fa-var-group: "\f0c0"; 260 | $fa-var-h-square: "\f0fd"; 261 | $fa-var-hacker-news: "\f1d4"; 262 | $fa-var-hand-o-down: "\f0a7"; 263 | $fa-var-hand-o-left: "\f0a5"; 264 | $fa-var-hand-o-right: "\f0a4"; 265 | $fa-var-hand-o-up: "\f0a6"; 266 | $fa-var-hdd-o: "\f0a0"; 267 | $fa-var-header: "\f1dc"; 268 | $fa-var-headphones: "\f025"; 269 | $fa-var-heart: "\f004"; 270 | $fa-var-heart-o: "\f08a"; 271 | $fa-var-history: "\f1da"; 272 | $fa-var-home: "\f015"; 273 | $fa-var-hospital-o: "\f0f8"; 274 | $fa-var-html5: "\f13b"; 275 | $fa-var-ils: "\f20b"; 276 | $fa-var-image: "\f03e"; 277 | $fa-var-inbox: "\f01c"; 278 | $fa-var-indent: "\f03c"; 279 | $fa-var-info: "\f129"; 280 | $fa-var-info-circle: "\f05a"; 281 | $fa-var-inr: "\f156"; 282 | $fa-var-instagram: "\f16d"; 283 | $fa-var-institution: "\f19c"; 284 | $fa-var-ioxhost: "\f208"; 285 | $fa-var-italic: "\f033"; 286 | $fa-var-joomla: "\f1aa"; 287 | $fa-var-jpy: "\f157"; 288 | $fa-var-jsfiddle: "\f1cc"; 289 | $fa-var-key: "\f084"; 290 | $fa-var-keyboard-o: "\f11c"; 291 | $fa-var-krw: "\f159"; 292 | $fa-var-language: "\f1ab"; 293 | $fa-var-laptop: "\f109"; 294 | $fa-var-lastfm: "\f202"; 295 | $fa-var-lastfm-square: "\f203"; 296 | $fa-var-leaf: "\f06c"; 297 | $fa-var-legal: "\f0e3"; 298 | $fa-var-lemon-o: "\f094"; 299 | $fa-var-level-down: "\f149"; 300 | $fa-var-level-up: "\f148"; 301 | $fa-var-life-bouy: "\f1cd"; 302 | $fa-var-life-buoy: "\f1cd"; 303 | $fa-var-life-ring: "\f1cd"; 304 | $fa-var-life-saver: "\f1cd"; 305 | $fa-var-lightbulb-o: "\f0eb"; 306 | $fa-var-line-chart: "\f201"; 307 | $fa-var-link: "\f0c1"; 308 | $fa-var-linkedin: "\f0e1"; 309 | $fa-var-linkedin-square: "\f08c"; 310 | $fa-var-linux: "\f17c"; 311 | $fa-var-list: "\f03a"; 312 | $fa-var-list-alt: "\f022"; 313 | $fa-var-list-ol: "\f0cb"; 314 | $fa-var-list-ul: "\f0ca"; 315 | $fa-var-location-arrow: "\f124"; 316 | $fa-var-lock: "\f023"; 317 | $fa-var-long-arrow-down: "\f175"; 318 | $fa-var-long-arrow-left: "\f177"; 319 | $fa-var-long-arrow-right: "\f178"; 320 | $fa-var-long-arrow-up: "\f176"; 321 | $fa-var-magic: "\f0d0"; 322 | $fa-var-magnet: "\f076"; 323 | $fa-var-mail-forward: "\f064"; 324 | $fa-var-mail-reply: "\f112"; 325 | $fa-var-mail-reply-all: "\f122"; 326 | $fa-var-male: "\f183"; 327 | $fa-var-map-marker: "\f041"; 328 | $fa-var-maxcdn: "\f136"; 329 | $fa-var-meanpath: "\f20c"; 330 | $fa-var-medkit: "\f0fa"; 331 | $fa-var-meh-o: "\f11a"; 332 | $fa-var-microphone: "\f130"; 333 | $fa-var-microphone-slash: "\f131"; 334 | $fa-var-minus: "\f068"; 335 | $fa-var-minus-circle: "\f056"; 336 | $fa-var-minus-square: "\f146"; 337 | $fa-var-minus-square-o: "\f147"; 338 | $fa-var-mobile: "\f10b"; 339 | $fa-var-mobile-phone: "\f10b"; 340 | $fa-var-money: "\f0d6"; 341 | $fa-var-moon-o: "\f186"; 342 | $fa-var-mortar-board: "\f19d"; 343 | $fa-var-music: "\f001"; 344 | $fa-var-navicon: "\f0c9"; 345 | $fa-var-newspaper-o: "\f1ea"; 346 | $fa-var-openid: "\f19b"; 347 | $fa-var-outdent: "\f03b"; 348 | $fa-var-pagelines: "\f18c"; 349 | $fa-var-paint-brush: "\f1fc"; 350 | $fa-var-paper-plane: "\f1d8"; 351 | $fa-var-paper-plane-o: "\f1d9"; 352 | $fa-var-paperclip: "\f0c6"; 353 | $fa-var-paragraph: "\f1dd"; 354 | $fa-var-paste: "\f0ea"; 355 | $fa-var-pause: "\f04c"; 356 | $fa-var-paw: "\f1b0"; 357 | $fa-var-paypal: "\f1ed"; 358 | $fa-var-pencil: "\f040"; 359 | $fa-var-pencil-square: "\f14b"; 360 | $fa-var-pencil-square-o: "\f044"; 361 | $fa-var-phone: "\f095"; 362 | $fa-var-phone-square: "\f098"; 363 | $fa-var-photo: "\f03e"; 364 | $fa-var-picture-o: "\f03e"; 365 | $fa-var-pie-chart: "\f200"; 366 | $fa-var-pied-piper: "\f1a7"; 367 | $fa-var-pied-piper-alt: "\f1a8"; 368 | $fa-var-pinterest: "\f0d2"; 369 | $fa-var-pinterest-square: "\f0d3"; 370 | $fa-var-plane: "\f072"; 371 | $fa-var-play: "\f04b"; 372 | $fa-var-play-circle: "\f144"; 373 | $fa-var-play-circle-o: "\f01d"; 374 | $fa-var-plug: "\f1e6"; 375 | $fa-var-plus: "\f067"; 376 | $fa-var-plus-circle: "\f055"; 377 | $fa-var-plus-square: "\f0fe"; 378 | $fa-var-plus-square-o: "\f196"; 379 | $fa-var-power-off: "\f011"; 380 | $fa-var-print: "\f02f"; 381 | $fa-var-puzzle-piece: "\f12e"; 382 | $fa-var-qq: "\f1d6"; 383 | $fa-var-qrcode: "\f029"; 384 | $fa-var-question: "\f128"; 385 | $fa-var-question-circle: "\f059"; 386 | $fa-var-quote-left: "\f10d"; 387 | $fa-var-quote-right: "\f10e"; 388 | $fa-var-ra: "\f1d0"; 389 | $fa-var-random: "\f074"; 390 | $fa-var-rebel: "\f1d0"; 391 | $fa-var-recycle: "\f1b8"; 392 | $fa-var-reddit: "\f1a1"; 393 | $fa-var-reddit-square: "\f1a2"; 394 | $fa-var-refresh: "\f021"; 395 | $fa-var-remove: "\f00d"; 396 | $fa-var-renren: "\f18b"; 397 | $fa-var-reorder: "\f0c9"; 398 | $fa-var-repeat: "\f01e"; 399 | $fa-var-reply: "\f112"; 400 | $fa-var-reply-all: "\f122"; 401 | $fa-var-retweet: "\f079"; 402 | $fa-var-rmb: "\f157"; 403 | $fa-var-road: "\f018"; 404 | $fa-var-rocket: "\f135"; 405 | $fa-var-rotate-left: "\f0e2"; 406 | $fa-var-rotate-right: "\f01e"; 407 | $fa-var-rouble: "\f158"; 408 | $fa-var-rss: "\f09e"; 409 | $fa-var-rss-square: "\f143"; 410 | $fa-var-rub: "\f158"; 411 | $fa-var-ruble: "\f158"; 412 | $fa-var-rupee: "\f156"; 413 | $fa-var-save: "\f0c7"; 414 | $fa-var-scissors: "\f0c4"; 415 | $fa-var-search: "\f002"; 416 | $fa-var-search-minus: "\f010"; 417 | $fa-var-search-plus: "\f00e"; 418 | $fa-var-send: "\f1d8"; 419 | $fa-var-send-o: "\f1d9"; 420 | $fa-var-share: "\f064"; 421 | $fa-var-share-alt: "\f1e0"; 422 | $fa-var-share-alt-square: "\f1e1"; 423 | $fa-var-share-square: "\f14d"; 424 | $fa-var-share-square-o: "\f045"; 425 | $fa-var-shekel: "\f20b"; 426 | $fa-var-sheqel: "\f20b"; 427 | $fa-var-shield: "\f132"; 428 | $fa-var-shopping-cart: "\f07a"; 429 | $fa-var-sign-in: "\f090"; 430 | $fa-var-sign-out: "\f08b"; 431 | $fa-var-signal: "\f012"; 432 | $fa-var-sitemap: "\f0e8"; 433 | $fa-var-skype: "\f17e"; 434 | $fa-var-slack: "\f198"; 435 | $fa-var-sliders: "\f1de"; 436 | $fa-var-slideshare: "\f1e7"; 437 | $fa-var-smile-o: "\f118"; 438 | $fa-var-soccer-ball-o: "\f1e3"; 439 | $fa-var-sort: "\f0dc"; 440 | $fa-var-sort-alpha-asc: "\f15d"; 441 | $fa-var-sort-alpha-desc: "\f15e"; 442 | $fa-var-sort-amount-asc: "\f160"; 443 | $fa-var-sort-amount-desc: "\f161"; 444 | $fa-var-sort-asc: "\f0de"; 445 | $fa-var-sort-desc: "\f0dd"; 446 | $fa-var-sort-down: "\f0dd"; 447 | $fa-var-sort-numeric-asc: "\f162"; 448 | $fa-var-sort-numeric-desc: "\f163"; 449 | $fa-var-sort-up: "\f0de"; 450 | $fa-var-soundcloud: "\f1be"; 451 | $fa-var-space-shuttle: "\f197"; 452 | $fa-var-spinner: "\f110"; 453 | $fa-var-spoon: "\f1b1"; 454 | $fa-var-spotify: "\f1bc"; 455 | $fa-var-square: "\f0c8"; 456 | $fa-var-square-o: "\f096"; 457 | $fa-var-stack-exchange: "\f18d"; 458 | $fa-var-stack-overflow: "\f16c"; 459 | $fa-var-star: "\f005"; 460 | $fa-var-star-half: "\f089"; 461 | $fa-var-star-half-empty: "\f123"; 462 | $fa-var-star-half-full: "\f123"; 463 | $fa-var-star-half-o: "\f123"; 464 | $fa-var-star-o: "\f006"; 465 | $fa-var-steam: "\f1b6"; 466 | $fa-var-steam-square: "\f1b7"; 467 | $fa-var-step-backward: "\f048"; 468 | $fa-var-step-forward: "\f051"; 469 | $fa-var-stethoscope: "\f0f1"; 470 | $fa-var-stop: "\f04d"; 471 | $fa-var-strikethrough: "\f0cc"; 472 | $fa-var-stumbleupon: "\f1a4"; 473 | $fa-var-stumbleupon-circle: "\f1a3"; 474 | $fa-var-subscript: "\f12c"; 475 | $fa-var-suitcase: "\f0f2"; 476 | $fa-var-sun-o: "\f185"; 477 | $fa-var-superscript: "\f12b"; 478 | $fa-var-support: "\f1cd"; 479 | $fa-var-table: "\f0ce"; 480 | $fa-var-tablet: "\f10a"; 481 | $fa-var-tachometer: "\f0e4"; 482 | $fa-var-tag: "\f02b"; 483 | $fa-var-tags: "\f02c"; 484 | $fa-var-tasks: "\f0ae"; 485 | $fa-var-taxi: "\f1ba"; 486 | $fa-var-tencent-weibo: "\f1d5"; 487 | $fa-var-terminal: "\f120"; 488 | $fa-var-text-height: "\f034"; 489 | $fa-var-text-width: "\f035"; 490 | $fa-var-th: "\f00a"; 491 | $fa-var-th-large: "\f009"; 492 | $fa-var-th-list: "\f00b"; 493 | $fa-var-thumb-tack: "\f08d"; 494 | $fa-var-thumbs-down: "\f165"; 495 | $fa-var-thumbs-o-down: "\f088"; 496 | $fa-var-thumbs-o-up: "\f087"; 497 | $fa-var-thumbs-up: "\f164"; 498 | $fa-var-ticket: "\f145"; 499 | $fa-var-times: "\f00d"; 500 | $fa-var-times-circle: "\f057"; 501 | $fa-var-times-circle-o: "\f05c"; 502 | $fa-var-tint: "\f043"; 503 | $fa-var-toggle-down: "\f150"; 504 | $fa-var-toggle-left: "\f191"; 505 | $fa-var-toggle-off: "\f204"; 506 | $fa-var-toggle-on: "\f205"; 507 | $fa-var-toggle-right: "\f152"; 508 | $fa-var-toggle-up: "\f151"; 509 | $fa-var-trash: "\f1f8"; 510 | $fa-var-trash-o: "\f014"; 511 | $fa-var-tree: "\f1bb"; 512 | $fa-var-trello: "\f181"; 513 | $fa-var-trophy: "\f091"; 514 | $fa-var-truck: "\f0d1"; 515 | $fa-var-try: "\f195"; 516 | $fa-var-tty: "\f1e4"; 517 | $fa-var-tumblr: "\f173"; 518 | $fa-var-tumblr-square: "\f174"; 519 | $fa-var-turkish-lira: "\f195"; 520 | $fa-var-twitch: "\f1e8"; 521 | $fa-var-twitter: "\f099"; 522 | $fa-var-twitter-square: "\f081"; 523 | $fa-var-umbrella: "\f0e9"; 524 | $fa-var-underline: "\f0cd"; 525 | $fa-var-undo: "\f0e2"; 526 | $fa-var-university: "\f19c"; 527 | $fa-var-unlink: "\f127"; 528 | $fa-var-unlock: "\f09c"; 529 | $fa-var-unlock-alt: "\f13e"; 530 | $fa-var-unsorted: "\f0dc"; 531 | $fa-var-upload: "\f093"; 532 | $fa-var-usd: "\f155"; 533 | $fa-var-user: "\f007"; 534 | $fa-var-user-md: "\f0f0"; 535 | $fa-var-users: "\f0c0"; 536 | $fa-var-video-camera: "\f03d"; 537 | $fa-var-vimeo-square: "\f194"; 538 | $fa-var-vine: "\f1ca"; 539 | $fa-var-vk: "\f189"; 540 | $fa-var-volume-down: "\f027"; 541 | $fa-var-volume-off: "\f026"; 542 | $fa-var-volume-up: "\f028"; 543 | $fa-var-warning: "\f071"; 544 | $fa-var-wechat: "\f1d7"; 545 | $fa-var-weibo: "\f18a"; 546 | $fa-var-weixin: "\f1d7"; 547 | $fa-var-wheelchair: "\f193"; 548 | $fa-var-wifi: "\f1eb"; 549 | $fa-var-windows: "\f17a"; 550 | $fa-var-won: "\f159"; 551 | $fa-var-wordpress: "\f19a"; 552 | $fa-var-wrench: "\f0ad"; 553 | $fa-var-xing: "\f168"; 554 | $fa-var-xing-square: "\f169"; 555 | $fa-var-yahoo: "\f19e"; 556 | $fa-var-yelp: "\f1e9"; 557 | $fa-var-yen: "\f157"; 558 | $fa-var-youtube: "\f167"; 559 | $fa-var-youtube-play: "\f16a"; 560 | $fa-var-youtube-square: "\f166"; 561 | 562 | -------------------------------------------------------------------------------- /static/font-awesome/scss/font-awesome.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables"; 7 | @import "mixins"; 8 | @import "path"; 9 | @import "core"; 10 | @import "larger"; 11 | @import "fixed-width"; 12 | @import "list"; 13 | @import "bordered-pulled"; 14 | @import "spinning"; 15 | @import "rotated-flipped"; 16 | @import "stacked"; 17 | @import "icons"; 18 | -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /static/js/plugins/flot/excanvas.min.js: -------------------------------------------------------------------------------- 1 | if(!document.createElement("canvas").getContext){(function(){var z=Math;var K=z.round;var J=z.sin;var U=z.cos;var b=z.abs;var k=z.sqrt;var D=10;var F=D/2;function T(){return this.context_||(this.context_=new W(this))}var O=Array.prototype.slice;function G(i,j,m){var Z=O.call(arguments,2);return function(){return i.apply(j,Z.concat(O.call(arguments)))}}function AD(Z){return String(Z).replace(/&/g,"&").replace(/"/g,""")}function r(i){if(!i.namespaces.g_vml_){i.namespaces.add("g_vml_","urn:schemas-microsoft-com:vml","#default#VML")}if(!i.namespaces.g_o_){i.namespaces.add("g_o_","urn:schemas-microsoft-com:office:office","#default#VML")}if(!i.styleSheets.ex_canvas_){var Z=i.createStyleSheet();Z.owningElement.id="ex_canvas_";Z.cssText="canvas{display:inline-block;overflow:hidden;text-align:left;width:300px;height:150px}"}}r(document);var E={init:function(Z){if(/MSIE/.test(navigator.userAgent)&&!window.opera){var i=Z||document;i.createElement("canvas");i.attachEvent("onreadystatechange",G(this.init_,this,i))}},init_:function(m){var j=m.getElementsByTagName("canvas");for(var Z=0;Z1){j--}if(6*j<1){return i+(Z-i)*6*j}else{if(2*j<1){return Z}else{if(3*j<2){return i+(Z-i)*(2/3-j)*6}else{return i}}}}function Y(Z){var AE,p=1;Z=String(Z);if(Z.charAt(0)=="#"){AE=Z}else{if(/^rgb/.test(Z)){var m=g(Z);var AE="#",AF;for(var j=0;j<3;j++){if(m[j].indexOf("%")!=-1){AF=Math.floor(C(m[j])*255)}else{AF=Number(m[j])}AE+=I[N(AF,0,255)]}p=m[3]}else{if(/^hsl/.test(Z)){var m=g(Z);AE=c(m);p=m[3]}else{AE=B[Z]||Z}}}return{color:AE,alpha:p}}var L={style:"normal",variant:"normal",weight:"normal",size:10,family:"sans-serif"};var f={};function X(Z){if(f[Z]){return f[Z]}var m=document.createElement("div");var j=m.style;try{j.font=Z}catch(i){}return f[Z]={style:j.fontStyle||L.style,variant:j.fontVariant||L.variant,weight:j.fontWeight||L.weight,size:j.fontSize||L.size,family:j.fontFamily||L.family}}function P(j,i){var Z={};for(var AF in j){Z[AF]=j[AF]}var AE=parseFloat(i.currentStyle.fontSize),m=parseFloat(j.size);if(typeof j.size=="number"){Z.size=j.size}else{if(j.size.indexOf("px")!=-1){Z.size=m}else{if(j.size.indexOf("em")!=-1){Z.size=AE*m}else{if(j.size.indexOf("%")!=-1){Z.size=(AE/100)*m}else{if(j.size.indexOf("pt")!=-1){Z.size=m/0.75}else{Z.size=AE}}}}}Z.size*=0.981;return Z}function AA(Z){return Z.style+" "+Z.variant+" "+Z.weight+" "+Z.size+"px "+Z.family}function t(Z){switch(Z){case"butt":return"flat";case"round":return"round";case"square":default:return"square"}}function W(i){this.m_=V();this.mStack_=[];this.aStack_=[];this.currentPath_=[];this.strokeStyle="#000";this.fillStyle="#000";this.lineWidth=1;this.lineJoin="miter";this.lineCap="butt";this.miterLimit=D*1;this.globalAlpha=1;this.font="10px sans-serif";this.textAlign="left";this.textBaseline="alphabetic";this.canvas=i;var Z=i.ownerDocument.createElement("div");Z.style.width=i.clientWidth+"px";Z.style.height=i.clientHeight+"px";Z.style.overflow="hidden";Z.style.position="absolute";i.appendChild(Z);this.element_=Z;this.arcScaleX_=1;this.arcScaleY_=1;this.lineScale_=1}var M=W.prototype;M.clearRect=function(){if(this.textMeasureEl_){this.textMeasureEl_.removeNode(true);this.textMeasureEl_=null}this.element_.innerHTML=""};M.beginPath=function(){this.currentPath_=[]};M.moveTo=function(i,Z){var j=this.getCoords_(i,Z);this.currentPath_.push({type:"moveTo",x:j.x,y:j.y});this.currentX_=j.x;this.currentY_=j.y};M.lineTo=function(i,Z){var j=this.getCoords_(i,Z);this.currentPath_.push({type:"lineTo",x:j.x,y:j.y});this.currentX_=j.x;this.currentY_=j.y};M.bezierCurveTo=function(j,i,AI,AH,AG,AE){var Z=this.getCoords_(AG,AE);var AF=this.getCoords_(j,i);var m=this.getCoords_(AI,AH);e(this,AF,m,Z)};function e(Z,m,j,i){Z.currentPath_.push({type:"bezierCurveTo",cp1x:m.x,cp1y:m.y,cp2x:j.x,cp2y:j.y,x:i.x,y:i.y});Z.currentX_=i.x;Z.currentY_=i.y}M.quadraticCurveTo=function(AG,j,i,Z){var AF=this.getCoords_(AG,j);var AE=this.getCoords_(i,Z);var AH={x:this.currentX_+2/3*(AF.x-this.currentX_),y:this.currentY_+2/3*(AF.y-this.currentY_)};var m={x:AH.x+(AE.x-this.currentX_)/3,y:AH.y+(AE.y-this.currentY_)/3};e(this,AH,m,AE)};M.arc=function(AJ,AH,AI,AE,i,j){AI*=D;var AN=j?"at":"wa";var AK=AJ+U(AE)*AI-F;var AM=AH+J(AE)*AI-F;var Z=AJ+U(i)*AI-F;var AL=AH+J(i)*AI-F;if(AK==Z&&!j){AK+=0.125}var m=this.getCoords_(AJ,AH);var AG=this.getCoords_(AK,AM);var AF=this.getCoords_(Z,AL);this.currentPath_.push({type:AN,x:m.x,y:m.y,radius:AI,xStart:AG.x,yStart:AG.y,xEnd:AF.x,yEnd:AF.y})};M.rect=function(j,i,Z,m){this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath()};M.strokeRect=function(j,i,Z,m){var p=this.currentPath_;this.beginPath();this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath();this.stroke();this.currentPath_=p};M.fillRect=function(j,i,Z,m){var p=this.currentPath_;this.beginPath();this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath();this.fill();this.currentPath_=p};M.createLinearGradient=function(i,m,Z,j){var p=new v("gradient");p.x0_=i;p.y0_=m;p.x1_=Z;p.y1_=j;return p};M.createRadialGradient=function(m,AE,j,i,p,Z){var AF=new v("gradientradial");AF.x0_=m;AF.y0_=AE;AF.r0_=j;AF.x1_=i;AF.y1_=p;AF.r1_=Z;return AF};M.drawImage=function(AO,j){var AH,AF,AJ,AV,AM,AK,AQ,AX;var AI=AO.runtimeStyle.width;var AN=AO.runtimeStyle.height;AO.runtimeStyle.width="auto";AO.runtimeStyle.height="auto";var AG=AO.width;var AT=AO.height;AO.runtimeStyle.width=AI;AO.runtimeStyle.height=AN;if(arguments.length==3){AH=arguments[1];AF=arguments[2];AM=AK=0;AQ=AJ=AG;AX=AV=AT}else{if(arguments.length==5){AH=arguments[1];AF=arguments[2];AJ=arguments[3];AV=arguments[4];AM=AK=0;AQ=AG;AX=AT}else{if(arguments.length==9){AM=arguments[1];AK=arguments[2];AQ=arguments[3];AX=arguments[4];AH=arguments[5];AF=arguments[6];AJ=arguments[7];AV=arguments[8]}else{throw Error("Invalid number of arguments")}}}var AW=this.getCoords_(AH,AF);var m=AQ/2;var i=AX/2;var AU=[];var Z=10;var AE=10;AU.push(" ','","");this.element_.insertAdjacentHTML("BeforeEnd",AU.join(""))};M.stroke=function(AM){var m=10;var AN=10;var AE=5000;var AG={x:null,y:null};var AL={x:null,y:null};for(var AH=0;AHAL.x){AL.x=Z.x}if(AG.y==null||Z.yAL.y){AL.y=Z.y}}}AK.push(' ">');if(!AM){R(this,AK)}else{a(this,AK,AG,AL)}AK.push("");this.element_.insertAdjacentHTML("beforeEnd",AK.join(""))}};function R(j,AE){var i=Y(j.strokeStyle);var m=i.color;var p=i.alpha*j.globalAlpha;var Z=j.lineScale_*j.lineWidth;if(Z<1){p*=Z}AE.push("')}function a(AO,AG,Ah,AP){var AH=AO.fillStyle;var AY=AO.arcScaleX_;var AX=AO.arcScaleY_;var Z=AP.x-Ah.x;var m=AP.y-Ah.y;if(AH instanceof v){var AL=0;var Ac={x:0,y:0};var AU=0;var AK=1;if(AH.type_=="gradient"){var AJ=AH.x0_/AY;var j=AH.y0_/AX;var AI=AH.x1_/AY;var Aj=AH.y1_/AX;var Ag=AO.getCoords_(AJ,j);var Af=AO.getCoords_(AI,Aj);var AE=Af.x-Ag.x;var p=Af.y-Ag.y;AL=Math.atan2(AE,p)*180/Math.PI;if(AL<0){AL+=360}if(AL<0.000001){AL=0}}else{var Ag=AO.getCoords_(AH.x0_,AH.y0_);Ac={x:(Ag.x-Ah.x)/Z,y:(Ag.y-Ah.y)/m};Z/=AY*D;m/=AX*D;var Aa=z.max(Z,m);AU=2*AH.r0_/Aa;AK=2*AH.r1_/Aa-AU}var AS=AH.colors_;AS.sort(function(Ak,i){return Ak.offset-i.offset});var AN=AS.length;var AR=AS[0].color;var AQ=AS[AN-1].color;var AW=AS[0].alpha*AO.globalAlpha;var AV=AS[AN-1].alpha*AO.globalAlpha;var Ab=[];for(var Ae=0;Ae')}else{if(AH instanceof u){if(Z&&m){var AF=-Ah.x;var AZ=-Ah.y;AG.push("')}}else{var Ai=Y(AO.fillStyle);var AT=Ai.color;var Ad=Ai.alpha*AO.globalAlpha;AG.push('')}}}M.fill=function(){this.stroke(true)};M.closePath=function(){this.currentPath_.push({type:"close"})};M.getCoords_=function(j,i){var Z=this.m_;return{x:D*(j*Z[0][0]+i*Z[1][0]+Z[2][0])-F,y:D*(j*Z[0][1]+i*Z[1][1]+Z[2][1])-F}};M.save=function(){var Z={};Q(this,Z);this.aStack_.push(Z);this.mStack_.push(this.m_);this.m_=d(V(),this.m_)};M.restore=function(){if(this.aStack_.length){Q(this.aStack_.pop(),this);this.m_=this.mStack_.pop()}};function H(Z){return isFinite(Z[0][0])&&isFinite(Z[0][1])&&isFinite(Z[1][0])&&isFinite(Z[1][1])&&isFinite(Z[2][0])&&isFinite(Z[2][1])}function y(i,Z,j){if(!H(Z)){return }i.m_=Z;if(j){var p=Z[0][0]*Z[1][1]-Z[0][1]*Z[1][0];i.lineScale_=k(b(p))}}M.translate=function(j,i){var Z=[[1,0,0],[0,1,0],[j,i,1]];y(this,d(Z,this.m_),false)};M.rotate=function(i){var m=U(i);var j=J(i);var Z=[[m,j,0],[-j,m,0],[0,0,1]];y(this,d(Z,this.m_),false)};M.scale=function(j,i){this.arcScaleX_*=j;this.arcScaleY_*=i;var Z=[[j,0,0],[0,i,0],[0,0,1]];y(this,d(Z,this.m_),true)};M.transform=function(p,m,AF,AE,i,Z){var j=[[p,m,0],[AF,AE,0],[i,Z,1]];y(this,d(j,this.m_),true)};M.setTransform=function(AE,p,AG,AF,j,i){var Z=[[AE,p,0],[AG,AF,0],[j,i,1]];y(this,Z,true)};M.drawText_=function(AK,AI,AH,AN,AG){var AM=this.m_,AQ=1000,i=0,AP=AQ,AF={x:0,y:0},AE=[];var Z=P(X(this.font),this.element_);var j=AA(Z);var AR=this.element_.currentStyle;var p=this.textAlign.toLowerCase();switch(p){case"left":case"center":case"right":break;case"end":p=AR.direction=="ltr"?"right":"left";break;case"start":p=AR.direction=="rtl"?"right":"left";break;default:p="left"}switch(this.textBaseline){case"hanging":case"top":AF.y=Z.size/1.75;break;case"middle":break;default:case null:case"alphabetic":case"ideographic":case"bottom":AF.y=-Z.size/2.25;break}switch(p){case"right":i=AQ;AP=0.05;break;case"center":i=AP=AQ/2;break}var AO=this.getCoords_(AI+AF.x,AH+AF.y);AE.push('');if(AG){R(this,AE)}else{a(this,AE,{x:-i,y:0},{x:AP,y:Z.size})}var AL=AM[0][0].toFixed(3)+","+AM[1][0].toFixed(3)+","+AM[0][1].toFixed(3)+","+AM[1][1].toFixed(3)+",0,0";var AJ=K(AO.x/D)+","+K(AO.y/D);AE.push('','','');this.element_.insertAdjacentHTML("beforeEnd",AE.join(""))};M.fillText=function(j,Z,m,i){this.drawText_(j,Z,m,i,false)};M.strokeText=function(j,Z,m,i){this.drawText_(j,Z,m,i,true)};M.measureText=function(j){if(!this.textMeasureEl_){var Z='';this.element_.insertAdjacentHTML("beforeEnd",Z);this.textMeasureEl_=this.element_.lastChild}var i=this.element_.ownerDocument;this.textMeasureEl_.innerHTML="";this.textMeasureEl_.style.font=this.font;this.textMeasureEl_.appendChild(i.createTextNode(j));return{width:this.textMeasureEl_.offsetWidth}};M.clip=function(){};M.arcTo=function(){};M.createPattern=function(i,Z){return new u(i,Z)};function v(Z){this.type_=Z;this.x0_=0;this.y0_=0;this.r0_=0;this.x1_=0;this.y1_=0;this.r1_=0;this.colors_=[]}v.prototype.addColorStop=function(i,Z){Z=Y(Z);this.colors_.push({offset:i,color:Z.color,alpha:Z.alpha})};function u(i,Z){q(i);switch(Z){case"repeat":case null:case"":this.repetition_="repeat";break;case"repeat-x":case"repeat-y":case"no-repeat":this.repetition_=Z;break;default:n("SYNTAX_ERR")}this.src_=i.src;this.width_=i.width;this.height_=i.height}function n(Z){throw new o(Z)}function q(Z){if(!Z||Z.nodeType!=1||Z.tagName!="IMG"){n("TYPE_MISMATCH_ERR")}if(Z.readyState!="complete"){n("INVALID_STATE_ERR")}}function o(Z){this.code=this[Z];this.message=Z+": DOM Exception "+this.code}var x=o.prototype=new Error;x.INDEX_SIZE_ERR=1;x.DOMSTRING_SIZE_ERR=2;x.HIERARCHY_REQUEST_ERR=3;x.WRONG_DOCUMENT_ERR=4;x.INVALID_CHARACTER_ERR=5;x.NO_DATA_ALLOWED_ERR=6;x.NO_MODIFICATION_ALLOWED_ERR=7;x.NOT_FOUND_ERR=8;x.NOT_SUPPORTED_ERR=9;x.INUSE_ATTRIBUTE_ERR=10;x.INVALID_STATE_ERR=11;x.SYNTAX_ERR=12;x.INVALID_MODIFICATION_ERR=13;x.NAMESPACE_ERR=14;x.INVALID_ACCESS_ERR=15;x.VALIDATION_ERR=16;x.TYPE_MISMATCH_ERR=17;G_vmlCanvasManager=E;CanvasRenderingContext2D=W;CanvasGradient=v;CanvasPattern=u;DOMException=o})()}; -------------------------------------------------------------------------------- /static/js/plugins/flot/jquery.flot.resize.js: -------------------------------------------------------------------------------- 1 | /* Flot plugin for automatically redrawing plots as the placeholder resizes. 2 | 3 | Copyright (c) 2007-2013 IOLA and Ole Laursen. 4 | Licensed under the MIT license. 5 | 6 | It works by listening for changes on the placeholder div (through the jQuery 7 | resize event plugin) - if the size changes, it will redraw the plot. 8 | 9 | There are no options. If you need to disable the plugin for some plots, you 10 | can just fix the size of their placeholders. 11 | 12 | */ 13 | 14 | /* Inline dependency: 15 | * jQuery resize event - v1.1 - 3/14/2010 16 | * http://benalman.com/projects/jquery-resize-plugin/ 17 | * 18 | * Copyright (c) 2010 "Cowboy" Ben Alman 19 | * Dual licensed under the MIT and GPL licenses. 20 | * http://benalman.com/about/license/ 21 | */ 22 | 23 | (function($,h,c){var a=$([]),e=$.resize=$.extend($.resize,{}),i,k="setTimeout",j="resize",d=j+"-special-event",b="delay",f="throttleWindow";e[b]=250;e[f]=true;$.event.special[j]={setup:function(){if(!e[f]&&this[k]){return false}var l=$(this);a=a.add(l);$.data(this,d,{w:l.width(),h:l.height()});if(a.length===1){g()}},teardown:function(){if(!e[f]&&this[k]){return false}var l=$(this);a=a.not(l);l.removeData(d);if(!a.length){clearTimeout(i)}},add:function(l){if(!e[f]&&this[k]){return false}var n;function m(s,o,p){var q=$(this),r=$.data(this,d);r.w=o!==c?o:q.width();r.h=p!==c?p:q.height();n.apply(this,arguments)}if($.isFunction(l)){n=l;return m}else{n=l.handler;l.handler=m}}};function g(){i=h[k](function(){a.each(function(){var n=$(this),m=n.width(),l=n.height(),o=$.data(this,d);if(m!==o.w||l!==o.h){n.trigger(j,[o.w=m,o.h=l])}});g()},e[b])}})(jQuery,this); 24 | 25 | (function ($) { 26 | var options = { }; // no options 27 | 28 | function init(plot) { 29 | function onResize() { 30 | var placeholder = plot.getPlaceholder(); 31 | 32 | // somebody might have hidden us and we can't plot 33 | // when we don't have the dimensions 34 | if (placeholder.width() == 0 || placeholder.height() == 0) 35 | return; 36 | 37 | plot.resize(); 38 | plot.setupGrid(); 39 | plot.draw(); 40 | } 41 | 42 | function bindEvents(plot, eventHolder) { 43 | plot.getPlaceholder().resize(onResize); 44 | } 45 | 46 | function shutdown(plot, eventHolder) { 47 | plot.getPlaceholder().unbind("resize", onResize); 48 | } 49 | 50 | plot.hooks.bindEvents.push(bindEvents); 51 | plot.hooks.shutdown.push(shutdown); 52 | } 53 | 54 | $.plot.plugins.push({ 55 | init: init, 56 | options: options, 57 | name: 'resize', 58 | version: '1.0' 59 | }); 60 | })(jQuery); -------------------------------------------------------------------------------- /static/js/plugins/flot/jquery.flot.tooltip.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | * jquery.flot.tooltip 3 | * 4 | * description: easy-to-use tooltips for Flot charts 5 | * version: 0.6.2 6 | * author: Krzysztof Urbas @krzysu [myviews.pl] 7 | * website: https://github.com/krzysu/flot.tooltip 8 | * 9 | * build on 2013-09-30 10 | * released under MIT License, 2012 11 | */ 12 | (function(t){var o={tooltip:!1,tooltipOpts:{content:"%s | X: %x | Y: %y",xDateFormat:null,yDateFormat:null,shifts:{x:10,y:20},defaultTheme:!0,onHover:function(){}}},i=function(t){this.tipPosition={x:0,y:0},this.init(t)};i.prototype.init=function(o){function i(t){var o={};o.x=t.pageX,o.y=t.pageY,s.updateTooltipPosition(o)}function e(t,o,i){var e=s.getDomElement();if(i){var n;n=s.stringFormat(s.tooltipOptions.content,i),e.html(n),s.updateTooltipPosition({x:o.pageX,y:o.pageY}),e.css({left:s.tipPosition.x+s.tooltipOptions.shifts.x,top:s.tipPosition.y+s.tooltipOptions.shifts.y}).show(),"function"==typeof s.tooltipOptions.onHover&&s.tooltipOptions.onHover(i,e)}else e.hide().html("")}var s=this;o.hooks.bindEvents.push(function(o,n){s.plotOptions=o.getOptions(),s.plotOptions.tooltip!==!1&&void 0!==s.plotOptions.tooltip&&(s.tooltipOptions=s.plotOptions.tooltipOpts,s.getDomElement(),t(o.getPlaceholder()).bind("plothover",e),t(n).bind("mousemove",i))}),o.hooks.shutdown.push(function(o,s){t(o.getPlaceholder()).unbind("plothover",e),t(s).unbind("mousemove",i)})},i.prototype.getDomElement=function(){var o;return t("#flotTip").length>0?o=t("#flotTip"):(o=t("
").attr("id","flotTip"),o.appendTo("body").hide().css({position:"absolute"}),this.tooltipOptions.defaultTheme&&o.css({background:"#fff","z-index":"100",padding:"0.4em 0.6em","border-radius":"0.5em","font-size":"0.8em",border:"1px solid #111",display:"none","white-space":"nowrap"})),o},i.prototype.updateTooltipPosition=function(o){var i=t("#flotTip").outerWidth()+this.tooltipOptions.shifts.x,e=t("#flotTip").outerHeight()+this.tooltipOptions.shifts.y;o.x-t(window).scrollLeft()>t(window).innerWidth()-i&&(o.x-=i),o.y-t(window).scrollTop()>t(window).innerHeight()-e&&(o.y-=e),this.tipPosition.x=o.x,this.tipPosition.y=o.y},i.prototype.stringFormat=function(t,o){var i=/%p\.{0,1}(\d{0,})/,e=/%s/,s=/%x\.{0,1}(?:\d{0,})/,n=/%y\.{0,1}(?:\d{0,})/;return"function"==typeof t&&(t=t(o.series.label,o.series.data[o.dataIndex][0],o.series.data[o.dataIndex][1],o)),o.series.percent!==void 0&&(t=this.adjustValPrecision(i,t,o.series.percent)),o.series.label!==void 0&&(t=t.replace(e,o.series.label)),this.isTimeMode("xaxis",o)&&this.isXDateFormat(o)&&(t=t.replace(s,this.timestampToDate(o.series.data[o.dataIndex][0],this.tooltipOptions.xDateFormat))),this.isTimeMode("yaxis",o)&&this.isYDateFormat(o)&&(t=t.replace(n,this.timestampToDate(o.series.data[o.dataIndex][1],this.tooltipOptions.yDateFormat))),"number"==typeof o.series.data[o.dataIndex][0]&&(t=this.adjustValPrecision(s,t,o.series.data[o.dataIndex][0])),"number"==typeof o.series.data[o.dataIndex][1]&&(t=this.adjustValPrecision(n,t,o.series.data[o.dataIndex][1])),o.series.xaxis.tickFormatter!==void 0&&(t=t.replace(s,o.series.xaxis.tickFormatter(o.series.data[o.dataIndex][0],o.series.xaxis))),o.series.yaxis.tickFormatter!==void 0&&(t=t.replace(n,o.series.yaxis.tickFormatter(o.series.data[o.dataIndex][1],o.series.yaxis))),t},i.prototype.isTimeMode=function(t,o){return o.series[t].options.mode!==void 0&&"time"===o.series[t].options.mode},i.prototype.isXDateFormat=function(){return this.tooltipOptions.xDateFormat!==void 0&&null!==this.tooltipOptions.xDateFormat},i.prototype.isYDateFormat=function(){return this.tooltipOptions.yDateFormat!==void 0&&null!==this.tooltipOptions.yDateFormat},i.prototype.timestampToDate=function(o,i){var e=new Date(o);return t.plot.formatDate(e,i)},i.prototype.adjustValPrecision=function(t,o,i){var e,s=o.match(t);return null!==s&&""!==RegExp.$1&&(e=RegExp.$1,i=i.toFixed(e),o=o.replace(t,i)),o};var e=function(t){new i(t)};t.plot.plugins.push({init:e,options:o,name:"tooltip",version:"0.6.1"})})(jQuery); -------------------------------------------------------------------------------- /static/js/plugins/morris/morris-data.js: -------------------------------------------------------------------------------- 1 | // Morris.js Charts sample data for SB Admin template 2 | 3 | $(function() { 4 | 5 | // Area Chart 6 | Morris.Area({ 7 | element: 'morris-area-chart', 8 | data: [{ 9 | period: '2010 Q1', 10 | iphone: 2666, 11 | ipad: null, 12 | itouch: 2647 13 | }, { 14 | period: '2010 Q2', 15 | iphone: 2778, 16 | ipad: 2294, 17 | itouch: 2441 18 | }, { 19 | period: '2010 Q3', 20 | iphone: 4912, 21 | ipad: 1969, 22 | itouch: 2501 23 | }, { 24 | period: '2010 Q4', 25 | iphone: 3767, 26 | ipad: 3597, 27 | itouch: 5689 28 | }, { 29 | period: '2011 Q1', 30 | iphone: 6810, 31 | ipad: 1914, 32 | itouch: 2293 33 | }, { 34 | period: '2011 Q2', 35 | iphone: 5670, 36 | ipad: 4293, 37 | itouch: 1881 38 | }, { 39 | period: '2011 Q3', 40 | iphone: 4820, 41 | ipad: 3795, 42 | itouch: 1588 43 | }, { 44 | period: '2011 Q4', 45 | iphone: 15073, 46 | ipad: 5967, 47 | itouch: 5175 48 | }, { 49 | period: '2012 Q1', 50 | iphone: 10687, 51 | ipad: 4460, 52 | itouch: 2028 53 | }, { 54 | period: '2012 Q2', 55 | iphone: 8432, 56 | ipad: 5713, 57 | itouch: 1791 58 | }], 59 | xkey: 'period', 60 | ykeys: ['iphone', 'ipad', 'itouch'], 61 | labels: ['iPhone', 'iPad', 'iPod Touch'], 62 | pointSize: 2, 63 | hideHover: 'auto', 64 | resize: true 65 | }); 66 | 67 | // Donut Chart 68 | Morris.Donut({ 69 | element: 'morris-donut-chart', 70 | data: [{ 71 | label: "Download Sales", 72 | value: 12 73 | }, { 74 | label: "In-Store Sales", 75 | value: 30 76 | }, { 77 | label: "Mail-Order Sales", 78 | value: 20 79 | }], 80 | resize: true 81 | }); 82 | 83 | // Line Chart 84 | Morris.Line({ 85 | // ID of the element in which to draw the chart. 86 | element: 'morris-line-chart', 87 | // Chart data records -- each entry in this array corresponds to a point on 88 | // the chart. 89 | data: [{ 90 | d: '2012-10-01', 91 | visits: 802 92 | }, { 93 | d: '2012-10-02', 94 | visits: 783 95 | }, { 96 | d: '2012-10-03', 97 | visits: 820 98 | }, { 99 | d: '2012-10-04', 100 | visits: 839 101 | }, { 102 | d: '2012-10-05', 103 | visits: 792 104 | }, { 105 | d: '2012-10-06', 106 | visits: 859 107 | }, { 108 | d: '2012-10-07', 109 | visits: 790 110 | }, { 111 | d: '2012-10-08', 112 | visits: 1680 113 | }, { 114 | d: '2012-10-09', 115 | visits: 1592 116 | }, { 117 | d: '2012-10-10', 118 | visits: 1420 119 | }, { 120 | d: '2012-10-11', 121 | visits: 882 122 | }, { 123 | d: '2012-10-12', 124 | visits: 889 125 | }, { 126 | d: '2012-10-13', 127 | visits: 819 128 | }, { 129 | d: '2012-10-14', 130 | visits: 849 131 | }, { 132 | d: '2012-10-15', 133 | visits: 870 134 | }, { 135 | d: '2012-10-16', 136 | visits: 1063 137 | }, { 138 | d: '2012-10-17', 139 | visits: 1192 140 | }, { 141 | d: '2012-10-18', 142 | visits: 1224 143 | }, { 144 | d: '2012-10-19', 145 | visits: 1329 146 | }, { 147 | d: '2012-10-20', 148 | visits: 1329 149 | }, { 150 | d: '2012-10-21', 151 | visits: 1239 152 | }, { 153 | d: '2012-10-22', 154 | visits: 1190 155 | }, { 156 | d: '2012-10-23', 157 | visits: 1312 158 | }, { 159 | d: '2012-10-24', 160 | visits: 1293 161 | }, { 162 | d: '2012-10-25', 163 | visits: 1283 164 | }, { 165 | d: '2012-10-26', 166 | visits: 1248 167 | }, { 168 | d: '2012-10-27', 169 | visits: 1323 170 | }, { 171 | d: '2012-10-28', 172 | visits: 1390 173 | }, { 174 | d: '2012-10-29', 175 | visits: 1420 176 | }, { 177 | d: '2012-10-30', 178 | visits: 1529 179 | }, { 180 | d: '2012-10-31', 181 | visits: 1892 182 | }, ], 183 | // The name of the data record attribute that contains x-visitss. 184 | xkey: 'd', 185 | // A list of names of data record attributes that contain y-visitss. 186 | ykeys: ['visits'], 187 | // Labels for the ykeys -- will be displayed when you hover over the 188 | // chart. 189 | labels: ['Visits'], 190 | // Disables line smoothing 191 | smooth: false, 192 | resize: true 193 | }); 194 | 195 | // Bar Chart 196 | Morris.Bar({ 197 | element: 'morris-bar-chart', 198 | data: [{ 199 | device: 'iPhone', 200 | geekbench: 136 201 | }, { 202 | device: 'iPhone 3G', 203 | geekbench: 137 204 | }, { 205 | device: 'iPhone 3GS', 206 | geekbench: 275 207 | }, { 208 | device: 'iPhone 4', 209 | geekbench: 380 210 | }, { 211 | device: 'iPhone 4S', 212 | geekbench: 655 213 | }, { 214 | device: 'iPhone 5', 215 | geekbench: 1571 216 | }], 217 | xkey: 'device', 218 | ykeys: ['geekbench'], 219 | labels: ['Geekbench'], 220 | barRatio: 0.4, 221 | xLabelAngle: 35, 222 | hideHover: 'auto', 223 | resize: true 224 | }); 225 | 226 | 227 | }); 228 | -------------------------------------------------------------------------------- /templates/block_page.html: -------------------------------------------------------------------------------- 1 | {% macro pager(_uri, total, limit, curr_page, left=3, right=7) -%} 2 | {% if '?' in _uri %} 3 | {% set uri = _uri + '&' %} 4 | {% else %} 5 | {% set uri = _uri + '?' %} 6 | {% endif %} 7 | {% if total > limit %} 8 | {% set page_num = total//limit if total%limit==0 else total//limit+1 %} 9 | {% set pre_page = curr_page - 1 %} 10 | {% set pre_page = 1 if pre_page < 1 else pre_page %} 11 | {% set next_page = curr_page + 1 %} 12 | {% set next_page = page_num if next_page > page_num else next_page %} 13 | {% set begin_idx = 1 if curr_page <= 3 else curr_page - left %} 14 | {% set end_idx = begin_idx + right %} 15 | {% set end_idx = page_num if end_idx > page_num else end_idx %} 16 | 41 | {%endif%} 42 | {%- endmacro %} -------------------------------------------------------------------------------- /templates/charts.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Admin - Bootstrap Admin Template 13 | {% include 'public_css.html' %} 14 | 15 | 16 | 17 | 18 | 19 |
20 | 21 | 22 | {% include 'public_nav.html' %} 23 | 24 |
25 | 26 |
27 | 28 | 29 |
30 |
31 |

32 | Charts 33 |

34 |
    35 |
  1. 36 | Dashboard 37 |
    2. 38 |
  2. 39 | Charts 40 |
    3. 41 |
42 |
43 |
44 | 45 | 46 | 47 |
48 |
49 |

Flot Charts

50 |

Flot is a pure JavaScript plotting library for jQuery, with a focus on simple usage, attractive looks and interactive features. For full usage instructions and documentation for Flot Charts, visit http://www.flotcharts.org/.

51 |
52 |
53 | 54 | 55 |
56 |
57 |
58 |
59 |

Line Graph Example with Tooltips

60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 | 70 |
71 | 72 | 73 |
74 | 75 | 76 |
77 | 78 | 79 | 80 | {% include 'public_js.html' %} 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /templates/forms.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Admin - Bootstrap Admin Template 10 | {% include "public_css.html" %} 11 | 12 | 13 |
14 | 15 | {% include "public_nav.html" %} 16 |
17 |
18 | 19 |
20 |
21 |

22 | Forms 23 |

24 |
    25 |
  1. 26 | Dashboard 27 |
    2. 28 |
  2. 29 | Forms 30 |
    3. 31 |
32 |
33 |
34 | 35 | 36 |
37 |
38 | 39 |
40 | 41 |
42 | 43 | 44 |

Example block-level help text here.

45 |
46 | 47 |
48 | 49 | 50 |
51 | 52 |
53 | 54 |

email@example.com

55 |
56 | 57 |
58 | 59 | 60 |
61 | 62 |
63 | 64 | 65 |
66 | 67 |
68 | 69 |
70 | 73 |
74 |
75 | 78 |
79 |
80 | 83 |
84 |
85 | 86 |
87 | 88 | 91 | 94 | 97 |
98 | 99 |
100 | 101 |
102 | 105 |
106 |
107 | 110 |
111 |
112 | 115 |
116 |
117 | 118 |
119 | 120 | 123 | 126 | 129 |
130 | 131 |
132 | 133 | 140 |
141 | 142 |
143 | 144 | 151 |
152 | 153 | 154 | 155 | 156 |
157 | 158 |
159 |
160 |

Disabled Form States

161 | 162 |
163 | 164 |
165 | 166 |
167 | 168 | 169 |
170 | 171 |
172 | 173 | 176 |
177 | 178 |
179 | 182 |
183 | 184 | 185 | 186 |
187 | 188 |
189 | 190 |

Form Validation

191 | 192 |
193 | 194 |
195 | 196 | 197 |
198 | 199 |
200 | 201 | 202 |
203 | 204 |
205 | 206 | 207 |
208 | 209 |
210 | 211 |

Input Groups

212 | 213 |
214 | 215 |
216 | @ 217 | 218 |
219 | 220 |
221 | 222 | .00 223 |
224 | 225 |
226 | 227 | 228 |
229 | 230 |
231 | $ 232 | 233 | .00 234 |
235 | 236 |
237 | 238 | 239 |
240 | 241 |
242 | 243 |

For complete documentation, please visit Bootstrap's Form Documentation.

244 | 245 |
246 |
247 | 248 | 249 |
250 | 251 | 252 |
253 | 254 | 255 |
256 | 257 | {% include "public_js.html" %} 258 | 259 | 260 | 261 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Admin - Bootstrap Admin Template 10 | {% include 'public_css.html' %} 11 | 12 | 13 | 14 | 15 |
16 | {% include 'public_nav.html' %} 17 |
18 | 19 |
20 | 21 | 22 |
23 |
24 |

25 | Dashboard Statistics Overview 26 |

27 |
    28 |
  1. 29 | Dashboard 30 |
    2. 31 |
32 |
33 |
34 | 35 | 36 |
37 |
38 |
39 | 40 | Like SB Admin? Try out SB Admin 2 for additional features! 41 |
42 |
43 |
44 | 45 | 46 |
47 |
48 |
49 |
50 |
51 |
52 | 53 |
54 |
55 |
26
56 |
New Comments!
57 |
58 |
59 |
60 | 61 |
62 | View Details 63 | 64 |
65 |
66 | 67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 | 75 |
76 |
77 |
12
78 |
New Tasks!
79 |
80 |
81 |
82 | 83 |
84 | View Details 85 | 86 |
87 |
88 | 89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 | 97 |
98 |
99 |
124
100 |
New Orders!
101 |
102 |
103 |
104 | 105 |
106 | View Details 107 | 108 |
109 |
110 | 111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 | 119 |
120 |
121 |
13
122 |
Support Tickets!
123 |
124 |
125 |
126 | 127 |
128 | View Details 129 | 130 |
131 |
132 | 133 |
134 |
135 |
136 | 137 | 138 |
139 |
140 |
141 |
142 |

Area Chart

143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 | 154 | 155 | {% include 'public_js.html' %} 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /templates/list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 搜索结果:{{keywords}} 6 | 7 | 8 | 9 | 10 |

{{ keywords }}的搜索结果是:

11 |
    12 | {% for row in results.data %} 13 |
  • 14 |

    {{ row.hit_title|safe }}({{ row.publish_time }})

    15 |

    命中词语:{{ row.hit_keywords }}

    16 |
    17 | {{ row.hit_body|safe }} 18 |
    19 |
    • 20 | {% endfor %} 21 |
22 | {% import "block_page.html" as page %} 23 | {{ page.pager('/search/'+keywords, results.total, results.size, results.page) }} 24 | 25 | 26 | -------------------------------------------------------------------------------- /templates/list.html_bak: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 搜索结果:{{keywords}} 6 | 7 | 8 | {% block keywords %}{% endblock %}的搜索结果是: 9 |
    10 | {% for row in results.data %} 11 |
  • 12 |

    {{ row.hit_title|e }}({{ row.publish_time }})

    13 |

    命中词语:{{ row.hit_keywords }}

    14 |

    命中文本:{{ row.hit }}

    15 |
    • 16 | {% endfor %} 17 |
18 |

一共{{ results.total }}项,每页{{ results.size }}项,当前第{{ results.page }}页

19 | 20 | 21 | -------------------------------------------------------------------------------- /templates/public_css.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 15 | -------------------------------------------------------------------------------- /templates/public_js.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /templates/public_left.html: -------------------------------------------------------------------------------- 1 |
2 | 16 |
17 | -------------------------------------------------------------------------------- /templates/public_nav.html: -------------------------------------------------------------------------------- 1 | 117 | -------------------------------------------------------------------------------- /templates/search_list.html_bak: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ title }} 6 | 7 | 8 | {{ title }} 9 | 10 | 11 | -------------------------------------------------------------------------------- /templates/tables.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Admin - Bootstrap Admin Template 13 | {% include 'public_css.html' %} 14 | 15 | 16 | 17 | 18 | 19 |
20 | 21 | 22 | {% include "public_nav.html" %} 23 | 24 |
25 | 26 |
27 | 28 | 29 |
30 |
31 |

32 | Tables 33 |

34 |
    35 |
  1. 36 | Dashboard 37 |
    2. 38 |
  2. 39 | Tables 40 |
    3. 41 |
42 |
43 |
44 | 45 | 46 |
47 |
48 |

Bordered Table

49 |
50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 |
PageVisits% New VisitsRevenue
/index.html126532.3%$321.33
/about.html26133.3%$234.12
/sales.html66521.3%$16.34
/blog.html951689.3%$1644.43
/404.html2334.3%$23.52
/services.html42160.3%$724.32
/blog/post.html123393.2%$126.34
104 |
105 |
106 |
107 |

Bordered with Striped Rows

108 |
109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 |
PageVisits% New VisitsRevenue
/index.html126532.3%$321.33
/about.html26133.3%$234.12
/sales.html66521.3%$16.34
/blog.html951689.3%$1644.43
/404.html2334.3%$23.52
/services.html42160.3%$724.32
/blog/post.html123393.2%$126.34
163 |
164 |
165 |
166 | 167 | 168 |
169 |
170 |

Basic Table

171 |
172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 |
PageVisits% New VisitsRevenue
/index.html126532.3%$321.33
/about.html26133.3%$234.12
/sales.html66521.3%$16.34
/blog.html951689.3%$1644.43
/404.html2334.3%$23.52
/services.html42160.3%$724.32
/blog/post.html123393.2%$126.34
226 |
227 |
228 |
229 |

Striped Rows

230 |
231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 |
PageVisits% New VisitsRevenue
/index.html126532.3%$321.33
/about.html26133.3%$234.12
/sales.html66521.3%$16.34
/blog.html951689.3%$1644.43
/404.html2334.3%$23.52
/services.html42160.3%$724.32
/blog/post.html123393.2%$126.34
285 |
286 |
287 |
288 | 289 | 290 |
291 |
292 |

Contextual Classes

293 |
294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 |
PageVisits% New VisitsRevenue
/index.html126532.3%$321.33
/about.html26133.3%$234.12
/sales.html66521.3%$16.34
/blog.html951689.3%$1644.43
/404.html2334.3%$23.52
/services.html42160.3%$724.32
/blog/post.html123393.2%$126.34
348 |
349 |
350 |
351 |

Bootstrap Docs

352 |

For complete documentation, please visit Bootstrap's Tables Documentation.

353 |
354 |
355 | 356 | 357 |
358 | 359 | 360 |
361 | 362 | 363 |
364 | 365 | {% include "public_js.html" %} 366 | 367 | 368 | 369 | 370 | -------------------------------------------------------------------------------- /templates/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Admin - Bootstrap Admin Template 10 | {% include 'public_css.html' %} 11 | 12 | 13 |
14 | 15 | {% include 'public_nav.html' %} 16 | 17 |
18 | 19 |
20 | 21 | 22 |
23 |
24 |

25 | 频道 26 |

27 |
    28 |
  1. 29 | 控制台 30 |
    2. 31 |
  2. 32 | 频道 33 |
    3. 34 |
35 |
36 |
37 |
38 |
39 |
40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | {% for item in list %} 51 | 52 | 53 | 54 | 55 | 56 | 57 | {% endfor %} 58 | 59 |
编号名字排序操作
{{ item.id }}{{ item.name }}{{ item.sort }}编辑删除
60 |
61 |
62 |
63 | 64 |
65 | 66 | 67 |
68 | 69 | 70 |
71 | 72 | 73 | 74 | {% include 'public_js.html' %} 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /web.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from flask import Flask 4 | from flask import abort 5 | from flask import render_template 6 | from flask import request 7 | from webspider.SearchApi import search 8 | from webspider.conf.config import DBSession 9 | from webspider.model.Classify import Classify 10 | 11 | db = DBSession() 12 | 13 | app = Flask(__name__) 14 | 15 | 16 | # @app.route('/') 17 | # def page_list(): 18 | # pageList = db.query(Page).all() 19 | # page = pageList[0] 20 | # title = page.title.decode("GBK") 21 | # return render_template('search_list.html', title=title) 22 | # 23 | # app = Flask(__name__) 24 | 25 | 26 | @app.route('/') 27 | @app.route('/index') 28 | def index(): 29 | # abort(404) 30 | results = None 31 | keywords = None 32 | return render_template('index.html', results=results, keywords=keywords) 33 | 34 | 35 | @app.route('/charts') 36 | def charts(): 37 | return render_template('charts.html') 38 | 39 | 40 | @app.route('/tables') 41 | def tables(): 42 | return render_template('tables.html') 43 | 44 | 45 | @app.route('/forms') 46 | def forms(): 47 | return render_template('forms.html') 48 | 49 | 50 | @app.route('/test') 51 | def tests(): 52 | list = db.query(Classify).all() 53 | return render_template('test.html', list=list) 54 | 55 | 56 | @app.route("/editc") 57 | def editClass(): 58 | return 1 59 | 60 | 61 | @app.route("/delc") 62 | def delClass(): 63 | return 1 64 | 65 | 66 | @app.route('/search') 67 | @app.route('/search/') 68 | @app.route('/search/') 69 | @app.route('/search//page/') 70 | def act_search(keywords="", page=1): 71 | if request.args: 72 | page = request.args['p'] if request.args['p'] else 1 73 | else: 74 | page = 1 75 | if not keywords: 76 | keywords = "" 77 | results = [] 78 | print(1) 79 | else: 80 | keywords = unicode(keywords) 81 | results = search(keywords, int(page)) 82 | return render_template('list.html', results=results, keywords=keywords) 83 | # keywords = unicode(keywords) 84 | # results = search(keywords, int(page)) 85 | # return render_template('list.html', results=results, keywords=keywords) 86 | 87 | 88 | # @app.route('/list') 89 | # def page_list(): 90 | # pageList = db.query(Page).all() 91 | # page = pageList[0] 92 | # return render_template('search_list.html', page=page) 93 | 94 | 95 | if __name__ == '__main__': 96 | app.run(debug=True) 97 | -------------------------------------------------------------------------------- /webspider/QiniuStorage.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from qiniu import Auth 3 | from qiniu import BucketManager, put_data 4 | from twisted.internet import threads 5 | 6 | from conf.config import qiniu_config as qnconf 7 | 8 | 9 | class Qiniu(object): 10 | def __init__(self, uri): 11 | assert uri.startswith('http://') 12 | self.qn = Auth(qnconf['access_key'], qnconf['secret_key']) 13 | self.token = self.qn.upload_token(qnconf['bucket_name']) 14 | self.bucket = BucketManager(self.qn) 15 | 16 | # def stat_image(self, key): 17 | # status, image_info = self.bucket.stat(qnconf['bucket_name'], key) 18 | # last_modified = int(status['putTime'] / 1000000) 19 | # return {'checksum': status['hash'], 'last_modified': last_modified} 20 | 21 | # def persist_image(self, key, image, buf, info): 22 | # buf.seek(0) 23 | # return threads.deferToThread(put_data, self.token, key, buf.getvalue()) 24 | 25 | def get_file_stat(self, key): 26 | stat, error = self.bucket.stat(qnconf['bucket_name'], key) 27 | return stat 28 | 29 | def stat_file(self, path, info): 30 | def _onsuccess(stat): 31 | if stat: 32 | checksum = stat['hash'] 33 | timestamp = stat['putTime'] / 10000000 34 | return {'checksum': checksum, 'last_modified': timestamp} 35 | else: 36 | return {} 37 | return threads.deferToThread(self.get_file_stat, path).addCallback(_onsuccess) 38 | 39 | def persist_file(self, path, buf, info, meta=None, headers=None): 40 | buf.seek(0) 41 | return threads.deferToThread(put_data, self.token, path, buf.getvalue()) 42 | -------------------------------------------------------------------------------- /webspider/QiniuStore.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | from conf.config import qiniu_config as qnconf 4 | from qiniu import Auth, BucketManager, put_data 5 | import logging 6 | import hashlib 7 | 8 | QINIU_ACCESS_KEY = qnconf["access_key"] 9 | QINIU_SECRET_KEY = qnconf["secret_key"] 10 | QINIU_DEFAULT_BUCKET = qnconf["bucket_name"] 11 | QINIU_DEFAULT_URL_PREFIX = qnconf["domain"] 12 | 13 | 14 | class QiniuCloud(object): 15 | """ 16 | 七牛云，上传图片 17 | """ 18 | 19 | def __init__(self, access_key=QINIU_ACCESS_KEY, secret_key=QINIU_SECRET_KEY): 20 | self.auth = Auth(access_key, secret_key) 21 | 22 | def upload_by_fetch(self, remote_file_path, key): 23 | """ 24 | 通过远程url上传文件，但是可能报错，请使用upload()方法 25 | :param remote_file_path: 26 | :param key: 27 | :return: 28 | """ 29 | pass 30 | # bucket = BucketManager(self.auth) 31 | # ret, info = bucket.fetch(remote_file_path, QINIU_DEFAULT_BUCKET, key) 32 | # print('ret: %s' % ret) 33 | # print('info: %s' % info) 34 | 35 | def upload(self, file_data, key): 36 | """ 37 | 通过二进制流上传文件 38 | :param file_data: 二进制数据 39 | :param key: key 40 | :return: 41 | """ 42 | try: 43 | token = self.auth.upload_token(QINIU_DEFAULT_BUCKET) 44 | ret, info = put_data(token, key, file_data) 45 | except Exception as e: 46 | logging.error('upload error, key: {0}, exception: {1}'.format(key, e)) 47 | 48 | if info.status_code == 200: 49 | logging.info('upload data to qiniu ok, key: {0}'.format(key)) 50 | return True 51 | else: 52 | logging.error('upload data to qiniu error, key: {0}'.format(key)) 53 | return False 54 | 55 | def upload_pics(self, pic_list): 56 | """ 57 | 上传图片列表，如果上传成功，则保存到mysql中 58 | :param pic_list: 图片列表 59 | :return: 60 | """ 61 | logging.info('in upload_pic_list...') 62 | pic_key_list = [] 63 | for pic_url in pic_list: 64 | pic_suffix = '.jpg' 65 | # key = key_prefix + str(info_id) + '/' + datetime.now().strftime('%Y%m%d%H%M%S%f') + pic_suffix 66 | key = "cd/" + hashlib.sha1(pic_url).hexdigest() + pic_suffix 67 | pic_data = requests.get(pic_url) 68 | if self.upload(pic_data.content, key): 69 | pic_key_list.append((pic_url, QINIU_DEFAULT_URL_PREFIX + key)) 70 | 71 | return pic_key_list 72 | -------------------------------------------------------------------------------- /webspider/SearchApi.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import unicode_literals 4 | from whoosh.index import open_dir 5 | from whoosh.fields import * 6 | from whoosh.qparser import QueryParser, MultifieldParser 7 | from conf.config import search_path 8 | # from jieba.analyse import ChineseAnalyzer 9 | from whoosh import scoring, sorting 10 | from whoosh.query import * 11 | import HTMLParser 12 | 13 | 14 | # sys.path.append("../") 15 | 16 | 17 | # analyzer = ChineseAnalyzer() 18 | 19 | 20 | def search(inputstring="", page=1, size=10): 21 | """ 22 | 搜索接口 23 | :type inputstring: 传入搜索字符串 24 | :param page: 页数 25 | :param size: 每页显示的多少 26 | """ 27 | inputstring = unicode(inputstring) 28 | res = [] 29 | results = [] 30 | # if __name__ == '__main__': 31 | # index_path = "./sindex" 32 | # else: 33 | # index_path = "./webspider/sindex" 34 | index_path = search_path 35 | try: 36 | ix = open_dir(index_path) # 搜索路径 37 | except Exception, e: 38 | pass 39 | else: 40 | if not inputstring: 41 | pass 42 | else: 43 | publish_time = sorting.FieldFacet("publish_time", reverse=True) 44 | qp = MultifieldParser(["title", "body"], schema=ix.schema) 45 | with ix.searcher(weighting=scoring.TF_IDF()) as searcher: 46 | querystring = qp.parse(inputstring) 47 | results = searcher.search(querystring, terms=True, limit=None, sortedby=[publish_time]) 48 | # print(len(results)) 49 | # results = searcher.search_page(querystring, page) 50 | html_parser = HTMLParser.HTMLParser() 51 | if len(results) > 0: 52 | for i in xrange((page - 1) * size, page * size): 53 | if i in xrange(len(results)): 54 | tmpret = results[i].fields() 55 | hit_keywords = set() 56 | for key, val in results[i].matched_terms(): 57 | hit_keywords.add(val.decode('utf-8')) 58 | # hit_keywords.add(val.encode('latin1').decode('gbk')) 59 | tmpret['words'] = results[i].matched_terms() 60 | tmpret['hit_keywords'] = " ".join(list(hit_keywords)) 61 | tmpret['hit_title'] = html_parser.unescape(results[i].highlights('title')) if results[i].highlights('title') else results[i]['title'] 62 | # tmpret['hit_title'] = html_parser.\ 63 | # unescape(results[i].highlights('title').encode('latin1')).decode('unicode_escape') \ 64 | # if results[i].highlights('title') else results[i]['title'] 65 | 66 | # print html_parser.unescape(results[i].highlights('title').encode('utf-8')) 67 | # tmpret['hit_body'] = results[i].highlights('body') 68 | res.append(tmpret) 69 | 70 | page_info = {'total': len(results), 'page': page, 'size': size, 'data': res} 71 | return page_info 72 | 73 | 74 | if __name__ == '__main__': 75 | ret = search("中国") 76 | print(ret) 77 | # 78 | # ix = open_dir("./sindex") # for read only 79 | # searcher = ix.searcher() 80 | # parser = QueryParser("body", schema=ix.schema) 81 | # q = parser.parse(u"中国") 82 | # results = searcher.search(q) 83 | # for hit in results: 84 | # print(hit) 85 | # print("=" * 10) 86 | -------------------------------------------------------------------------------- /webspider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/webspider/__init__.py -------------------------------------------------------------------------------- /webspider/addSearch.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import unicode_literals 4 | import sys 5 | import os 6 | import datetime 7 | from whoosh.index import create_in 8 | from whoosh.fields import * 9 | from conf.config import DBSession, search_path 10 | from model.Page import Page 11 | from sqlalchemy import text 12 | 13 | from jieba.analyse import ChineseAnalyzer 14 | analyzer = ChineseAnalyzer() 15 | # sys.path.append("../") 16 | 17 | schema = Schema( 18 | page_id=NUMERIC(), 19 | title=TEXT(stored=True, analyzer=analyzer), 20 | body=TEXT(stored=True, analyzer=analyzer), 21 | url=ID(stored=True), 22 | # site_name=ID(stored=True), 23 | # html_title=TEXT(stored=True), 24 | # source_site=ID(stored=True), 25 | publish_time=DATETIME(stored=True), 26 | save_time=DATETIME(stored=True), 27 | # save_path=TEXT(stored=True) 28 | ) 29 | if not os.path.exists(search_path): 30 | os.mkdir(search_path) 31 | 32 | ix = create_in(search_path, schema) 33 | writer = ix.writer() 34 | # fromtimestamp int->time 35 | # time = 0 36 | # datetime.datetime.fromtimestamp(time) 37 | # strptime string->time 38 | # format = '%Y-%m-%d %H:%M:%S' str = '2015-10-27 15:09:38' 39 | # datetime.datetime.strptime(str,format) 40 | 41 | timeformat = '%Y-%m-%d %H:%M:%S' 42 | db = DBSession() 43 | """ 过滤条件参照该文件最下面的增量索引的方案 """ 44 | # 新增数据 45 | # sql = text('insert into users (u_name, u_password) values (:name, :password)') 46 | # data = db.execute(sql, {'name': 'nate1', 'password': password}) 47 | # 删除数据 48 | # sql = text('delete from users where u_id = :id') 49 | # data = session.execute(sql, {'id': last_id}) 50 | # sql = text('select * from users') 51 | # data = session.execute(sql) 52 | 53 | # 查询SQL版本 54 | sql = text('select * from Page where id < :id') 55 | pageList = db.execute(sql, {'id': 20}) 56 | # 查询ORM版本 57 | # pageList = db.query(Page).filter(Page.id < 20).all() 58 | 59 | i = 0 60 | for row in pageList: 61 | publish_time = datetime.datetime.strptime(row.publish_time, timeformat) 62 | save_time = datetime.datetime.fromtimestamp(row.save_time) 63 | # print(type(row.title)) 64 | writer.add_document( 65 | page_id=row.id, 66 | title=row.title, 67 | body=row.body, 68 | url=row.url, 69 | # site_name=row.site_name, 70 | # html_title=row.html_title, 71 | # source_site=row.source_site, 72 | publish_time=publish_time, 73 | save_time=save_time, 74 | # save_path=row.save_path 75 | ) 76 | i += 1 77 | print("inserting the %s rows" % i)\ 78 | # 可以在循环内部提交 79 | db.close() 80 | writer.commit() 81 | print("all had inserted!!!") 82 | """ 83 | 使用标志位的办法增量添加. 84 | 1. 标志位为0的是待添加对象, 85 | 2. 标志位为1是已经添加的对象. 86 | 3. 这里可以设置一个添加的日志文件. 87 | 4. 上传完成之后修改数据库和生成日志文件. 88 | 5. 日志文件只用简单的串行化page_id存储即可. 89 | """ 90 | -------------------------------------------------------------------------------- /webspider/conf/__init__.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /webspider/conf/config.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | # import MySQLdb.cursors 4 | import redis 5 | from sqlalchemy import create_engine 6 | from sqlalchemy.orm import sessionmaker 7 | 8 | # 初始化数据库连接: 9 | engine = create_engine('mysql+mysqldb://root:root@127.0.0.1:3306/spider?charset=utf8') 10 | # 创建DBSession类型: 11 | DBSession = sessionmaker(bind=engine) 12 | confRedis = redis.StrictRedis( # redis 服务器 13 | host='127.0.0.1', 14 | port=6379, db=4, 15 | password='pass123456' 16 | ) 17 | check_proxy_url = "http://www.baidu.com" # 默认的检查代理是否可用的链接 18 | 19 | # base_path = "D:/PycharmProjects/webspider/" # 项目部署的根目录 20 | # img_base_path = "D:/PycharmProjects/webspider/" # 项目图片保存的跟路径 21 | img_save_path = "D:/PycharmProjects/webspider/image/" # 图片保存的相对路径 22 | # img_file = '%H' 23 | 24 | log_open = True # 是否开启日志记录 25 | log_path = 'D:/PycharmProjects/webspider/webspider/log/%Y_%m_%d' # 日志文件的保存路径 支持绝对路径 时间参数自动解析 26 | log_file = '%H' # 日志文件保存格式 27 | log_format = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" # 日志记录存储格式 28 | 29 | html_path = 'D:/PycharmProjects/webspider/webspider/html' # 采集到的html保存路径 支持绝对路径 30 | 31 | # 索引文件路径 32 | search_path = 'D:/PycharmProjects/webspider/webspider/search_index' 33 | 34 | # page 里面的flag的默认值 35 | default_page_flag = 1 36 | 37 | # Data Source=.;Initial Catalog=master;Integrated Security=True 38 | # Server=192.168.18.196;Initial Catalog=GB123;User ID=admin;Password=123456789 39 | # qiniu 40 | qiniu_config = { 41 | 'access_key': "*", 42 | 'secret_key': "*", 43 | 'bucket_name': "*", 44 | 'domain': "*" 45 | } 46 | -------------------------------------------------------------------------------- /webspider/delProxy.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | # from sqlalchemy import exc 4 | from conf.config import DBSession 5 | from util import WebProxy as webProxy 6 | from model.Proxy import Proxy 7 | 8 | db = DBSession() 9 | ips = db.query(Proxy).all() 10 | 11 | for item in ips: 12 | ret = webProxy.check_proxy(item.ip, item.port) 13 | if not ret: 14 | webProxy.delete_proxy(item.ip, item.port) 15 | 16 | -------------------------------------------------------------------------------- /webspider/downloaderMiddlewareSet.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | import sys 4 | # import logging 5 | from random import choice 6 | from util.UserAgent import get_user_agent 7 | from util import WebProxy as webProxy 8 | from conf.config import confRedis 9 | from scrapy.exceptions import IgnoreRequest 10 | # from scrapy.http import XmlResponse 11 | # import re 12 | # import urlparse 13 | # from pprint import pprint 14 | # import gzip 15 | # from scrapy.utils import gz 16 | 17 | 18 | class SetProxyMiddleware(object): 19 | """ 设置采集代理服务器 """ 20 | def __init__(self): 21 | self.proxies = webProxy.get_proxy_list() 22 | 23 | def process_request(self, request, spider): 24 | """ "http://127.0.0.1:8118" """ 25 | proxys = choice(self.proxies) 26 | request.meta['proxy'] = "http://%s:%d" % (proxys.ip, proxys.port) 27 | 28 | 29 | class SetUserAgentMiddleware(object): 30 | """ 设置userAgent """ 31 | def __init__(self): 32 | self.user_agent = get_user_agent() 33 | 34 | def process_request(self, request, spider): 35 | request.headers.setdefault('User-Agent', self.user_agent) 36 | request.headers.setdefault('Referer', 'http://www.baidu.com') 37 | 38 | 39 | class SetHeaderMiddleware(object): 40 | """ 设置header协议 """ 41 | def __init__(self): 42 | self.user_agent = get_user_agent() 43 | 44 | def process_request(self, request, spider): 45 | headers_list = { 46 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 47 | 'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4,und;q=0.2', 48 | 'Cache-Control': 'no-cache', 49 | 'Connection': 'keep-alive', 50 | 'Accept-Encoding': 'gzip, deflate, sdch', 51 | 'Transfer-Encoding': 'chunked', 52 | 'Referer': 'http://www.baidu.com', 53 | 'User-Agent': self.user_agent 54 | } 55 | for key, value in headers_list.items(): 56 | request.headers.setdefault(key, value) 57 | 58 | 59 | class IngoreHttpRequestMiddleware(object): 60 | """ 从下载器处开始拦截/过滤URL """ 61 | def process_response(self, request, response, spider): 62 | if confRedis.exists('url:%s' % response.url): 63 | raise IgnoreRequest("IgnoreRequest : %s" % response.url) 64 | else: 65 | # Redis.set('url:%s' % response.url, 1) 66 | return response 67 | 68 | 69 | class SetUtf8Middleware(object): 70 | """A downloader middleware to force utf8 encoding for all responses.""" 71 | 72 | def process_response(self, request, response, spider): 73 | reload(sys) 74 | sys.setdefaultencoding("GBK") 75 | ubody = response.body_as_unicode().encode('GBK') 76 | # if isinstance(response, XmlResponse): 77 | # ubody = response.body_as_unicode().encode('utf8') 78 | # elif gz.is_gzipped(response): 79 | # ubody = gz.gunzip(response.body_as_unicode().encode('utf8')) 80 | # else: 81 | # ubody = response.body_as_unicode().encode('utf8') 82 | # logging.warning("Ignoring non-XML sitemap: %s" % response) 83 | return response.replace(body=ubody, encoding='GBK') 84 | 85 | -------------------------------------------------------------------------------- /webspider/getProxy.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import logging 5 | from datetime import datetime 6 | from scrapy.crawler import CrawlerProcess 7 | from scrapy.settings import Settings 8 | from scrapy.utils.log import configure_logging 9 | from conf.config import log_path, log_file, log_format 10 | from spiders.ProxySpider import GetProxySpider 11 | 12 | 13 | def run_spider(): 14 | settings = Settings() 15 | settings.set("USER_AGENT", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36") 16 | settings.set("ITEM_PIPELINES", { 17 | 'pipelines.FilterProxyPipline': 1, 18 | 'pipelines.SaveProxyPipeline': 2 19 | }) 20 | settings.set("LOG_STDOUT ", True) 21 | 22 | # 配置日志记录规则设置 23 | # configure_logging({ 24 | # 'filename': datetime.now().strftime('%Y_%m_%d_%H_proxy.log'), 25 | # 'format': '%(asctime)s %(levelname)-8s %(name)-15s %(message)s', 26 | # 'level': logging.INFO 27 | # }) 28 | configure_logging(install_root_handler=False) 29 | # 初始化日志路径 30 | logpath = datetime.now().strftime(log_path) 31 | if not os.path.isdir(logpath): 32 | os.makedirs(logpath) 33 | logging.basicConfig( 34 | filename=datetime.now().strftime('%s/%s_proxy.log' % (logpath, log_file)), 35 | format=log_format, 36 | level=logging.INFO 37 | ) 38 | process = CrawlerProcess(settings) 39 | process.crawl(GetProxySpider) 40 | process.start() 41 | 42 | if __name__ == '__main__': 43 | run_spider() 44 | -------------------------------------------------------------------------------- /webspider/model/Article.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy import Column, String, Integer, Text, Boolean 4 | from sqlalchemy.ext.declarative import declarative_base 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Article(Base): 10 | __tablename__ = 'Article' 11 | 12 | id = Column(Integer, primary_key=True) 13 | url = Column(String(500)) 14 | urlmd5 = Column(String(32)) 15 | site_name = Column(String(200)) 16 | html_title = Column(String(500)) 17 | save_path = Column(String(200)) 18 | save_time = Column(Integer) 19 | title = Column(String(200)) 20 | thumb = Column(String(500)) 21 | img_list = Column(String(1000)) 22 | body = Column(Text) 23 | publish_time = Column(String) 24 | source_site = Column(String) 25 | flag = Column(Boolean) 26 | -------------------------------------------------------------------------------- /webspider/model/Classify.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy import Column, String, Integer, Text, Boolean 4 | from sqlalchemy.ext.declarative import declarative_base 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Classify(Base): 10 | __tablename__ = 'classify' 11 | 12 | id = Column(Integer, primary_key=True) 13 | name = Column(String(20)) 14 | sort = Column(Integer) 15 | -------------------------------------------------------------------------------- /webspider/model/Page.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy import Column, String, Integer, Text, Boolean 4 | from sqlalchemy.ext.declarative import declarative_base 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Page(Base): 10 | __tablename__ = 'Page' 11 | 12 | id = Column(Integer, primary_key=True) 13 | rule_id = Column(Integer) 14 | url = Column(String(500)) 15 | urlmd5 = Column(String(32)) 16 | site_name = Column(String(200)) 17 | html_title = Column(String(500)) 18 | save_path = Column(String(200)) 19 | save_time = Column(Integer) 20 | title = Column(String(200)) 21 | thumb = Column(String(500)) 22 | img_list = Column(String(1000)) 23 | body = Column(Text) 24 | publish_time = Column(String(30)) 25 | source_site = Column(String(50)) 26 | flag = Column(Boolean) 27 | -------------------------------------------------------------------------------- /webspider/model/Proxy.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy import Column, String, Integer, DateTime 4 | from sqlalchemy.ext.declarative import declarative_base 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Proxy(Base): 10 | __tablename__ = 'Proxy' 11 | 12 | id = Column(Integer, primary_key=True) 13 | haship = Column(String(32)) 14 | ip = Column(String(16)) 15 | port = Column(Integer) 16 | create_time = Column(Integer) 17 | -------------------------------------------------------------------------------- /webspider/model/Rule.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from sqlalchemy import Column, String, Integer, Boolean 4 | from sqlalchemy.ext.declarative import declarative_base 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class Rule(Base): 10 | __tablename__ = 'Rule' 11 | 12 | rule_id = Column(Integer, primary_key=True) 13 | weburl_id = Column(Integer) 14 | name = Column(String) 15 | allow_domains = Column(String) 16 | start_urls = Column(String) 17 | next_page = Column(String) 18 | extract_from = Column(String) 19 | allow_url = Column(String) 20 | title_xpath = Column(String) 21 | thumb_img_xpath = Column(String) 22 | body_xpath = Column(String) 23 | publish_time_xpath = Column(String) 24 | source_site_xpath = Column(String) 25 | enable = Column(Boolean) 26 | -------------------------------------------------------------------------------- /webspider/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengxiaochuang/ScrapyDemo/a4ffde0cc8e4d25765a50bbeaf007582677f96c1/webspider/model/__init__.py -------------------------------------------------------------------------------- /webspider/pipelines.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | import hashlib 3 | import re 4 | import time 5 | from scrapy.exceptions import DropItem 6 | from scrapy.http import Request, Response 7 | from scrapy.pipelines.images import ImagesPipeline, FilesPipeline 8 | from sqlalchemy import exc 9 | from QiniuStorage import Qiniu 10 | from conf.config import DBSession, confRedis, html_path, qiniu_config, default_page_flag 11 | from model.Page import Page 12 | from model.Proxy import Proxy 13 | from util import WebProxy as webProxy 14 | from util import common as utils 15 | 16 | 17 | # from model.Article import Article as Page 18 | 19 | 20 | class FilterProxyPipline(object): 21 | """ 过滤不能用的代理 """ 22 | 23 | def process_item(self, item, spider): 24 | ret = webProxy.check_proxy(item['ip'], item['port']) 25 | if ret: 26 | return item 27 | else: 28 | raise DropItem("Duplicate item found: %s:%s" % (item['ip'], item['port'])) 29 | 30 | 31 | class SaveProxyPipeline(object): 32 | """ 保存代理到数据库 """ 33 | 34 | def process_item(self, item, spider): 35 | # try: 36 | # con = MySQLdb.connect(**config.db_config) 37 | # cur = con.cursor() 38 | # sql = "INSERT INTO leiju_proxy (ip,port,proto,checked_at,created_at) VALUES (%s,%s,%s,%s,%s)" 39 | # parmam = [(item['ip'], item['port'], 'http', int(time.time()), int(time.time()))] 40 | # cur.executemany(sql, parmam) 41 | # con.commit() 42 | # con.close() 43 | # # return item 44 | # except Exception, e: 45 | # logging.info("SaveError: %s:%s %s" % (item['ip'], item['port'], format(e))) 46 | # raise DropItem("SaveError: %s:%s %s" % (item['ip'], item['port'], format(e))) 47 | db = DBSession() 48 | md5 = hashlib.md5() 49 | md5.update(item['ip'] + "." + item['port']) 50 | haship = md5.hexdigest() 51 | proxy = Proxy(haship=haship, ip=item['ip'], port=item['port'], create_time=int(time.time())) 52 | db.add(proxy) 53 | try: 54 | db.commit() 55 | except exc.SQLAlchemyError, e: 56 | raise DropItem("SaveError: %s:%s %s" % (item['ip'], item['port'], format(e))) 57 | finally: 58 | db.close() 59 | 60 | 61 | # class FilterUrlPipline(object): 62 | # """ use redis to fileter url """ 63 | # def process_item(self, item, spider): 64 | # redis = confRedis 65 | # redis.set('url:%s' % item["url"], 1) 66 | # return item 67 | 68 | 69 | class SaveCommonPipline(object): 70 | """ save common spider result to local """ 71 | 72 | def process_item(self, item, spider): 73 | db = DBSession() 74 | redis = confRedis 75 | 76 | rule_id = spider.rule_id 77 | url = item['url'] 78 | md5 = hashlib.md5() 79 | md5.update(url) 80 | urlmd5 = md5.hexdigest() 81 | site_name = utils.get_site(item['url']) 82 | # site_name = spider.rule['allow_domains'] 83 | html_title = item['html_title'] 84 | # html_body = item['html_body'] 85 | save_path = utils.md5dir(item['url']) 86 | save_time = int(time.time()) 87 | title = item['title'] if 'title' in item else "" 88 | body = item['body'] if 'body' in item else "" 89 | thumb = item['thumb'] if 'thumb' in item else "" 90 | img_list = item['img_list'] if 'img_list' in item else "" 91 | 92 | # TODO 这里使用一个分析方法,分析抓取到数据的发布时间,然后转换成时间戳 93 | publish_time = utils.smart2date(item['publish_time']) if 'publish_time' in item else "" 94 | source_site = item['source_site'] if 'source_site' in item else "" 95 | flag = default_page_flag 96 | 97 | page = Page(rule_id=rule_id, url=item['url'], urlmd5=urlmd5, site_name=site_name, html_title=html_title, 98 | save_path=save_path, 99 | save_time=save_time, title=title, 100 | thumb=thumb, img_list=img_list, 101 | body=body, publish_time=publish_time, 102 | source_site=source_site, flag=flag) 103 | has = db.query(Page).filter(Page.urlmd5 == urlmd5).first() 104 | if has: 105 | page = Page(rule_id=rule_id, url=item['url'], site_name=site_name, html_title=html_title, 106 | save_path=save_path, 107 | save_time=save_time, title=title, 108 | thumb=thumb, img_list=img_list, 109 | body=body, publish_time=publish_time, 110 | source_site=source_site, flag=flag) 111 | 112 | db.add(page) 113 | try: 114 | db.commit() 115 | utils.save_file('%s/%s' % (html_path, save_path), item['html_body']) 116 | redis.set('url:%s' % url, 1) 117 | except exc.SQLAlchemyError, e: 118 | raise DropItem("SaveDbError: %s,%s" % (url, format(e))) 119 | 120 | 121 | class ImgPipline(ImagesPipeline): 122 | """ 普通图片下载 """ 123 | 124 | def file_path(self, request, response=None, info=None): 125 | # image_name = ImgSpider.image_names[request.url] 126 | # return image_name 127 | image_guid = hashlib.sha1(request.url).hexdigest() 128 | part1 = image_guid[0:2] 129 | # part2 = md5url[2:4] 130 | # part3 = md5url[4:6] 131 | # filepath = "%s/%s/%s/%s/%s/%s.html" % (datestr, site.hostname, part1, part2, part3, md5url) 132 | filepath = "%s/%s.jpg" % (part1, image_guid) 133 | # return 'full/%s' % image_guid 134 | return filepath 135 | 136 | def get_media_requests(self, item, info): 137 | if item['img_urls']: 138 | for image_url in item['img_urls']['real']: 139 | yield Request(image_url) 140 | 141 | def item_completed(self, results, item, info): 142 | i = 0 143 | image_paths = [] 144 | for ok, x in results: 145 | if ok: 146 | # TODO 正则替换items里面的img 147 | item["body"] = re.sub(item['img_urls']["original"][i], x["path"], item["body"]) 148 | image_paths.append(x['path']) 149 | i += 1 150 | # item['or_url'] = or_url 151 | item["thumb"] = image_paths[0] if image_paths else "" 152 | item["img_list"] = ','.join(image_paths) if image_paths else "" 153 | return item 154 | 155 | 156 | class CoverImagesPipeline(ImagesPipeline): 157 | """ 七牛云存储图片上传 """ 158 | ImagesPipeline.STORE_SCHEMES['http'] = Qiniu 159 | URL_PREFIX = None 160 | 161 | @classmethod 162 | def from_settings(cls, settings): 163 | # cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) 164 | # cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) 165 | # cls.EXPIRES = settings.getint('IMAGES_EXPIRES', 90) 166 | # cls.THUMBS = settings.get('IMAGES_THUMBS', {}) 167 | qiniu = cls.STORE_SCHEMES['http'] 168 | return super(CoverImagesPipeline, cls).from_settings(settings) 169 | 170 | def get_media_requests(self, item, info): 171 | if item['img_urls']: 172 | for image_url in item['img_urls']['real']: 173 | yield Request(image_url) 174 | 175 | def item_completed(self, results, item, info): 176 | i = 0 177 | image_paths = [] 178 | for ok, x in results: 179 | if ok: 180 | # TODO 正则替换items里面的img 181 | tmp_path = qiniu_config["domain"] + x["path"] 182 | item["body"] = re.sub(item['img_urls']["original"][i], tmp_path, item["body"]) 183 | image_paths.append(tmp_path) 184 | i += 1 185 | item["thumb"] = image_paths[0] if image_paths else "" 186 | return item 187 | -------------------------------------------------------------------------------- /webspider/run.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from conf.config import DBSession, log_format, log_file, log_path, log_open, img_save_path 4 | from model.Rule import Rule 5 | from spiders.RuleSpider import RuleSpider 6 | 7 | from scrapy.crawler import CrawlerProcess 8 | from scrapy.settings import Settings 9 | from scrapy.utils.log import configure_logging 10 | import logging 11 | from datetime import datetime 12 | import os 13 | 14 | 15 | db = DBSession() 16 | 17 | 18 | def run_spider(): 19 | settings = Settings() 20 | settings.set("COOKIES_ENABLES", False) # 禁止cookies追踪 21 | settings.set("ITEM_PIPELINES", { 22 | 'pipelines.ImgPipline': 150, # 保存图片到本地 23 | # 'pipelines.CoverImagesPipeline': 150, # 保存图片到七牛云 24 | 'pipelines.SaveCommonPipline': 200, # 保存数据库 25 | # 'pipelines.FilterUrlPipline': 300, 26 | }) 27 | 28 | settings.set("DOWNLOADER_MIDDLEWARES", { 29 | 'downloaderMiddlewareSet.IngoreHttpRequestMiddleware': 1, # redis去重 30 | # 'downloaderMiddlewareSet.CountDropMiddleware': 2, 31 | 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, # 自动useragent 32 | # 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None, # header头很容易造成读取失败,不建议开启 33 | # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': None, 34 | 'downloaderMiddlewareSet.SetUserAgentMiddleware': 400, # 设置useragent 35 | # 'downloaderMiddlewareSet.SetHeaderMiddleware': 550, 36 | 'downloaderMiddlewareSet.SetProxyMiddleware': 750, # 设置代理 37 | # 'downloaderMiddlewareSet.SetUtf8Middleware': 1000, # 最开始设置字符集过滤用的,不用开启 38 | }) 39 | 40 | settings.set("LOG_STDOUT ", False) 41 | # settings.set("CONCURRENT_REQUESTS", 20) 42 | settings.set("COOKIES_ENABLED", False) # cookies追踪,采集新闻站可以关闭 43 | settings.set("RETRY_ENABLED", False) # 设置重试,如果开启默认重试3次 44 | settings.set("REDIRECT_ENABLED", False) # 重定向类网站是否采集 45 | # settings.set("AJAXCRAWL_ENABLED", True) 46 | settings.set("DOWNLOAD_DELAY", 3) # 延迟下载 47 | settings.set("DOWNLOAD_TIMEOUT", 15) # 下载超时的阈值,超过15秒就关闭连接 48 | 49 | # 图片处理 50 | # settings.set("IMAGES_STORE", "http://qiniu") # 七牛云图片 51 | settings.set("IMAGES_STORE", img_save_path) # 图片下载路径配置 52 | # settings.set("IMAGES_MIN_HEIGHT", "80") # 图片最小高度 小于则不下载 53 | # settings.set("IMAGES_MIN_WIDTH", "80") # 图片最小宽度 小于则不下载 54 | 55 | settings.set("TELNETCONSOLE_ENABLED", False) 56 | # 配置日志记录规则设置 57 | configure_logging(install_root_handler=False) 58 | # configure_logging() 59 | # 初始化日志路径 60 | 61 | if log_open is True: 62 | settings.set("LOG_LEVEL", 'INFO') 63 | logpath = datetime.now().strftime(log_path) 64 | if not os.path.isdir(logpath): 65 | os.makedirs(logpath) 66 | logging.basicConfig( 67 | filename=datetime.now().strftime('%s/%s_spider.log' % (logpath, log_file)), 68 | format=log_format, 69 | level=logging.INFO 70 | ) 71 | 72 | # 拼装爬虫 73 | process = CrawlerProcess(settings) 74 | # 简单的过滤规则 支持sql 可以参看sql demo 75 | # 新增数据 76 | # sql = text('insert into users (u_name, u_password) values (:name, :password)') 77 | # data = db.execute(sql, {'name': 'nate1', 'password': password}) 78 | # 删除数据 79 | # sql = text('delete from users where u_id = :id') 80 | # data = session.execute(sql, {'id': last_id}) 81 | # sql = text('select * from users') 82 | # data = session.execute(sql) 83 | rules = db.query(Rule).filter(Rule.enable > 0).all() 84 | 85 | for rule in rules: 86 | process.crawl(RuleSpider, rule) 87 | process.start() 88 | 89 | if __name__ == '__main__': 90 | run_spider() 91 | -------------------------------------------------------------------------------- /webspider/search/__init__.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /webspider/search/search._py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | import sys 4 | from whoosh.index import open_dir 5 | from whoosh.fields import * 6 | from whoosh.qparser import QueryParser, MultifieldParser 7 | from jieba.analyse import ChineseAnalyzer 8 | from whoosh import scoring, sorting 9 | from whoosh.query import * 10 | # import cgi 11 | import HTMLParser 12 | 13 | # sys.path.append("../") 14 | 15 | # analyzer = ChineseAnalyzer() 16 | ix = open_dir("./sindex") # for read only 17 | 18 | # inputstring = "中国and(国际or物流)and开幕" 19 | # inputstring = "12 string english" 20 | # inputstring = "中国国际物流节开幕" 21 | # keywords = [] 22 | # for t in analyzer(inputstring): 23 | # keywords.append(t.text) 24 | # keywords = inputstring.split(" ") 25 | # keywordstr = " ".join(keywords) 26 | 27 | # searcher = ix.searcher(weighting=scoring.TF_IDF()) 28 | # parser = QueryParser("body", schema=ix.schema) 29 | 30 | # for keyword in keywords: 31 | # print("result of %s" % keyword) 32 | # print(keyword) 33 | # q = parser.parse(keyword) 34 | # results = searcher.search(q, terms=True) 35 | # for hit in results: 36 | # print(hit['title']) 37 | # print("="*10) 38 | 39 | # qp = QueryParser("title", schema=ix.schema) 40 | # qp = MultifieldParser(["title", "body"], schema=ix.schema) 41 | # with ix.searcher(weighting=scoring.TF_IDF()) as searcher: 42 | # # q = qp.parse(keywordstr) 43 | # # terms_list = [terms for terms in q.terms()] 44 | # # print(terms_list) 45 | # # querystring = Or(terms_list) 46 | # # querystring = And([Term("title", "中国"), Term("title", "国际"), Term("title", "物流"), Term("title", "国际物流节"), Term("title", "开幕")]) 47 | # # print(q) 48 | # # querystring = Or([Term("body", word) for word in keywords]) 49 | # querystring = qp.parse(inputstring) 50 | # results = searcher.search(querystring, terms=True, limit=100, collapse_limit=1) 51 | # return_list = [] 52 | # ret = [] 53 | # for i in xrange(len(results)): 54 | # tmpret = results[i].fields() 55 | # tmpret['words'] = results[i].matched_terms() 56 | # ret.append(tmpret) 57 | # print ret 58 | 59 | 60 | def search(inputstring=None, page=1, size=10): 61 | inputstring = unicode(inputstring) 62 | publish_time = sorting.FieldFacet("publish_time", reverse=True) 63 | qp = MultifieldParser(["title", "body"], schema=ix.schema) 64 | with ix.searcher(weighting=scoring.TF_IDF()) as searcher: 65 | querystring = qp.parse(inputstring) 66 | results = searcher.search(querystring, terms=True, limit=None, sortedby=[publish_time]) 67 | # results = searcher.search_page(querystring, page) 68 | res = [] 69 | html_parser = HTMLParser.HTMLParser() 70 | if len(results) > 0: 71 | for i in xrange((page-1)*size, page*size): 72 | # for i in xrange(len(results)): 73 | # tmpret = results[i].fields() 74 | # # tmpret = {} 75 | # tmpret['words'] = results[i].matched_terms() 76 | # ret.append(tmpret) 77 | if i in xrange(len(results)): 78 | tmpret = results[i].fields() 79 | hit_keywords = set() 80 | for key, val in results[i].matched_terms(): 81 | hit_keywords.add(val.decode('utf-8')) 82 | tmpret['words'] = results[i].matched_terms() 83 | tmpret['hit_keywords'] = " ".join(list(hit_keywords)) 84 | tmpret['hit_title'] = html_parser.unescape(results[i].highlights('title').decode('utf-8')).decode('unicode_escape') if results[i].highlights('title') else results[i]['title'] 85 | print html_parser.unescape(results[i].highlights('title').encode('utf-8')) 86 | # tmpret['hit_body'] = results[i].highlights('body') 87 | res.append(tmpret) 88 | page_info = {'total': len(results), 'page': page, 'size': size, 'data': res} 89 | return page_info 90 | 91 | 92 | # def search_page(inputstring=None, page=1, size=10): 93 | # qp = MultifieldParser(["title", "body"], schema=ix.schema) 94 | # with ix.searcher(weighting=scoring.TF_IDF()) as searcher: 95 | # querystring = qp.parse(inputstring) 96 | # # results = searcher.search(querystring, terms=True, limit=None) 97 | # results = searcher.search_page(querystring, page, size) 98 | # ret = [] 99 | # print(results) 100 | # # for i in xrange(size): 101 | # # tmpret = results[i].fields() 102 | # # # tmpret = {} 103 | # # tmpret['words'] = results[i].matched_terms() 104 | # # ret.append(tmpret) 105 | # page_info = {} 106 | # page_info['total'] = results.docnum() 107 | # page_info['page'] = page 108 | # page_info['size'] = size 109 | # page_info['data'] = ret 110 | # return page_info 111 | 112 | if __name__ == '__main__': 113 | ret = search("四川") 114 | print(ret) 115 | -------------------------------------------------------------------------------- /webspider/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for webspider project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 10 | # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'webspider' 13 | 14 | SPIDER_MODULES = ['webspider.spiders'] 15 | NEWSPIDER_MODULE = 'webspider.spiders' 16 | 17 | 18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 19 | #USER_AGENT = 'webspider (+http://www.yourdomain.com)' 20 | 21 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 22 | #CONCURRENT_REQUESTS=32 23 | 24 | # Configure a delay for requests for the same website (default: 0) 25 | # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay 26 | # See also autothrottle settings and docs 27 | #DOWNLOAD_DELAY=3 28 | # The download delay setting will honor only one of: 29 | #CONCURRENT_REQUESTS_PER_DOMAIN=16 30 | #CONCURRENT_REQUESTS_PER_IP=16 31 | 32 | # Disable cookies (enabled by default) 33 | #COOKIES_ENABLED=False 34 | 35 | # Disable Telnet Console (enabled by default) 36 | #TELNETCONSOLE_ENABLED=False 37 | 38 | # Override the default request headers: 39 | #DEFAULT_REQUEST_HEADERS = { 40 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 41 | # 'Accept-Language': 'en', 42 | #} 43 | 44 | # Enable or disable spider middlewares 45 | # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 46 | #SPIDER_MIDDLEWARES = { 47 | # 'webspider.middlewares.MyCustomSpiderMiddleware': 543, 48 | #} 49 | 50 | # Enable or disable downloader middlewares 51 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 52 | #DOWNLOADER_MIDDLEWARES = { 53 | # 'webspider.middlewares.MyCustomDownloaderMiddleware': 543, 54 | #} 55 | 56 | # Enable or disable extensions 57 | # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html 58 | #EXTENSIONS = { 59 | # 'scrapy.telnet.TelnetConsole': None, 60 | #} 61 | 62 | # Configure item pipelines 63 | # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html 64 | ITEM_PIPELINES = { 65 | # 'webspider.pipelines.CoverImagesPipeline': 1, 66 | } 67 | # ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1} 68 | # IMAGES_STORE = 'http://qiniu' 69 | # IMAGES_EXPIRES = 0 70 | # IMAGES_MIN_HEIGHT = 80 # 图片的最小高度 71 | # IMAGES_MIN_WIDTH = 80 # 图片的最小宽度 72 | 73 | # Enable and configure the AutoThrottle extension (disabled by default) 74 | # See http://doc.scrapy.org/en/latest/topics/autothrottle.html 75 | # NOTE: AutoThrottle will honour the standard settings for concurrency and delay 76 | #AUTOTHROTTLE_ENABLED=True 77 | # The initial download delay 78 | #AUTOTHROTTLE_START_DELAY=5 79 | # The maximum download delay to be set in case of high latencies 80 | #AUTOTHROTTLE_MAX_DELAY=60 81 | # Enable showing throttling stats for every response received: 82 | #AUTOTHROTTLE_DEBUG=False 83 | 84 | # Enable and configure HTTP caching (disabled by default) 85 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 86 | #HTTPCACHE_ENABLED=True 87 | #HTTPCACHE_EXPIRATION_SECS=0 88 | #HTTPCACHE_DIR='httpcache' 89 | #HTTPCACHE_IGNORE_HTTP_CODES=[] 90 | #HTTPCACHE_STORAGE='scrapy.extensions.httpcache.FilesystemCacheStorage' 91 | -------------------------------------------------------------------------------- /webspider/spiders/ImgSpider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | from scrapy.linkextractors import LinkExtractor 4 | from scrapy.spiders import CrawlSpider, Rule 5 | from item.ImgItem import ImgItem 6 | import urlparse 7 | 8 | 9 | class ImgSpider(CrawlSpider): 10 | name = 'imgspider' 11 | start_urls = ['http://news.163.com/15/1221/11/BBBS3IAG00014AED.html#f=dlist'] 12 | # start_urls = ['http://www.cocloud.net/2013/09/10/scrapy_img_spider.html'] 13 | image_names = {} 14 | 15 | def parse(self, response): 16 | item = ImgItem() 17 | # item['image_urls'] = response.xpath("//div[@class='container']//img/@src").extract() 18 | # item['img_urls'] = response.xpath("//div[@id='endText']//img/@src").extract() 19 | # for index, value in enumerate(item['image_urls']): 20 | # number = self.start_urls.index(response.url) * len(item['image_urls']) + index 21 | # self.image_names[value] = 'full/%04d.jpg' % number 22 | # image_urls = response.xpath(self.rule.body_xpath + "//img/@src").extract() 23 | body = response.xpath("//div[@id='endText']").extract() 24 | item['body'] = body[0] 25 | image_urls = response.xpath("//div[@id='endText']//img/@src").extract() 26 | # article["img_urls"]["original"] = [] 27 | # article["img_urls"]["real"] = [] 28 | item["img_urls"] = {"original": [], "real": []} 29 | if image_urls: 30 | for img in image_urls: 31 | item["img_urls"]["original"].append(img) 32 | item["img_urls"]["real"].append(urlparse.urljoin(response.url, img)) 33 | else: 34 | item["img_urls"] = None 35 | yield item 36 | -------------------------------------------------------------------------------- /webspider/spiders/ProxySpider.py: -------------------------------------------------------------------------------- 1 | #! python 2 | # -*- coding: utf-8 -*- 3 | from pyquery import PyQuery as pyQuery 4 | from scrapy.spiders import Spider 5 | 6 | from item.ProxyItem import ProxyItem 7 | 8 | 9 | class GetProxySpider(Spider): 10 | name = "get_proxy" 11 | allowed_domains = ["proxynova.com"] 12 | start_urls = ( 13 | 'http://www.proxynova.com/proxy-server-list/country-cn/', 14 | 'http://www.proxynova.com/proxy-server-list/elite-proxies/', 15 | 'http://www.proxynova.com/proxy-server-list/anonymous-proxies/' 16 | ) 17 | 18 | def parse(self, response): 19 | sel = pyQuery(response.body) 20 | sites = sel('#tbl_proxy_list > tbody:eq(0) > tr') 21 | proxy = ProxyItem() 22 | for row in sites: 23 | dom = pyQuery(row) 24 | if dom('td center div'): 25 | continue 26 | else: 27 | ip = dom('td:eq(0) span').text() 28 | proxy["ip"] = ip.strip() if ip else "" 29 | port = dom('td:eq(1)').text() 30 | proxy["port"] = port.strip() if port else "" 31 | yield proxy 32 | 33 | -------------------------------------------------------------------------------- /webspider/spiders/RuleSpider.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | import sys 4 | import urlparse 5 | from scrapy.linkextractors import LinkExtractor 6 | from scrapy.spiders import CrawlSpider, Rule 7 | from scrapy.selector import Selector 8 | from item.ArticleItem import ArticleItem as Article 9 | import re 10 | 11 | 12 | class RuleSpider(CrawlSpider): 13 | def __init__(self, rule): 14 | self.rule = rule 15 | self.name = rule.name 16 | self.allowed_domains = rule.allow_domains.split(",") 17 | self.weburl_id = rule.weburl_id 18 | self.rule_id = rule.rule_id 19 | self.thumb_img_xpath = rule.thumb_img_xpath 20 | self.start_urls = rule.start_urls.split(",") 21 | rule_list = [] 22 | if rule.next_page: 23 | rule_list.append(Rule(LinkExtractor(restrict_xpaths=rule.next_page))) 24 | rule_list.append(Rule(LinkExtractor(allow=[rule.allow_url], restrict_xpaths=[rule.extract_from]), 25 | callback='parse_item')) 26 | self.rules = tuple(rule_list) 27 | super(RuleSpider, self).__init__() 28 | 29 | def parse_item(self, response): 30 | reload(sys) 31 | sys.setdefaultencoding("GBK") 32 | article = Article() 33 | article["url"] = response.url 34 | # 本身的html标题 35 | html_title = response.xpath("//title/text()").extract() 36 | article["html_title"] = html_title[0] if html_title else "" 37 | article["html_body"] = response.body 38 | # 标题 39 | try: 40 | title = response.xpath(self.rule.title_xpath).extract() 41 | article["title"] = title[0] if title else "" 42 | except Exception: 43 | article["title"] = "" 44 | # 内容 45 | try: 46 | # html_filter = re.sub(response.xpath("//iframe"), "", response.body) 47 | body = response.xpath(self.rule.body_xpath).extract() 48 | # article["body"] = "" 49 | # for texts in len(body): 50 | # article["body"] += re.sub(Selector(text=texts).xpath("//iframe"), "", texts) 51 | article["body"] = '\n'.join(body) if body else "" 52 | except Exception: 53 | article["body"] = "" 54 | # 发布时间 55 | try: 56 | publish_time = response.xpath(self.rule.publish_time_xpath).extract() 57 | article["publish_time"] = publish_time[0] if publish_time else "" 58 | except Exception: 59 | article["publish_time"] = "" 60 | # 来源 61 | try: 62 | source_site = response.xpath(self.rule.source_site_xpath).extract() 63 | article["source_site"] = source_site[0] if source_site else "" 64 | except Exception: 65 | article["source_site"] = "" 66 | 67 | # 图片 68 | image_urls = response.xpath(self.rule.body_xpath + "//img/@src").extract() 69 | # article["img_urls"]["original"] = [] 70 | # article["img_urls"]["real"] = [] 71 | article["img_urls"] = {"original": [], "real": []} 72 | if image_urls: 73 | for img in image_urls: 74 | article["img_urls"]["original"].append(img) 75 | article["img_urls"]["real"].append(urlparse.urljoin(response.url, img)) 76 | else: 77 | article["img_urls"] = None 78 | article["thumb"] = "" 79 | if article["title"] == "" or article["body"] == "": 80 | return None 81 | else: 82 | return article 83 | -------------------------------------------------------------------------------- /webspider/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /webspider/spiders/item/ArticleItem.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | import scrapy 4 | 5 | 6 | class ArticleItem(scrapy.Item): 7 | url = scrapy.Field() 8 | html_title = scrapy.Field() 9 | html_body = scrapy.Field() 10 | title = scrapy.Field() 11 | body = scrapy.Field() 12 | publish_time = scrapy.Field() 13 | source_site = scrapy.Field() 14 | img_urls = scrapy.Field() 15 | thumb = scrapy.Field() 16 | img_list = scrapy.Field() 17 | -------------------------------------------------------------------------------- /webspider/spiders/item/ImgItem.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from scrapy import Item, Field 3 | 4 | 5 | class ImgItem(Item): 6 | img_urls = Field() 7 | images = Field() 8 | image_paths = Field() 9 | or_url = Field() 10 | body = Field() 11 | first_img = Field() 12 | -------------------------------------------------------------------------------- /webspider/spiders/item/ProxyItem.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | from scrapy import Item, Field 4 | 5 | 6 | class ProxyItem(Item): 7 | ip = Field() 8 | port = Field() 9 | -------------------------------------------------------------------------------- /webspider/spiders/item/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __mtime__ = '2015/12/18' 5 | -------------------------------------------------------------------------------- /webspider/util/UserAgent.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import random 4 | 5 | # Several different User-Agents to diversify the requests. 6 | # Keep the User-Agents updated. Last update: 13th November 2014 7 | # Get them here: http://techblog.willshouse.com/2012/01/03/most-common-user-agents/ 8 | 9 | user_agents = [ 10 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 11 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 12 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5', 13 | 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', 14 | 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 15 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 16 | 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', 17 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0', 18 | 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 19 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/7.1.2 Safari/537.85.11', 20 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:34.0) Gecko/20100101 Firefox/34.0', 21 | 'Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0', 22 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 23 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.1.25 (KHTML, like Gecko) QuickLook/5.0', 24 | 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', 25 | 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 26 | 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 27 | 'Mozilla/5.0 (Windows NT 5.1; rv:34.0) Gecko/20100101 Firefox/34.0', 28 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25', 29 | 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)', 30 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36', 31 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/7.1 Safari/537.85.10', 32 | 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', 33 | 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0', 34 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.59.10 (KHTML, like Gecko) Version/5.1.9 Safari/534.59.10', 35 | 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', 36 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25', 37 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36', 38 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 39 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 40 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/6.1.6 Safari/537.78.2', 41 | 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 42 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 43 | 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 44 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.10 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.10', 45 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 46 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36', 47 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/7.0.6 Safari/537.78.2', 48 | 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0', 49 | 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko', 50 | 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 51 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:34.0) Gecko/20100101 Firefox/34.0', 52 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36', 53 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.45 Safari/537.36', 54 | 'Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0', 55 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:34.0) Gecko/20100101 Firefox/34.0', 56 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36', 57 | 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', 58 | 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', 59 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36', 60 | 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36', 61 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 62 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/6.2.2 Safari/537.85.11', 63 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:35.0) Gecko/20100101 Firefox/35.0', 64 | 'Mozilla/5.0 (Windows NT 6.1; rv:35.0) Gecko/20100101 Firefox/35.0'] 65 | 66 | 67 | def get_user_agent(): 68 | return random.choice(user_agents) 69 | -------------------------------------------------------------------------------- /webspider/util/WebProxy.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | 主要是封装了代理的访问,切换,添加删除等操作 5 | """ 6 | import requests 7 | import logging 8 | from sqlalchemy import exc 9 | import UserAgent as userAgent 10 | from conf.config import DBSession, check_proxy_url 11 | from model.Proxy import Proxy 12 | 13 | 14 | def get_proxy_list(): 15 | """ 获取代理列表""" 16 | db = DBSession() 17 | proxy_list = db.query(Proxy).all() 18 | db.close() 19 | return proxy_list 20 | 21 | 22 | def get_proxy_url(): 23 | """获取一个代理链接""" 24 | proxy_list = get_proxy_list() 25 | proxy_url = "http://%s:%s" % (proxy_list[0]['ip'], proxy_list[0]['port']) 26 | return proxy_url 27 | 28 | 29 | # def create_proxy(timeout=3): 30 | # """ 31 | # 爬虫抓取代理 32 | # 返回一个dict类型 timeout header 和proxy 33 | # 返回前检查一下代理是否能用 34 | # 不能用则删除并重新换一个代理 35 | # :param timeout: 超时时间 36 | # """ 37 | # proxy_list = get_proxy_list() 38 | # proxy = choice(proxy_list) 39 | # if check_proxy(ip=proxy['ip'], port=proxy['port']): 40 | # request = {'proxies': proxy, 'header': get_user_agent(), 'timeout': timeout} 41 | # return request 42 | # else: 43 | # # 删除代理ip 且重新生成一个 44 | # delete_proxy(proxy['ip'], proxy['port']) 45 | # request = create_proxy(timeout) 46 | # return request 47 | 48 | 49 | def get_user_agent(): 50 | """ 获取头协议 """ 51 | user_agent = {'User-Agent': userAgent.get_user_agent()} 52 | return user_agent 53 | 54 | 55 | def check_proxy(ip, port): 56 | """ 检查代理是否正常 57 | :param ip: 代理IP 58 | :param port: 代理端口 59 | """ 60 | proxy_check_url = check_proxy_url if check_proxy_url else 'http://www.baidu.com' 61 | 62 | proxy_url = "%s:%s" % (ip, port) 63 | proxies = {"http": proxy_url, } 64 | header = get_user_agent() 65 | try: 66 | req = requests.get(proxy_check_url, proxies=proxies, timeout=3, headers=header) 67 | if req.status_code == requests.codes.ok: 68 | return True 69 | else: 70 | return False 71 | except Exception, e: 72 | return False 73 | 74 | 75 | def delete_proxy(ip, port): 76 | """ 删除数据库中的代理 77 | :param ip: ip 78 | :param port: 端口 79 | """ 80 | if ip != "" and port != "": 81 | db = DBSession() 82 | db.query(Proxy).filter(Proxy.ip == ip).filter(Proxy.port == port).delete() 83 | try: 84 | db.commit() 85 | return True 86 | except exc.SQLAlchemyError, e: 87 | logging.info("Delete Proxy Error:", format(e)) 88 | return False 89 | -------------------------------------------------------------------------------- /webspider/util/__init__.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /webspider/util/common.py: -------------------------------------------------------------------------------- 1 | # python 2 | # -*- coding: utf-8 -*- 3 | import hashlib 4 | import datetime 5 | from urlparse import urlparse 6 | import logging 7 | import os 8 | import re 9 | 10 | 11 | def md5dir(url): 12 | """ 13 | 根据URL md5 返回文件路径 14 | 规则:时间/站点/md5(1-2)/md5(3-4)/md5(5-6)/md5 15 | :param url: 链接 16 | """ 17 | md5url = hashlib.md5(url).hexdigest() 18 | datestr = datetime.datetime.now().strftime('%Y_%m_%d') 19 | site = urlparse(url) 20 | part1 = md5url[0:2] 21 | # part2 = md5url[2:4] 22 | # part3 = md5url[4:6] 23 | # filepath = "%s/%s/%s/%s/%s/%s.html" % (datestr, site.hostname, part1, part2, part3, md5url) 24 | filepath = "%s/%s/%s/%s.html" % (datestr, site.hostname, part1, md5url) 25 | return filepath 26 | 27 | 28 | def get_site(url): 29 | site = urlparse(url) 30 | return site.hostname 31 | 32 | 33 | def save_file(dirname, data): 34 | """ save file to local 35 | :param dirname: 路径 36 | :param data: 数据 37 | """ 38 | try: 39 | path, file_name = os.path.split(dirname) 40 | if not os.path.isdir(path): 41 | os.makedirs(path) 42 | filename = open(dirname, 'w') 43 | filename.write(data) 44 | except Exception, e: 45 | logging.error("SaveFileError:%s,%s" % (format(e), dirname)) 46 | finally: 47 | filename.close() 48 | 49 | 50 | def smart2date(date_string): 51 | """ 52 | 网易搜狐的时间解析字符串,转化成标准时间 53 | 如果解析失败,返回当前时间 54 | 样例: 55 | 2015/12/20 12:11:56 56 | 2015\12\20 12:11:56 57 | 2015-12-20 15:43:50 58 | 2015年12月20日 12时00分00秒 59 | """ 60 | # re_data = r"(\d{1,4}[-|\/|年]\d{1,2}[-|\/|月]\d{1,2}[\s|日]+\\d{1,2}:\d{1,2}:\d{1,2})" 61 | # re_data = re.compile("(\d{1,4}[-|\/|年])(\d{1,2}[-|\/|月])(\d{1,2}[\s|日]+)(\d{1,2}[\:|时]+)(\d{1,2}[\:|分]+)(\d{1,2}[秒]?)") 62 | # re_data = re.compile("\d{1,4}[-|\/|年]\d{1,2}[-|\/|月]\d{1,2}[\s|日]+\d{1,2}[\:|时]+\d{1,2}[\:|分]+\d{1,2}[秒]?") 63 | 64 | # 中文智能化匹配在python中受unicode影响所以用if条件分支处理 65 | re_en = re.compile("(\d{1,4})[-\/](\d{1,2})[-\/](\d{1,2})[\s]+(\d{1,2}):(\d{1,2}):(\d{1,2})") 66 | re_cn = re.compile(u"(\d{1,4})年(\d{1,2})月(\d{1,2})日.?(\d{1,2})时(\d{1,2})分(\d{1,2})秒") 67 | re_d_en = re.compile("(\d{1,4})[-\/](\d{1,2})[-\/](\d{1,2})") 68 | re_d_cn = re.compile(u"(\d{1,4})年(\d{1,2})月(\d{1,2})日") 69 | match = re_en.search(date_string) 70 | # 极其愚蠢的正则 71 | if match: 72 | date_str = match.group(0) 73 | else: 74 | match = re_cn.search(date_string) 75 | if match: 76 | date_str = match.group(0) 77 | else: 78 | match = re_d_en.search(date_string) 79 | if match: 80 | date_str = match.group(0) 81 | else: 82 | match = re_d_cn.search(date_string) 83 | if match: 84 | date_str = match.group(0) 85 | else: 86 | date_str = "" 87 | return date_str 88 | 89 | # re_data = re.compile("\d{1,4}.+\d{1,2}.+\d{1,2}.+\d{1,2}[\:|时]+\d{1,2}[\:|分]+\d{1,2}秒?") 90 | # match = re_data.search(date_string) 91 | # if match: 92 | # print match.group() 93 | # else: 94 | # print "None" 95 | 96 | if __name__ == '__main__': 97 | print smart2date(u"2016年02月16日 10:09") 98 | print smart2date(u"2016-02-16 17:30") 99 | pass 100 | # print smart2date("2015年12月20日 12时00分00秒 中国") 101 | # print smart2date("2015-12-20 15:43:50") 102 | # re_data = re.compile(u"20\d{1,2}[-|/|\\|年]\d{1,2}[-|/|\\|月]\d{1,2}日?\s+\d{1,2}[:|时]\d{1,2}[:|分]\d{1,2}秒?") 103 | # re_data = re.compile("((\d{1,4})[-\/](\d{1,2})[-\/](\d{1,2})[\s]+(\d{1,2}):(\d{1,2}):(\d{1,2}))") 104 | # re_data = re.compile("((\d{1,4})年(\d{1,2})月(\d{1,2})日\s+(\d{1,2})时(\d{1,2})分(\d{1,2})秒)") 105 | # re_data = re.compile(u"((\d{1,4})[-\/年](\d{1,2})[-\/月](\d{1,2})[日\s]+(\d{1,2})[:时](\d{1,2})[:分](\d{1,2})[秒])") 106 | # date_string = u"2015-12-20 15:43:50" 107 | # # date_string = u"2015年12月20日 12时00分00秒 " 108 | # match = re_data.match(date_string) 109 | # if match: 110 | # print match.groups() 111 | # else: 112 | # print "None" 113 | # string = "2015年12月20日 12时00分00秒 " 114 | # if "年" in string && "月" in string && "日" in string: 115 | # print(1) 116 | # else: 117 | # print(0) 118 | --------------------------------------------------------------------------------