├── .gitattributes ├── README.md ├── crawler_flask.py ├── model.py ├── requirements.txt ├── static ├── css │ ├── bootstrap.min.css │ └── styles.css ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ └── glyphicons-halflings-regular.woff └── js │ ├── bootstrap.min.js │ ├── jquery-1.11.1.min.js │ └── lumino.glyphs.js └── templates ├── images.html ├── indent.html ├── index.html ├── links.html └── text.html /.gitattributes: -------------------------------------------------------------------------------- 1 | special-vendored-path/* linguist-vendored 2 | /static/js/lumino.glyphs.js linguist-vendored 3 | /static/css/styles.css linguist-vendored 4 | /static/css/bootstrap.min.css linguist-vendored 5 | /templates/*.html linguist-vendored 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # webpage-scraper 2 | `webpage-scraper` is a *flask* based application which allows the users to : 3 | 4 | - Input URL with the freedom of inputting it with/ without the protocol and sub-domain specifiers. 5 | - Fetch a list of URLs to all the images on the webpage with an option to download all the images in a directory with name specified by the user. 6 | - Get a list of all the hyperlinks on the webpage. Save them into a text file with a name specified by the user. 7 | - Get the indented html source code of the webpage and save it in a .html file with a user-provided name. 8 | - Fetch the text on the webpage stripping the html code. Save it in a text file with a filename of user's choice. 9 | - The database is deployed on [mLab](http://mlab.com/) and uses *MongoDB* for fast access to long list of images, hyperlinks and text for a URL that has been requested by some other user in the past, thus, reducing processing time for subsequent users. 10 | 11 | ## Pre- requisites 12 | 13 | To install requirements: 14 | 15 | ``` 16 | [sudo] pip install requirements 17 | ``` 18 | 19 | If you don't have [pip](https://pip.pypa.io) installed, [this Python installation guide](http://docs.python-guide.org/en/latest/starting/installation) can guide you through the process. 20 | 21 | 22 | To install MongoDB Community Edition: 23 | 24 | - on OSX, refer to: 25 | https://docs.mongodb.com/manual/tutorial/install-mongodb-on-os-x/ 26 | 27 | - on Ubuntu, refer to: 28 | https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/ 29 | 30 | - on Windows, refer to: 31 | https://docs.mongodb.com/manual/tutorial/install-mongodb-on-windows/ 32 | 33 | 34 | **Make sure you have MongoDB installed** 35 | 36 | ## Getting started 37 | ``` 38 | git clone http://github.com/mansimarkaur/webpage-scraper 39 | cd webpage-scraper 40 | python crawler_flask.py 41 | ``` 42 | 43 | Open **http://127.0.0.1:5000/** in your browser. 44 | Input URL and **have fun** :+1: 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /crawler_flask.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import urllib 3 | import os 4 | import sys 5 | from flask import Flask, flash, request, render_template 6 | from bs4 import BeautifulSoup 7 | from flask_mongoalchemy import MongoAlchemy 8 | 9 | crawler = Flask(__name__) 10 | crawler.secret_key = 'continents9794' 11 | 12 | from model import * 13 | #from model import text_db 14 | 15 | @crawler.route("/") 16 | def main() : 17 | return render_template("index.html") 18 | 19 | @crawler.route("/driver", methods = ['POST']) 20 | def driver() : 21 | global url 22 | url = request.form["inputName"] #fetches user entered URL 23 | print url 24 | if not(url.startswith("http://") or url.startswith("https://")) : 25 | if not url.startswith("www.") : 26 | url = "www." + url 27 | url = "http://" + url 28 | if not url.endswith("/") : 29 | url = url + "/" 30 | print url 31 | try : 32 | link = urllib2.urlopen(url) #returns obj 33 | except : 34 | flash('Webpage did not return a OK status') 35 | return render_template("index.html") 36 | global soup 37 | soup = BeautifulSoup(link, 'html.parser') #returns beautifulsoup obj 38 | job = request.form["submit"] 39 | dict = { 40 | "images" : image, 41 | "hyperlinks" : hyperlinks, 42 | "text" : text, 43 | "formatter" : formatter 44 | } 45 | func = dict.get(job) #calls function acc to button pressed 46 | return func() 47 | 48 | 49 | #displays image URLs 50 | @crawler.route("/download_images", methods = ['POST']) 51 | def image() : 52 | images = images_db.query.images_query(url).first() 53 | if images == None : 54 | img = soup.find_all("img") #finds all 55 | if len(img) == 0 : 56 | return render_template("index.html", text = ["No images to fetch"]) 57 | images = [] 58 | for i in img : 59 | images.append(url + i.get("src")) #adds src attribute value to images list 60 | images_db(sites = url, images = images).save() 61 | else : 62 | images = images.images 63 | try : 64 | to_download = bool(request.form['submit']) 65 | dir_name = request.form['name'] 66 | except : 67 | to_download = False 68 | if to_download : 69 | download(images, dir_name) 70 | flash('Download completed') 71 | return render_template("images.html", text = images) 72 | 73 | 74 | def download(image, dir_name) : 75 | try : 76 | os.stat(dir_name) 77 | except : 78 | os.mkdir(dir_name) 79 | for i in image : 80 | name = i[i.rfind("/")+1:] 81 | if name == -1 : 82 | name = i 83 | print dir_name+'/'+name 84 | urllib.urlretrieve(i, dir_name+'/'+name) 85 | 86 | #displays hyperlinks 87 | @crawler.route('/download_links', methods = ['POST']) 88 | def hyperlinks() : 89 | links = links_db.query.links_query(url).first() 90 | if links == None : 91 | print "me" 92 | link = soup.find_all("a") #finds all 93 | if len(link) == 0 : 94 | return render_template("index.html", text = ["No links to fetch"]) 95 | links = [] 96 | for i in link : 97 | l = i.get("href") #adds href attribute value to links list 98 | if l[:4] != "http" : 99 | l = url + l 100 | links.append(l) 101 | links_db(sites = url, links = links).save() 102 | else : 103 | links = links.links 104 | try : 105 | to_download = bool(request.form['submit']) 106 | file_name = request.form['name'] 107 | except : 108 | to_download = False 109 | if to_download : 110 | with open(file_name + ".txt", "w+") as getlink : 111 | for i in links : 112 | getlink.write(i) 113 | flash('Download completed') 114 | return render_template("links.html", text = links) 115 | 116 | #displays text after stripping html tags from src code 117 | @crawler.route('/text', methods = ['POST']) 118 | def text() : 119 | t = text_db.query.text_query(url).first() 120 | if t == None : 121 | t = soup.get_text()#.encode('UTF-8') 122 | text_db(sites = url, text = t).save() 123 | else : 124 | t = t.text 125 | try : 126 | to_download = bool(request.form['submit']) 127 | file_name = request.form['name'] 128 | except : 129 | to_download = False 130 | if to_download : 131 | with open(file_name + ".txt", "w+") as pretty : 132 | pretty.write(t.encode('UTF-8')) 133 | flash('Download completed') 134 | return render_template("text.html", text = t) 135 | 136 | #displays formatted html src code 137 | @crawler.route('/download_code', methods = ['POST']) 138 | def formatter() : 139 | code = indent_db.query.indent_query(url).first() 140 | if code == None : 141 | code = soup.prettify()#.encode('UTF-8') 142 | indent_db(sites = url, indent = code).save() 143 | else : 144 | code = code.indent 145 | try : 146 | to_download = bool(request.form['submit']) 147 | file_name = request.form['name'] 148 | except : 149 | to_download = False 150 | if to_download : 151 | with open(file_name + ".html", "w+") as source_code : 152 | source_code.write(code.encode('UTF-8')) 153 | flash('Download completed') 154 | return render_template("indent.html", text = code) 155 | 156 | 157 | 158 | if __name__ == "__main__" : 159 | crawler.run() 160 | 161 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | from flask_mongoalchemy import MongoAlchemy, BaseQuery 2 | from flask import Flask 3 | 4 | crawler = Flask(__name__) 5 | crawler.secret_key = 'continents9794' 6 | crawler.config['MONGOALCHEMY_DATABASE'] = 'website-scraper' 7 | crawler.config['MONGOALCHEMY_CONNECTION_STRING'] = 'mongodb://scraper:continents@ds023495.mlab.com:23495/website-scraper' 8 | 9 | db = MongoAlchemy(crawler) 10 | 11 | class db_query(BaseQuery) : 12 | def text_query(self, url) : 13 | return self.filter(self.type.sites == url).fields('text') 14 | 15 | def images_query(self, url) : 16 | return self.filter(self.type.sites == url).fields('images') 17 | 18 | def links_query(self, url) : 19 | return self.filter(self.type.sites == url).fields('links') 20 | 21 | def indent_query(self, url) : 22 | return self.filter(self.type.sites == url).fields('indent') 23 | 24 | class images_db(db.Document) : 25 | query_class = db_query 26 | sites = db.StringField() 27 | images = db.ListField(db.StringField()) 28 | 29 | class links_db(db.Document) : 30 | query_class = db_query 31 | sites = db.StringField() 32 | links = db.ListField(db.StringField()) 33 | 34 | class indent_db(db.Document) : 35 | query_class = db_query 36 | sites = db.StringField() 37 | indent = db.StringField() 38 | 39 | class text_db(db.Document) : 40 | query_class = db_query 41 | sites = db.StringField() 42 | text = db.StringField() 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==0.11 2 | requests==2.10.0 3 | BeautifulSoup4==4.4.0 4 | flask_mongoalchemy==0.7.2 -------------------------------------------------------------------------------- /static/css/styles.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | background: #f1f4f7; 4 | padding-top: 50px; 5 | color: #5f6468; 6 | } 7 | 8 | #foot-er{ 9 | background-color: #222; 10 | width: 100%; 11 | bottom: 0px; 12 | } 13 | 14 | .row{ 15 | margin-bottom: 7%; 16 | margin-top: 8%; 17 | } 18 | 19 | #panel{ 20 | float: none; 21 | margin-left: 18%; 22 | } 23 | 24 | #hyperlinks{ 25 | padding-left: 4px !important; 26 | } 27 | 28 | p { 29 | color: #777; 30 | } 31 | 32 | a, a:hover, a:focus { 33 | color: #30a5ff; 34 | } 35 | 36 | h1, h2, h3, h4, h5, h6 { 37 | color: #5f6468; 38 | } 39 | 40 | h1 { 41 | font-weight: 300; 42 | font-size: 40px; 43 | } 44 | 45 | h2 { 46 | font-weight: 300; 47 | margin-bottom: 20px; 48 | } 49 | 50 | h3, h4 { 51 | font-weight: 300; 52 | } 53 | 54 | .large { 55 | font-size: 2em; 56 | } 57 | 58 | .text-muted { 59 | color: #9fadbb; 60 | } 61 | 62 | .color-gray { color: #5f6468; } 63 | .color-blue { color: #30a5ff; } 64 | .color-teal { color: #1ebfae; } 65 | .color-orange { color: #ffb53e; } 66 | .color-red { color: #f9243f; } 67 | 68 | .bg-primary .glyphicon-remove { color: #5f6468;} 69 | .bg-primary .glyphicon-remove:hover { color: #ef4040;} 70 | 71 | .no-padding { 72 | padding: 0; margin: 0; 73 | } 74 | 75 | .glyphicon-xl { 76 | font-size: 6em; 77 | } 78 | 79 | .glyphicon-l { 80 | font-size: 3em; 81 | } 82 | 83 | .glyphicon-m { 84 | font-size: 1.5em; 85 | } 86 | 87 | .glyphicon-s { 88 | font-size: 0.75em; 89 | } 90 | 91 | .form-control { 92 | border: 1px solid #eee; 93 | box-shadow: none; 94 | margin-top: 10%; 95 | } 96 | 97 | .form-control:focus { 98 | border: 1px solid #30a5ff; 99 | outline: 0; 100 | box-shadow: inset 0px 0px 0px 1px #30a5ff; 101 | } 102 | 103 | .navbar-header .navbar-brand { 104 | color: #fff; 105 | font-size: 16px; 106 | text-transform: uppercase; 107 | font-weight: 500; 108 | letter-spacing: 2px; 109 | } 110 | 111 | #name{ 112 | float: right; 113 | } 114 | 115 | .navbar-header .navbar-brand span { 116 | color: #30a5ff; 117 | } 118 | 119 | .butt{ 120 | width: 20%; 121 | padding: 0px; 122 | border: none; 123 | margin-left: 2%; 124 | margin-right: 2%; 125 | } 126 | 127 | /*Buttons*/ 128 | 129 | a.btn:hover, 130 | button:hover { 131 | opacity: 0.8; 132 | } 133 | 134 | a.btn:active, 135 | button:active { 136 | box-shadow: inset 0px 0px 500px rgba(0,0,0,.1); 137 | opacity: 1; 138 | } 139 | 140 | .btn-default, 141 | .btn-default:hover, 142 | .btn-default:focus, 143 | .btn-default:active, 144 | .btn-default.active, 145 | .open > .dropdown-toggle.btn-default, 146 | .btn-default.disabled, 147 | .btn-default[disabled], 148 | fieldset[disabled] .btn-default, 149 | .btn-default.disabled:hover, 150 | .btn-default[disabled]:hover, 151 | fieldset[disabled] .btn-default:hover, 152 | .btn-default.disabled:focus, 153 | .btn-default[disabled]:focus, 154 | fieldset[disabled] .btn-default:focus, 155 | .btn-default.disabled:active, 156 | .btn-default[disabled]:active, 157 | fieldset[disabled] .btn-default:active, 158 | .btn-default.disabled.active, 159 | .btn-default[disabled].active, 160 | fieldset[disabled] .btn-default.active { 161 | background-color: #e9ecf2; 162 | border-color: #e9ecf2; 163 | color: #1b3548; 164 | } 165 | 166 | .btn-primary, 167 | .btn-primary:hover, 168 | .btn-primary:focus, 169 | .btn-primary:active, 170 | .btn-primary.active, 171 | .open > .dropdown-toggle.btn-primary, 172 | .btn-primary.disabled, 173 | .btn-primary[disabled], 174 | fieldset[disabled] .btn-primary, 175 | .btn-primary.disabled:hover, 176 | .btn-primary[disabled]:hover, 177 | fieldset[disabled] .btn-primary:hover, 178 | .btn-primary.disabled:focus, 179 | .btn-primary[disabled]:focus, 180 | fieldset[disabled] .btn-primary:focus, 181 | .btn-primary.disabled:active, 182 | .btn-primary[disabled]:active, 183 | fieldset[disabled] .btn-primary:active, 184 | .btn-primary.disabled.active, 185 | .btn-primary[disabled].active, 186 | fieldset[disabled] .btn-primary.active { 187 | background-color: #30a5ff; 188 | border-color: #30a5ff; 189 | } 190 | 191 | .btn-success, 192 | .btn-success:hover, 193 | .btn-success:focus, 194 | .btn-success:active, 195 | .btn-success.active, 196 | .open > .dropdown-toggle.btn-success, 197 | .btn-success.disabled, 198 | .btn-success[disabled], 199 | fieldset[disabled] .btn-success, 200 | .btn-success.disabled:hover, 201 | .btn-success[disabled]:hover, 202 | fieldset[disabled] .btn-success:hover, 203 | .btn-success.disabled:focus, 204 | .btn-success[disabled]:focus, 205 | fieldset[disabled] .btn-success:focus, 206 | .btn-success.disabled:active, 207 | .btn-success[disabled]:active, 208 | fieldset[disabled] .btn-success:active, 209 | .btn-success.disabled.active, 210 | .btn-success[disabled].active, 211 | fieldset[disabled] .btn-success.active { 212 | background-color: #8ad919; 213 | border-color: #8ad919; 214 | } 215 | 216 | .btn-warning, 217 | .btn-warning:hover, 218 | .btn-warning:focus, 219 | .btn-warning:active, 220 | .btn-warning.active, 221 | .open > .dropdown-toggle.btn-warning, 222 | .btn-warning.disabled, 223 | .btn-warning[disabled], 224 | fieldset[disabled] .btn-warning, 225 | .btn-warning.disabled:hover, 226 | .btn-warning[disabled]:hover, 227 | fieldset[disabled] .btn-warning:hover, 228 | .btn-warning.disabled:focus, 229 | .btn-warning[disabled]:focus, 230 | fieldset[disabled] .btn-warning:focus, 231 | .btn-warning.disabled:active, 232 | .btn-warning[disabled]:active, 233 | fieldset[disabled] .btn-warning:active, 234 | .btn-warning.disabled.active, 235 | .btn-warning[disabled].active, 236 | fieldset[disabled] .btn-warning.active { 237 | background-color: #ffb53e; 238 | border-color: #ffb53e; 239 | } 240 | 241 | .btn-danger, 242 | .btn-danger:hover, 243 | .btn-danger:focus, 244 | .btn-danger:active, 245 | .btn-danger.active, 246 | .open > .dropdown-toggle.btn-danger, 247 | .btn-danger.disabled, 248 | .btn-danger[disabled], 249 | fieldset[disabled] .btn-danger, 250 | .btn-danger.disabled:hover, 251 | .btn-danger[disabled]:hover, 252 | fieldset[disabled] .btn-danger:hover, 253 | .btn-danger.disabled:focus, 254 | .btn-danger[disabled]:focus, 255 | fieldset[disabled] .btn-danger:focus, 256 | .btn-danger.disabled:active, 257 | .btn-danger[disabled]:active, 258 | fieldset[disabled] .btn-danger:active, 259 | .btn-danger.disabled.active, 260 | .btn-danger[disabled].active, 261 | fieldset[disabled] .btn-danger.active { 262 | background-color: #f9243f; 263 | border-color: #f9243f; 264 | } 265 | 266 | /*Backgrounds*/ 267 | 268 | .bg-primary { 269 | color: #1b3548; 270 | background-color: #e9ecf2; 271 | } 272 | a.bg-primary:hover { 273 | background-color: #e9ecf2; 274 | } 275 | .bg-success { 276 | color: #fff; 277 | background-color: #8ad919; 278 | } 279 | a.bg-success:hover { 280 | background-color: #8ad919; 281 | } 282 | .bg-success a { 283 | color: rgba(255, 255, 255, .75); 284 | } 285 | .bg-info { 286 | color: #fff; 287 | background-color: #30a5ff; 288 | } 289 | a.bg-info:hover { 290 | background-color: #30a5ff; 291 | } 292 | .bg-info a { 293 | color: rgba(255, 255, 255, .75); 294 | } 295 | .bg-warning { 296 | color: #fff; 297 | background-color: #ffb53e; 298 | } 299 | a.bg-warning:hover { 300 | background-color: #ffb53e; 301 | } 302 | .bg-warning a { 303 | color: rgba(255, 255, 255, .75); 304 | } 305 | .bg-danger { 306 | color: #fff; 307 | background-color: #f9243f; 308 | } 309 | a.bg-danger:hover { 310 | background-color: #f9243f; 311 | } 312 | .bg-danger a { 313 | color: rgba(255, 255, 255, .75); 314 | } 315 | 316 | /*Panels*/ 317 | 318 | .panel { 319 | border: 0; 320 | } 321 | 322 | .panel-heading { 323 | font-size: 18px; 324 | font-weight: 300; 325 | letter-spacing: 0.025em; 326 | height: 66px; 327 | line-height: 45px; 328 | } 329 | 330 | .panel-default .panel-heading { 331 | background: #fff; 332 | border-bottom: 1px solid #eee; 333 | color: #5f6468; 334 | } 335 | 336 | .panel-footer { 337 | background: #fff; 338 | border-top: 1px solid #eee; 339 | } 340 | 341 | .panel-widget { 342 | padding: 0; 343 | position: relative; 344 | } 345 | 346 | .panel-widget .panel-footer { 347 | border: 0; 348 | text-align: center; 349 | } 350 | 351 | .panel-footer .input-group { 352 | padding: 0px; 353 | margin: 0 -5px; 354 | } 355 | 356 | .panel-footer .input-group-btn:last-child>.btn, 357 | .panel-footer .input-group-btn:last-child>.btn-group { 358 | margin: 0; 359 | } 360 | 361 | .panel-widget .panel-footer a { 362 | color: #999; 363 | } 364 | 365 | .panel-widget .panel-footer a:hover { 366 | color: #666; 367 | text-decoration: none; 368 | } 369 | 370 | .panel-blue { background: #30a5ff; color: #fff; } 371 | .panel-teal { background: #1ebfae; color: #fff; } 372 | .panel-orange { background: #ffb53e; color: #fff; } 373 | .panel-red { background: #f9243f; color: #fff; } 374 | 375 | .panel-blue .panel-body p, 376 | .panel-teal .panel-body p, 377 | .panel-orange .panel-body p, 378 | .panel-red .panel-body p { 379 | color: #fff; 380 | color: rgba(255, 255, 255, .8); 381 | } 382 | 383 | .panel-blue .panel-heading, 384 | .panel-teal .panel-heading, 385 | .panel-orange .panel-heading, 386 | .panel-red .panel-heading { 387 | border-bottom: 1px solid rgba(255, 255, 255, .2); 388 | } 389 | 390 | .panel-blue .text-muted, 391 | .panel-teal .text-muted, 392 | .panel-orange .text-muted, 393 | .panel-red .text-muted { 394 | color: rgba(255, 255, 255, .5); 395 | } 396 | 397 | .dark-overlay { 398 | background: rgba(0, 0, 0, .05); 399 | text-align: center; 400 | } 401 | 402 | .widget-left { 403 | height: 80px; 404 | padding-top: 15px; 405 | text-align: center; 406 | border-top-left-radius: 4px; 407 | border-bottom-left-radius: 4px; 408 | } 409 | 410 | .widget-right { 411 | text-align: left; 412 | line-height: 1.6em; 413 | margin: 0px; 414 | padding: 20px; 415 | height: 80px; 416 | color: #999; 417 | font-weight: 300; 418 | background: #fff; 419 | border-top-right-radius: 4px; 420 | border-bottom-right-radius: 4px; 421 | } 422 | 423 | @media (max-width: 768px) { 424 | .widget-right { 425 | width: 100%; 426 | margin: 0; 427 | text-align: center; 428 | border-top-left-radius: 0px; 429 | border-top-right-radius: 0px; 430 | border-bottom-left-radius: 4px; 431 | border-bottom-right-radius: 4px; 432 | } 433 | } 434 | 435 | @media (max-width: 768px) { 436 | .widget-left { 437 | border-top-left-radius: 4px; 438 | border-top-right-radius: 4px; 439 | border-bottom-left-radius: 0px; 440 | border-bottom-right-radius: 0px; 441 | } 442 | } 443 | 444 | .widget-right .text-muted { 445 | color: #9fadbb; 446 | } 447 | .widget-right .large { 448 | color: #5f6468; 449 | } 450 | 451 | .panel-blue .widget-left { background: #30a5ff; color: #fff; } 452 | .panel-teal .widget-left { background: #1ebfae; color: #fff; } 453 | .panel-orange .widget-left { background: #ffb53e; color: #fff; } 454 | .panel-red .widget-left { background: #f9243f; color: #fff; } 455 | 456 | .panel-widget { 457 | background: #fff; 458 | margin-bottom: 0px; 459 | } 460 | 461 | /*Jumbotron*/ 462 | 463 | .jumbotron { 464 | background: #fff; 465 | border-bottom: 1px solid #eee; 466 | color: #5f6468; 467 | width: 50%; 468 | margin-left: 25%; 469 | margin-top: 5%; 470 | margin-bottom: 5%; 471 | } 472 | 473 | /*Tabs*/ 474 | 475 | .panel .tabs { 476 | margin: 0; 477 | padding: 0; 478 | } 479 | 480 | .tab-content { 481 | padding: 15px; 482 | } 483 | 484 | 485 | /*Navbar*/ 486 | 487 | .navbar { 488 | border: 0; 489 | } 490 | 491 | .navbar input { 492 | border: 0; 493 | background: #444; 494 | color: #fff; 495 | } 496 | 497 | .navbar input:focus { 498 | color: #fff; 499 | background: #555; 500 | border: 1px solid #30a5ff; 501 | outline: 0; 502 | box-shadow: inset 0px 0px 0px 1px #30a5ff; 503 | } 504 | 505 | @media (min-width: 768px) { 506 | .navbar-header { 507 | width: 100%; 508 | } 509 | } 510 | 511 | /*Breadcrumbs*/ 512 | 513 | .breadcrumb { 514 | border-radius: 0; 515 | padding: 10px 15px; 516 | background: #e9ecf2; 517 | box-shadow: 0 1px 1px rgba(0, 0, 0, .05); 518 | margin: 0 ; 519 | } 520 | 521 | h1.page-header { 522 | margin-top: 30px; 523 | border-bottom: 0; 524 | } 525 | 526 | .panel-heading .glyphicon { 527 | margin-right: 10px; 528 | } 529 | 530 | /*Todo List Widget*/ 531 | 532 | .todo-list-item .glyphicon { 533 | margin-right:5px; 534 | color: #9fadbb; 535 | } 536 | 537 | .todo-list-item .glyphicon:hover { 538 | margin-right:5px; 539 | color: #1b3548; 540 | } 541 | 542 | .todo-list { 543 | padding: 0; 544 | margin: -15px; 545 | background: #fff; 546 | color: #5f6468; 547 | } 548 | 549 | #checkbox { 550 | margin: 0; 551 | } 552 | 553 | .todo-list .checkbox { 554 | display:inline-block; 555 | margin: 0px; 556 | } 557 | 558 | .panel-body input[type=checkbox]:checked + label { 559 | text-decoration: none; 560 | color: #777; 561 | } 562 | 563 | .todo-list-item { 564 | list-style: none; 565 | line-height: 0.9; 566 | padding: 14px 15px 8px 15px; 567 | } 568 | 569 | .todo-list-item:hover, a.todo-list-item:focus { 570 | text-decoration: none; 571 | background-color: #f6f6f6; 572 | } 573 | 574 | .todo-list-item .trash .glyph:hover { 575 | color: #ef4040; 576 | } 577 | 578 | /*Icons*/ 579 | 580 | 581 | .sidebar .glyph, .user-menu .glyph { 582 | height: 16px; 583 | width: 16px; 584 | margin: 0 10px 0 0; 585 | stroke-width: 3px; 586 | } 587 | 588 | .user-menu .glyph { 589 | stroke-width: 4px; 590 | } 591 | 592 | .breadcrumb .glyph { 593 | height: 14px; 594 | width: 14px; 595 | margin: -2px 0 0 0; 596 | stroke-width: 4px; 597 | } 598 | 599 | .alert .glyph, .panel-heading .glyph { 600 | width: 26px; 601 | height: 26px; 602 | margin: 0 10px 0 0; 603 | stroke-width: 2px; 604 | } 605 | 606 | .panel-widget .glyph { 607 | stroke-width: 2px; 608 | } 609 | 610 | .todo-list .glyph { 611 | width: 14px; 612 | height: 14px; 613 | stroke-width: 4px; 614 | color: #999; 615 | } 616 | 617 | .glyph.table { 618 | background: none; 619 | border: none; 620 | } 621 | 622 | /*Icon Grid*/ 623 | 624 | .icon-grid div { 625 | border: 1px solid #ddd; 626 | margin: 0 0 -1px -1px; 627 | text-align: center; 628 | padding: 10px 0 20px 0; 629 | } 630 | 631 | .icon-grid svg { 632 | width: 35%; 633 | display: block; 634 | margin: 0 auto; 635 | } 636 | 637 | .icon-grid h4 { 638 | display: none; 639 | } 640 | 641 | .icon-grid pre { 642 | margin: 10px 10px -10px 10px; 643 | border-radius: 0; 644 | font-size: 10px; 645 | border-color: #ddd; 646 | height: 65px; 647 | overflow: scroll; 648 | } 649 | 650 | 651 | 652 | 653 | 654 | 655 | 656 | 657 | 658 | -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mansimarkaur/webpage-scraper/f495b9f1fca00d7db03ddbcd649522cc9fdf21f1/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mansimarkaur/webpage-scraper/f495b9f1fca00d7db03ddbcd649522cc9fdf21f1/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mansimarkaur/webpage-scraper/f495b9f1fca00d7db03ddbcd649522cc9fdf21f1/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /static/js/bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap v3.2.0 (http://getbootstrap.com) 3 | * Copyright 2011-2014 Twitter, Inc. 4 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) 5 | */ 6 | if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.2.0",d.prototype.close=function(b){function c(){f.detach().trigger("closed.bs.alert").remove()}var d=a(this),e=d.attr("data-target");e||(e=d.attr("href"),e=e&&e.replace(/.*(?=#[^\s]*$)/,""));var f=a(e);b&&b.preventDefault(),f.length||(f=d.hasClass("alert")?d:d.parent()),f.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",c).emulateTransitionEnd(150):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.2.0",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),d[e](null==f[b]?this.options[b]:f[b]),setTimeout(a.proxy(function(){"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")&&(c.prop("checked")&&this.$element.hasClass("active")?a=!1:b.find(".active").removeClass("active")),a&&c.prop("checked",!this.$element.hasClass("active")).trigger("change")}a&&this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),c.preventDefault()})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b).on("keydown.bs.carousel",a.proxy(this.keydown,this)),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=this.sliding=this.interval=this.$active=this.$items=null,"hover"==this.options.pause&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.2.0",c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0},c.prototype.keydown=function(a){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.to=function(b){var c=this,d=this.getItemIndex(this.$active=this.$element.find(".item.active"));return b>this.$items.length-1||0>b?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){c.to(b)}):d==b?this.pause().cycle():this.slide(b>d?"next":"prev",a(this.$items[b]))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,c){var d=this.$element.find(".item.active"),e=c||d[b](),f=this.interval,g="next"==b?"left":"right",h="next"==b?"first":"last",i=this;if(!e.length){if(!this.options.wrap)return;e=this.$element.find(".item")[h]()}if(e.hasClass("active"))return this.sliding=!1;var j=e[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:g});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,f&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(e)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:g});return a.support.transition&&this.$element.hasClass("slide")?(e.addClass(b),e[0].offsetWidth,d.addClass(g),e.addClass(g),d.one("bsTransitionEnd",function(){e.removeClass([b,g].join(" ")).addClass("active"),d.removeClass(["active",g].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(1e3*d.css("transition-duration").slice(0,-1))):(d.removeClass("active"),e.addClass("active"),this.sliding=!1,this.$element.trigger(m)),f&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this},a(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}}),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.collapse"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b);!e&&f.toggle&&"show"==b&&(b=!b),e||d.data("bs.collapse",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.transitioning=null,this.options.parent&&(this.$parent=a(this.options.parent)),this.options.toggle&&this.toggle()};c.VERSION="3.2.0",c.DEFAULTS={toggle:!0},c.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},c.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var c=a.Event("show.bs.collapse");if(this.$element.trigger(c),!c.isDefaultPrevented()){var d=this.$parent&&this.$parent.find("> .panel > .in");if(d&&d.length){var e=d.data("bs.collapse");if(e&&e.transitioning)return;b.call(d,"hide"),e||d.data("bs.collapse",null)}var f=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[f](0),this.transitioning=1;var g=function(){this.$element.removeClass("collapsing").addClass("collapse in")[f](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return g.call(this);var h=a.camelCase(["scroll",f].join("-"));this.$element.one("bsTransitionEnd",a.proxy(g,this)).emulateTransitionEnd(350)[f](this.$element[0][h])}}},c.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse").removeClass("in"),this.transitioning=1;var d=function(){this.transitioning=0,this.$element.trigger("hidden.bs.collapse").removeClass("collapsing").addClass("collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(d,this)).emulateTransitionEnd(350):d.call(this)}}},c.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()};var d=a.fn.collapse;a.fn.collapse=b,a.fn.collapse.Constructor=c,a.fn.collapse.noConflict=function(){return a.fn.collapse=d,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(c){var d,e=a(this),f=e.attr("data-target")||c.preventDefault()||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""),g=a(f),h=g.data("bs.collapse"),i=h?"toggle":e.data(),j=e.attr("data-parent"),k=j&&a(j);h&&h.transitioning||(k&&k.find('[data-toggle="collapse"][data-parent="'+j+'"]').not(e).addClass("collapsed"),e[g.hasClass("in")?"addClass":"removeClass"]("collapsed")),b.call(g,i)})}(jQuery),+function(a){"use strict";function b(b){b&&3===b.which||(a(e).remove(),a(f).each(function(){var d=c(a(this)),e={relatedTarget:this};d.hasClass("open")&&(d.trigger(b=a.Event("hide.bs.dropdown",e)),b.isDefaultPrevented()||d.removeClass("open").trigger("hidden.bs.dropdown",e))}))}function c(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.2.0",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=c(e),g=f.hasClass("open");if(b(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(' 137 | 138 | 139 | 140 | 141 | 142 | 149 | 156 | 162 | 163 | 164 | 165 | 166 | 167 | -------------------------------------------------------------------------------- /templates/links.html: -------------------------------------------------------------------------------- 1 | {% extends "index.html" %} 2 | 3 | {% block panel %} 4 |
5 |
6 | Hyperlinks 7 |
8 |
9 |
22 |
23 | 24 | {% endblock %} 25 | 26 | {% block download %} 27 |
28 |
29 |
30 |
31 |
32 | 33 |
34 | 35 |
36 |
37 |
38 |
39 | 40 |
41 |
42 |
43 |
44 |
45 |
46 | {% endblock %} 47 | {% block flash %} 48 | {%if get_flashed_messages()%} 49 | {% for msg in get_flashed_messages()%} 50 | 51 | {{msg}} 52 | {% endfor %} 53 | {% endif %} 54 | {% endblock %} 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /templates/text.html: -------------------------------------------------------------------------------- 1 | {% extends "index.html" %} 2 | 3 | {% block panel %} 4 |
5 |
Text
6 |
7 |
8 |
 9 | 							

{{ text }}

10 |
11 |
12 | 13 |
14 | 15 |
16 | {% endblock %} 17 | 18 | {% block download %} 19 |
20 |
21 |
22 |
23 |
24 | 25 |
26 | 27 |
28 |
29 |
30 |
31 | 32 |
33 |
34 |
35 |
36 |
37 |
38 | {% endblock %} 39 | {% block flash %} 40 | {%if get_flashed_messages()%} 41 | {% for msg in get_flashed_messages()%} 42 | 43 | {{msg}} 44 | {% endfor %} 45 | {% endif %} 46 | {% endblock %} 47 | 48 | 49 | 50 | --------------------------------------------------------------------------------