├── .gitignore ├── README.md ├── manage.py ├── requirements.txt ├── scraper ├── __init__.py ├── admin.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── crawl.py ├── models.py ├── tasks.py ├── tests.py └── views.py ├── setup ├── test ├── static │ ├── page1 │ ├── page2 │ ├── page3 │ ├── page4 │ ├── page5 │ ├── page6 │ ├── page7 │ ├── page8 │ └── page9 └── testsite.py └── website ├── __init__.py ├── settings.py ├── urls.py └── wsgi.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.log 4 | database 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Scraper 2 | ======= 3 | 4 | Crawls all the urls on a specified domain and creates an entry for each in the 5 | Django database. 6 | 7 | Installation 8 | ------------ 9 | 10 | Create the database: python manage.py syncdb 11 | 12 | The location of the database file is hardcoded in the settings file. I 13 | couldn't figure out a way around this so it will have to be modified. Edit 14 | website/settings.py and modify the Name field under Databases to reflect the 15 | absolute path of the database. 16 | 17 | Invocation 18 | ---------- 19 | 20 | Set environment variables: . ./setup 21 | 22 | Start RabbitMQ: sudo rabbitmq-server 23 | 24 | Start Celery: cd $SCRAPER_HOME/scraper; celery -A tasks worker --loglevel=info 25 | 26 | Run the scraper: cd $SCRAPER_HOME; python manage.py crawl <url> 27 | 28 | Start the admin interface: cd $SCRAPER_HOME; python manage.py runserver 29 | 30 | View the admin interface: http://localhost:8000/admin 31 | 32 | Cleanup 33 | ------- 34 | 35 | Stop Celery Tasks: celeryctl purge 36 | 37 | Stop RabbitMQ: sudo rabbitmqctl stop 38 | 39 | Testing 40 | ------- 41 | 42 | Run test website: cd $SCRAPER_HOME/test; python testsite.py 43 | 44 | Run scraper against test website: cd $SCRAPER_HOME; python manage.py crawl 45 | http://localhost:8080/page1 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "website.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.5.1 2 | amqp==1.0.11 3 | anyjson==0.3.3 4 | billiard==2.7.3.28 5 | bottle==0.11.6 6 | celery==3.0.19 7 | django-celery==3.0.17 8 | kombu==2.5.10 9 | lxml==3.2.0 10 | pybloomfiltermmap==0.3.11 11 | python-dateutil==2.1 12 | pytz==2013b 13 | requests==1.2.0 14 | six==1.3.0 15 | tldextract==1.1.3 16 | -------------------------------------------------------------------------------- /scraper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/staceysern/scraper/a63f12e620628f6580c0c3c7911c3e82042249f6/scraper/__init__.py -------------------------------------------------------------------------------- /scraper/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from scraper.models import Page 3 | 4 | admin.site.register(Page) 5 | -------------------------------------------------------------------------------- /scraper/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/staceysern/scraper/a63f12e620628f6580c0c3c7911c3e82042249f6/scraper/management/__init__.py -------------------------------------------------------------------------------- /scraper/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/staceysern/scraper/a63f12e620628f6580c0c3c7911c3e82042249f6/scraper/management/commands/__init__.py -------------------------------------------------------------------------------- /scraper/management/commands/crawl.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from scraper.tasks import crawl 3 | 4 | class Command(BaseCommand): 5 | args = '' 6 | help = ('Kicks off celery tasks to crawl a domain starting at the' 7 | 'specified url') 8 | 9 | def handle(self, *args, **options): 10 | if len(args) < 1: 11 | print 'usage: python manage.py crawl domain' 12 | else: 13 | crawl(args[0]) 14 | 15 | -------------------------------------------------------------------------------- /scraper/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class Page(models.Model): 4 | url = models.CharField(max_length=200) 5 | onsale = models.BooleanField('on-sale') 6 | 7 | def __unicode__(self): 8 | return self.url 9 | 10 | class Meta: 11 | app_label = 'scraper' 12 | 13 | 14 | -------------------------------------------------------------------------------- /scraper/tasks.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import lxml.html 3 | import pybloomfilter 4 | import re 5 | import requests 6 | import tldextract 7 | 8 | from celery import Celery, group, task 9 | from celery.utils.log import get_task_logger 10 | from scraper.models import Page 11 | 12 | logger = get_task_logger(__name__) 13 | 14 | celery = Celery('tasks', backend='amqp', broker='amqp://guest@localhost//') 15 | 16 | 17 | @task(name="app.tasks.crawl") 18 | def crawl(start): 19 | """ 20 | Starting from the specified url, crawl all urls on the same domain. 21 | """ 22 | 23 | # Delete any page records in the database from a previous search 24 | Page.objects.all().delete() 25 | 26 | # Extract the domain and top level domain 27 | extract = tldextract.extract(start) 28 | domain = extract.domain 29 | tld = extract.tld 30 | 31 | # Use a bloom filter to keep track of urls which have been crawled. 32 | # This offers fast determination of set membership. A negative result is 33 | # always correct but a positive result may be incorrent in which case a 34 | # page would not not be crawled. This filter is sized to hold 100,000 35 | # items with an error rate of less than .1 percent. 36 | crawled = pybloomfilter.BloomFilter(100000, 0.001, "/tmp/"+domain+tld) 37 | 38 | frontier = [start] 39 | 40 | # Until there are no more links to explore, search each url on the 41 | # frontier 'in parallel' and add any uncrawled links for the same domain 42 | # to the frontier for the next round. 43 | while frontier: 44 | logger.debug("Frontier {}\n".format(frontier)) 45 | 46 | # Construct a list of scrape subtasks for each url on the 47 | # frontier then mark each url as crawled and empty the frontier. 48 | tasks = [scrape.s(url, domain, tld) for url in frontier] 49 | crawled.update(frontier) 50 | frontier = [] 51 | 52 | # Kick off all the searches at once and wait until they are all done 53 | # before proceeding. The result is list of links found on each url. 54 | # No processing has been done so these lists may include duplicates and 55 | # non-http urls. 56 | result = group(tasks)() 57 | urls = result.get() 58 | logger.debug("Results {}\n\n".format(urls)) 59 | 60 | # Join all the lists together and remove duplicates 61 | urls = set(itertools.chain.from_iterable(urls)) 62 | 63 | # Filter out urls that have been called already or do not use http 64 | frontier = filter(lambda x: not x in crawled and 65 | x.startswith('http://'), urls) 66 | 67 | return True 68 | 69 | regex = re.compile(".*?on-sale") 70 | 71 | 72 | @task(name="app.tasks.search_page") 73 | def scrape(url, domain, tld): 74 | """ 75 | Retrieve the page specified by the url, record in the database whether it 76 | contains the string "on-sale", and return a list of links on the page 77 | in the specified domain and top level domain. 78 | 79 | The list of links includes any duplicates that may be on the page. Since 80 | the list will be processed further, there is no need to do extra work to 81 | filter them out here. 82 | 83 | The domain and tld arguments are redundant since that information is 84 | embedded in the url but it is faster to pass the information in that to 85 | recompute them for each url. 86 | """ 87 | 88 | # In certain cases, when there is an error fetching the page, it may make 89 | # sense to retry but, for now, just log an error and return the empty 90 | # list. 91 | try: 92 | response = requests.get(url) 93 | except requests.exceptions.RequestException as err: 94 | logger.error("url {}: {}".format(url, err)) 95 | return[] 96 | 97 | if response.status_code != 200: 98 | logger.error("url {}: status code {}".format(url, response.status_code)) 99 | return [] 100 | 101 | onsale = regex.search(response.text) is not None 102 | p = Page(url=url, onsale=onsale) 103 | p.save() 104 | 105 | # Parse the text of the page into a document and select the url in the 106 | # href for each a tag. Then check whether it is in the specified domain 107 | # and top level domain. 108 | links = [] 109 | dom = lxml.html.fromstring(response.text) 110 | for link in dom.xpath('//a/@href'): 111 | extract = tldextract.extract(link) 112 | if extract.domain == domain and extract.tld == tld: 113 | links.append(link) 114 | 115 | return links 116 | -------------------------------------------------------------------------------- /scraper/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /scraper/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /setup: -------------------------------------------------------------------------------- 1 | export SCRAPER_HOME=$PWD 2 | export PYTHONPATH=$PYTHONPATH:$PWD 3 | export PYTHONPATH=$PYTHONPATH:$PWD/website 4 | export DJANGO_SETTINGS_MODULE=website.settings 5 | 6 | echo "DJANGO_SETTINGS_MODULE set to $DJANGO_SETTINGS_MODULE" 7 | -------------------------------------------------------------------------------- /test/static/page1: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 7 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 8 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 9 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 10 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 11 | cookie bear claw bear claw. Chocolate cake donut jujubes chupa chups chocolate 12 | cake. Gummies croissant carrot cake pastry. 13 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 14 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 15 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 16 | Jelly beans wypas soufflé. Wafer faworki jujubes applicake. Cake icing 18 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 19 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 20 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 21 | Candy canes ice cream carrot cake. Cake soufflé ice cream muffin. 22 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 23 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 24 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 25 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 26 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 27 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 28 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 29 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 30 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 31 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 32 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 33 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 35 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 36 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 37 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 38 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 39 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 40 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 41 | chocolate cake liquorice lemon drops cotton candy. 42 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 43 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 44 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 45 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 46 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 47 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 48 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 49 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 50 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/static/page2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 7 | Jelly beans wypas soufflé. Wafer faworki jujubes applicake. Cake icing 9 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 10 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 11 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 12 | Candy canes ice cream carrot cake. Cake soufflé ice cream muffin. 13 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 14 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 15 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 16 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 17 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 18 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 19 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 20 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 21 | apple pie wypas. Chupa chups muffin sweet chocolate on-sale faworki pie. Topping 22 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 23 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 24 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 26 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 27 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 28 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 29 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 30 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 31 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 32 | chocolate cake liquorice lemon drops cotton candy. 33 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 34 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 35 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 36 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 37 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 38 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 39 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 40 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 41 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 42 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 43 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 44 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 45 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 46 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 47 | cookie bear claw bear claw. Chocolate cake donut jujubes chupa chups chocolate 48 | cake. Gummies croissant carrot cake pastry. 49 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 50 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/static/page3: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 7 | Candy canes ice cream carrot cake. Cake soufflé ice cream muffin. 8 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 9 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 10 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 11 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 12 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 13 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 14 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 15 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 16 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 17 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 18 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 19 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 21 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 22 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 23 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 24 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 25 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 26 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 27 | chocolate cake liquorice lemon drops cotton candy. 28 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 29 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 30 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 31 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 32 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 33 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 34 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 35 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 36 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 37 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 38 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 39 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 40 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 41 | cake. Gummies croissant carrot cake pastry. 42 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 43 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 44 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 45 | Jelly beans wypas soufflé. Wafer faworki jujubes applicake. Cake icing 47 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 48 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 49 | 50 | 51 | -------------------------------------------------------------------------------- /test/static/page4: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 7 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 8 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 9 | brownie. Jelly applicake halvah chocolate cake. 10 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 11 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 12 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 13 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 14 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 15 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 16 | chocolate cake liquorice lemon drops cotton candy. 17 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 18 | icing on-sale pie sweet sweet. Ice cream bear claw marshmallow cheesecake 19 | 20 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 21 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 22 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 23 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 24 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 25 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 26 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 27 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 28 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 29 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 30 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 31 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 32 | cake. Gummies croissant carrot cake pastry. 33 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 34 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 35 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 36 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 37 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 38 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 39 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 40 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 41 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 42 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 43 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 44 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 45 | 46 | 47 | -------------------------------------------------------------------------------- /test/static/page5: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 7 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 8 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 9 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 10 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 11 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 12 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 13 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 14 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 15 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 16 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 18 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 19 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 20 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 21 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 22 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 23 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 24 | chocolate cake liquorice lemon drops cotton candy. 25 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 26 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 27 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 28 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 29 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 30 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 31 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 32 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 33 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 34 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 35 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 36 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 37 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 38 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 39 | cookie bear claw bear claw. Chocolate cake donut jujubes chupa chups chocolate 40 | cake. Gummies croissant carrot cake pastry. 41 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 42 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 43 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 44 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 45 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 46 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 47 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/static/page6: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 7 | Jelly beans wypas soufflé. Wafer faworki jujubes applicake. Cake icing 9 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 10 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 11 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 12 | Candy canes ice cream carrot cake. Cake soufflé ice cream muffin. 13 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 14 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 15 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 16 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 17 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 18 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 19 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 20 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 21 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 22 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 23 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 24 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 26 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 27 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 28 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 29 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 30 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 31 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 32 | chocolate cake liquorice lemon drops cotton candy. 33 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 34 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 35 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 36 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 37 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 38 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 39 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 40 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 41 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 42 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 43 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 44 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 45 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 46 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 47 | cookie bear claw bear claw. Chocolate cake donut jujubes chupa chups chocolate 48 | cake. Gummies croissant carrot cake pastry. 49 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 50 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/static/page7: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 7 | Candy canes ice cream carrot cake. Cake soufflé ice cream muffin. 8 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 9 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 10 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 11 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 12 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 13 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 14 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 15 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 16 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 17 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 18 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 19 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 21 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 22 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 23 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 24 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 25 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 26 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 27 | chocolate cake liquorice lemon drops cotton candy. 28 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 29 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 30 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 31 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 32 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 33 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 34 | jujubes chocolate on-sale chupa chups chocolate. Toffee marzipan gummi bears 35 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 36 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 37 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 38 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 39 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 40 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 41 | cake. Gummies croissant carrot cake pastry. 42 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 43 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 44 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 45 | Jelly beans wypas soufflé. Wafer faworki jujubes applicake. Cake icing 47 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 48 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 49 | 50 | 51 | -------------------------------------------------------------------------------- /test/static/page8: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 7 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 8 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 9 | brownie. Jelly applicake halvah chocolate cake. 10 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 11 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 12 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 13 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 14 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 15 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 16 | chocolate cake liquorice lemon drops cotton candy. 17 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 18 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 19 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 20 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 21 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 22 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 23 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 24 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 25 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 26 | Cupcake ipsum dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 27 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 28 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 29 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 30 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 31 | cake. Gummies croissant carrot cake pastry. 32 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 33 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 34 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 35 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 36 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 37 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 38 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 39 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 40 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 41 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 42 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 43 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 44 | 45 | 46 | -------------------------------------------------------------------------------- /test/static/page9: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Biscuit lemon drops bear claw tart caramels sugar plum gummi bears cheesecake 7 | croissant. Chocolate bar chocolate marzipan tiramisu halvah wafer. Toffee 8 | chocolate cake jelly beans wafer apple pie. Donut faworki dessert sesame snaps 9 | chocolate cake gingerbread pudding. Sugar plum cake muffin cookie lollipop 10 | icing. Lollipop chupa chups gingerbread. Wafer faworki sugar plum marzipan 11 | dessert. Tootsie roll cotton candy gummi bears jelly beans dessert. Jelly-o 12 | danish lemon drops halvah danish dessert lemon drops apple pie icing. Jujubes 13 | apple pie wypas. Chupa chups muffin sweet chocolate wafer faworki pie. Topping 14 | marshmallow sugar plum fruitcake fruitcake sesame snaps. 15 | Sweet apple pie jelly-o. Cupcake gingerbread ice cream bear claw cupcake wypas 16 | brownie. Jelly applicake halvah chocolate cake. Dessert toffee gingerbread 18 | sugar plum dessert donut. Tart halvah cupcake gingerbread. Donut donut 19 | marshmallow cupcake candy bear claw gummies biscuit applicake. Topping faworki 20 | tiramisu sesame snaps apple pie oat cake. Lollipop halvah sweet roll sweet bear 21 | claw applicake icing. Toffee caramels croissant candy wypas marzipan. Biscuit 22 | powder applicake jujubes cookie bonbon jujubes tootsie roll. Lemon drops sugar 23 | plum cake pastry caramels gummies sugar plum halvah. Muffin wafer cupcake 24 | chocolate cake liquorice lemon drops cotton candy. 25 | Bear claw jelly beans marzipan chupa chups sweet roll muffin icing. Tiramisu 26 | icing jujubes pie sweet sweet. Ice cream bear claw marshmallow cheesecake 27 | cotton candy applicake. Oat cake chupa chups candy liquorice pie carrot cake 28 | bonbon macaroon jelly-o. Powder caramels apple pie. Caramels dessert tiramisu 29 | tootsie roll tiramisu faworki apple pie oat cake bear claw. Bonbon chocolate 30 | bar toffee. Tiramisu jelly beans bonbon lollipop. Bear claw jujubes applicake 31 | jujubes chocolate pudding chupa chups chocolate. Toffee marzipan gummi bears 32 | pudding pastry. Bear claw fruitcake jelly beans. Bear claw gingerbread 33 | cheesecake candy canes jelly beans gummies danish cookie toffee. Jelly beans 34 | Cupcake on-sale dolor sit. Amet cake muffin caramels wypas. Cake caramels brownie 35 | ice cream sesame snaps. Muffin chupa chups cookie cotton candy wypas. Gummies 36 | faworki lemon drops powder. Cupcake halvah tart sesame snaps. Jelly-o sesame 37 | snaps donut brownie sesame snaps powder chupa chups pie cake. Cake chocolate 38 | bar donut gummies jujubes. Tart toffee candy canes caramels topping. Marzipan 39 | cookie bear claw bear claw. Chocolate cake donut jujubes chupa chups chocolate 40 | cake. Gummies croissant carrot cake pastry. 41 | Sesame snaps muffin sugar plum lollipop macaroon sweet roll. Caramels tiramisu 42 | pie ice cream fruitcake applicake tiramisu cookie bear claw. Halvah pie 43 | chocolate soufflé gummi bears sesame snaps. Candy canes gingerbread tiramisu. 44 | croissant sugar plum brownie wafer ice cream jelly beans. Icing tart sweet roll 45 | lemon drops sugar plum applicake oat cake. Liquorice sweet cheesecake gummies 46 | cookie dessert caramels. Tart powder sugar plum. Oat cake marzipan oat cake. 47 | Tootsie roll bonbon wafer faworki sweet marzipan dragée biscuit apple pie. 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/testsite.py: -------------------------------------------------------------------------------- 1 | from bottle import route, run, static_file 2 | 3 | @route('/') 4 | def index(): 5 | return "This is the root" 6 | 7 | @route('/') 8 | def server_static(filename): 9 | return static_file(filename, 'static') 10 | 11 | run(host='localhost', port=8080, debug=True) 12 | -------------------------------------------------------------------------------- /website/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/staceysern/scraper/a63f12e620628f6580c0c3c7911c3e82042249f6/website/__init__.py -------------------------------------------------------------------------------- /website/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for website project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@example.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 15 | 'NAME': '/Users/shira/Code/python/django/website/database', # Or path to database file if using sqlite3. 16 | # The following settings are not used with sqlite3: 17 | 'USER': '', 18 | 'PASSWORD': '', 19 | 'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP. 20 | 'PORT': '', # Set to empty string for default. 21 | } 22 | } 23 | 24 | # Hosts/domain names that are valid for this site; required if DEBUG is False 25 | # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts 26 | ALLOWED_HOSTS = [] 27 | 28 | # Local time zone for this installation. Choices can be found here: 29 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 30 | # although not all choices may be available on all operating systems. 31 | # In a Windows environment this must be set to your system time zone. 32 | TIME_ZONE = 'America/Chicago' 33 | 34 | # Language code for this installation. All choices can be found here: 35 | # http://www.i18nguy.com/unicode/language-identifiers.html 36 | LANGUAGE_CODE = 'en-us' 37 | 38 | SITE_ID = 1 39 | 40 | # If you set this to False, Django will make some optimizations so as not 41 | # to load the internationalization machinery. 42 | USE_I18N = True 43 | 44 | # If you set this to False, Django will not format dates, numbers and 45 | # calendars according to the current locale. 46 | USE_L10N = True 47 | 48 | # If you set this to False, Django will not use timezone-aware datetimes. 49 | USE_TZ = True 50 | 51 | # Absolute filesystem path to the directory that will hold user-uploaded files. 52 | # Example: "/var/www/example.com/media/" 53 | MEDIA_ROOT = '' 54 | 55 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 56 | # trailing slash. 57 | # Examples: "http://example.com/media/", "http://media.example.com/" 58 | MEDIA_URL = '' 59 | 60 | # Absolute path to the directory static files should be collected to. 61 | # Don't put anything in this directory yourself; store your static files 62 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 63 | # Example: "/var/www/example.com/static/" 64 | STATIC_ROOT = '' 65 | 66 | # URL prefix for static files. 67 | # Example: "http://example.com/static/", "http://static.example.com/" 68 | STATIC_URL = '/static/' 69 | 70 | # Additional locations of static files 71 | STATICFILES_DIRS = ( 72 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 73 | # Always use forward slashes, even on Windows. 74 | # Don't forget to use absolute paths, not relative paths. 75 | ) 76 | 77 | # List of finder classes that know how to find static files in 78 | # various locations. 79 | STATICFILES_FINDERS = ( 80 | 'django.contrib.staticfiles.finders.FileSystemFinder', 81 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 82 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 83 | ) 84 | 85 | # Make this unique, and don't share it with anybody. 86 | SECRET_KEY = '751j)vfb)1^#iza@@dr+l=tvk8k6j-q-nh2*43=y4l(i8dl)=_' 87 | 88 | # List of callables that know how to import templates from various sources. 89 | TEMPLATE_LOADERS = ( 90 | 'django.template.loaders.filesystem.Loader', 91 | 'django.template.loaders.app_directories.Loader', 92 | # 'django.template.loaders.eggs.Loader', 93 | ) 94 | 95 | MIDDLEWARE_CLASSES = ( 96 | 'django.middleware.common.CommonMiddleware', 97 | 'django.contrib.sessions.middleware.SessionMiddleware', 98 | 'django.middleware.csrf.CsrfViewMiddleware', 99 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 100 | 'django.contrib.messages.middleware.MessageMiddleware', 101 | # Uncomment the next line for simple clickjacking protection: 102 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 103 | ) 104 | 105 | ROOT_URLCONF = 'website.urls' 106 | 107 | # Python dotted path to the WSGI application used by Django's runserver. 108 | WSGI_APPLICATION = 'website.wsgi.application' 109 | 110 | TEMPLATE_DIRS = ( 111 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 112 | # Always use forward slashes, even on Windows. 113 | # Don't forget to use absolute paths, not relative paths. 114 | ) 115 | 116 | INSTALLED_APPS = ( 117 | 'django.contrib.auth', 118 | 'django.contrib.contenttypes', 119 | 'django.contrib.sessions', 120 | 'django.contrib.sites', 121 | 'django.contrib.messages', 122 | 'django.contrib.staticfiles', 123 | 'djcelery', 124 | 'scraper', 125 | 'django.contrib.admin', 126 | # Uncomment the next line to enable admin documentation: 127 | # 'django.contrib.admindocs', 128 | ) 129 | 130 | # A sample logging configuration. The only tangible logging 131 | # performed by this configuration is to send an email to 132 | # the site admins on every HTTP 500 error when DEBUG=False. 133 | # See http://docs.djangoproject.com/en/dev/topics/logging for 134 | # more details on how to customize your logging configuration. 135 | LOGGING = { 136 | 'version': 1, 137 | 'disable_existing_loggers': False, 138 | 'filters': { 139 | 'require_debug_false': { 140 | '()': 'django.utils.log.RequireDebugFalse' 141 | }, 142 | 'simple': { 143 | 'format': '%(levelname)s %(message)s', 144 | 'datefmt': '%y %b %d, %H:%M:%S', 145 | }, 146 | }, 147 | 'handlers': { 148 | 'mail_admins': { 149 | 'level': 'ERROR', 150 | 'filters': ['require_debug_false'], 151 | 'class': 'django.utils.log.AdminEmailHandler' 152 | }, 153 | 'console': { 154 | 'level': 'INFO', 155 | 'class': 'logging.StreamHandler', 156 | }, 157 | 'celery': { 158 | 'level': 'DEBUG', 159 | 'class': 'logging.handlers.RotatingFileHandler', 160 | 'filename': 'celery.log', 161 | 'maxBytes': 1024 * 1024 * 100, 162 | }, 163 | }, 164 | 'loggers': { 165 | 'django.request': { 166 | 'handlers': ['mail_admins'], 167 | 'level': 'ERROR', 168 | 'propagate': True, 169 | }, 170 | 'celery':{ 171 | 'handlers': ['celery', 'console'], 172 | 'level': 'DEBUG', 173 | } 174 | } 175 | } 176 | 177 | import djcelery 178 | djcelery.setup_loader() 179 | -------------------------------------------------------------------------------- /website/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | from django.contrib import admin 5 | admin.autodiscover() 6 | 7 | urlpatterns = patterns('', 8 | # Examples: 9 | # url(r'^$', 'website.views.home', name='home'), 10 | # url(r'^website/', include('website.foo.urls')), 11 | 12 | # Uncomment the admin/doc line below to enable admin documentation: 13 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 14 | 15 | # Uncomment the next line to enable the admin: 16 | url(r'^admin/', include(admin.site.urls)), 17 | ) 18 | -------------------------------------------------------------------------------- /website/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for website project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 19 | # if running multiple sites in the same mod_wsgi process. To fix this, use 20 | # mod_wsgi daemon mode with each site in its own daemon process, or use 21 | # os.environ["DJANGO_SETTINGS_MODULE"] = "website.settings" 22 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "website.settings") 23 | 24 | # This application object is used by any WSGI server configured to use this 25 | # file. This includes Django's development server, if the WSGI_APPLICATION 26 | # setting points here. 27 | from django.core.wsgi import get_wsgi_application 28 | application = get_wsgi_application() 29 | 30 | # Apply WSGI middleware here. 31 | # from helloworld.wsgi import HelloWorldApplication 32 | # application = HelloWorldApplication(application) 33 | --------------------------------------------------------------------------------