├── MyScraper ├── MyScraper │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ └── settings.cpython-37.pyc │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── myspider.cpython-37.pyc │ │ └── test.cpython-37.pyc │ │ └── myspider.py ├── books-1.html ├── books-2.html ├── books.csv ├── logs │ └── books │ │ ├── 2019-10-07T082239.069202.log │ │ ├── 2019-10-07T082701.718961.log │ │ ├── 2019-10-07T083304.294844.log │ │ ├── 2019-10-07T083621.056639.log │ │ ├── 2019-10-07T090621.997313.log │ │ ├── 2019-10-07T090641.534036.log │ │ ├── 2019-10-07T100621.938827.log │ │ ├── 2019-10-07T100630.053647.log │ │ ├── 2019-10-07T101707.474919.log │ │ ├── 2019-10-07T102610.866422.log │ │ ├── 2019-10-07T102624.737591.log │ │ ├── 2019-10-07T102659.069092.log │ │ ├── 2019-10-07T103856.640909.log │ │ ├── 2019-10-07T111119.847865.log │ │ ├── 2019-10-07T111215.651508.log │ │ ├── 2019-10-07T111238.643612.log │ │ ├── 2019-10-07T111248.305023.log │ │ ├── 2019-10-07T114318.602780.log │ │ ├── 2019-10-07T114809.139644.log │ │ ├── 2019-10-07T114817.478700.log │ │ ├── 2019-10-07T114855.099519.log │ │ ├── 2019-10-07T114904.925623.log │ │ ├── 2019-10-07T114919.692716.log │ │ ├── 2019-10-07T114930.035186.log │ │ ├── 2019-10-07T114947.603807.log │ │ ├── 2019-10-07T114955.877524.log │ │ ├── 2019-10-07T115101.577688.log │ │ ├── 2019-10-07T115140.439844.log │ │ ├── 2019-10-07T120325.525823.log │ │ ├── 2019-10-07T120419.884232.log │ │ ├── 2019-10-07T120501.620502.log │ │ ├── 2019-10-07T120520.756547.log │ │ ├── 2019-10-07T120538.901427.log │ │ ├── 2019-10-07T124445.085868.log │ │ ├── 2019-10-07T124512.672163.log │ │ ├── 2019-10-07T124927.253684.log │ │ └── 2019-10-07T125003.119352.log └── scrapy.cfg ├── requirements.txt └── webapp ├── __pycache__ ├── __init__.cpython-37.pyc └── app.cpython-37.pyc ├── app.py ├── form.html ├── static └── style.css └── templates └── simple.html /MyScraper/MyScraper/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/MyScraper/MyScraper/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /MyScraper/MyScraper/__pycache__/settings.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/MyScraper/MyScraper/__pycache__/settings.cpython-37.pyc -------------------------------------------------------------------------------- /MyScraper/MyScraper/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://docs.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class MyscraperItem(scrapy.Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | pass 15 | -------------------------------------------------------------------------------- /MyScraper/MyScraper/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # https://docs.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | 10 | 11 | class MyscraperSpiderMiddleware(object): 12 | # Not all methods need to be defined. If a method is not defined, 13 | # scrapy acts as if the spider middleware does not modify the 14 | # passed objects. 15 | 16 | @classmethod 17 | def from_crawler(cls, crawler): 18 | # This method is used by Scrapy to create your spiders. 19 | s = cls() 20 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 21 | return s 22 | 23 | def process_spider_input(self, response, spider): 24 | # Called for each response that goes through the spider 25 | # middleware and into the spider. 26 | 27 | # Should return None or raise an exception. 28 | return None 29 | 30 | def process_spider_output(self, response, result, spider): 31 | # Called with the results returned from the Spider, after 32 | # it has processed the response. 33 | 34 | # Must return an iterable of Request, dict or Item objects. 35 | for i in result: 36 | yield i 37 | 38 | def process_spider_exception(self, response, exception, spider): 39 | # Called when a spider or process_spider_input() method 40 | # (from other spider middleware) raises an exception. 41 | 42 | # Should return either None or an iterable of Request, dict 43 | # or Item objects. 44 | pass 45 | 46 | def process_start_requests(self, start_requests, spider): 47 | # Called with the start requests of the spider, and works 48 | # similarly to the process_spider_output() method, except 49 | # that it doesn’t have a response associated. 50 | 51 | # Must return only requests (not items). 52 | for r in start_requests: 53 | yield r 54 | 55 | def spider_opened(self, spider): 56 | spider.logger.info('Spider opened: %s' % spider.name) 57 | 58 | 59 | class MyscraperDownloaderMiddleware(object): 60 | # Not all methods need to be defined. If a method is not defined, 61 | # scrapy acts as if the downloader middleware does not modify the 62 | # passed objects. 63 | 64 | @classmethod 65 | def from_crawler(cls, crawler): 66 | # This method is used by Scrapy to create your spiders. 67 | s = cls() 68 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 69 | return s 70 | 71 | def process_request(self, request, spider): 72 | # Called for each request that goes through the downloader 73 | # middleware. 74 | 75 | # Must either: 76 | # - return None: continue processing this request 77 | # - or return a Response object 78 | # - or return a Request object 79 | # - or raise IgnoreRequest: process_exception() methods of 80 | # installed downloader middleware will be called 81 | return None 82 | 83 | def process_response(self, request, response, spider): 84 | # Called with the response returned from the downloader. 85 | 86 | # Must either; 87 | # - return a Response object 88 | # - return a Request object 89 | # - or raise IgnoreRequest 90 | return response 91 | 92 | def process_exception(self, request, exception, spider): 93 | # Called when a download handler or a process_request() 94 | # (from other downloader middleware) raises an exception. 95 | 96 | # Must either: 97 | # - return None: continue processing this exception 98 | # - return a Response object: stops process_exception() chain 99 | # - return a Request object: stops process_exception() chain 100 | pass 101 | 102 | def spider_opened(self, spider): 103 | spider.logger.info('Spider opened: %s' % spider.name) 104 | -------------------------------------------------------------------------------- /MyScraper/MyScraper/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class MyscraperPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /MyScraper/MyScraper/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for MyScraper project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # https://docs.scrapy.org/en/latest/topics/settings.html 9 | # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html 10 | # https://docs.scrapy.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'MyScraper' 13 | 14 | SPIDER_MODULES = ['MyScraper.spiders'] 15 | NEWSPIDER_MODULE = 'MyScraper.spiders' 16 | 17 | 18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 19 | #USER_AGENT = 'MyScraper (+http://www.yourdomain.com)' 20 | 21 | # Obey robots.txt rules 22 | ROBOTSTXT_OBEY = True 23 | 24 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 25 | #CONCURRENT_REQUESTS = 32 26 | 27 | # Configure a delay for requests for the same website (default: 0) 28 | # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay 29 | # See also autothrottle settings and docs 30 | #DOWNLOAD_DELAY = 3 31 | # The download delay setting will honor only one of: 32 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 33 | #CONCURRENT_REQUESTS_PER_IP = 16 34 | 35 | # Disable cookies (enabled by default) 36 | #COOKIES_ENABLED = False 37 | 38 | # Disable Telnet Console (enabled by default) 39 | #TELNETCONSOLE_ENABLED = False 40 | 41 | # Override the default request headers: 42 | #DEFAULT_REQUEST_HEADERS = { 43 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 44 | # 'Accept-Language': 'en', 45 | #} 46 | 47 | # Enable or disable spider middlewares 48 | # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html 49 | #SPIDER_MIDDLEWARES = { 50 | # 'MyScraper.middlewares.MyscraperSpiderMiddleware': 543, 51 | #} 52 | 53 | # Enable or disable downloader middlewares 54 | # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html 55 | #DOWNLOADER_MIDDLEWARES = { 56 | # 'MyScraper.middlewares.MyscraperDownloaderMiddleware': 543, 57 | #} 58 | 59 | # Enable or disable extensions 60 | # See https://docs.scrapy.org/en/latest/topics/extensions.html 61 | #EXTENSIONS = { 62 | # 'scrapy.extensions.telnet.TelnetConsole': None, 63 | #} 64 | 65 | # Configure item pipelines 66 | # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html 67 | #ITEM_PIPELINES = { 68 | # 'MyScraper.pipelines.MyscraperPipeline': 300, 69 | #} 70 | 71 | # Enable and configure the AutoThrottle extension (disabled by default) 72 | # See https://docs.scrapy.org/en/latest/topics/autothrottle.html 73 | #AUTOTHROTTLE_ENABLED = True 74 | # The initial download delay 75 | #AUTOTHROTTLE_START_DELAY = 5 76 | # The maximum download delay to be set in case of high latencies 77 | #AUTOTHROTTLE_MAX_DELAY = 60 78 | # The average number of requests Scrapy should be sending in parallel to 79 | # each remote server 80 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 81 | # Enable showing throttling stats for every response received: 82 | #AUTOTHROTTLE_DEBUG = False 83 | 84 | # Enable and configure HTTP caching (disabled by default) 85 | # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 86 | #HTTPCACHE_ENABLED = True 87 | #HTTPCACHE_EXPIRATION_SECS = 0 88 | #HTTPCACHE_DIR = 'httpcache' 89 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 90 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 91 | -------------------------------------------------------------------------------- /MyScraper/MyScraper/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /MyScraper/MyScraper/spiders/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/MyScraper/MyScraper/spiders/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /MyScraper/MyScraper/spiders/__pycache__/myspider.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/MyScraper/MyScraper/spiders/__pycache__/myspider.cpython-37.pyc -------------------------------------------------------------------------------- /MyScraper/MyScraper/spiders/__pycache__/test.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/MyScraper/MyScraper/spiders/__pycache__/test.cpython-37.pyc -------------------------------------------------------------------------------- /MyScraper/MyScraper/spiders/myspider.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | import scrapy 3 | 4 | dictionary = {'One':1, 'Two':2, 'Three':3, 'Four':4, 'Five':5} 5 | 6 | pages=int(input('How Many Pages Do you Want to Scrape?')) 7 | 8 | class ThespiderSpider(scrapy.Spider): 9 | name = 'books' 10 | start_urls = ['http://books.toscrape.com/catalogue/page-{}.html'.format(i+1) for i in range(pages)] 11 | 12 | def parse(self, response): 13 | data={} 14 | books=response.css('ol.row') 15 | for book in books: 16 | for b in book.css('article.product_pod'): 17 | yield { 18 | 'Title' : b.css('a::attr(title)').getall()[0], 19 | 'Price' : b.css('div.product_price p.price_color::text').getall()[0], 20 | 'Stock' : b.css('div.product_price p.instock.availability::text').getall()[1].strip(), 21 | 'Star' : [v for k,v in dictionary.items() if k in b.css('p::attr(class)').getall()[0].split()[-1]][0] 22 | } 23 | 24 | 25 | -------------------------------------------------------------------------------- /MyScraper/books.csv: -------------------------------------------------------------------------------- 1 | Title,Price,Star,Stock 2 | A Light in the Attic,£51.77,Three,In stock 3 | Tipping the Velvet,£53.74,One,In stock 4 | Soumission,£50.10,One,In stock 5 | Sharp Objects,£47.82,Four,In stock 6 | Sapiens: A Brief History of Humankind,£54.23,Five,In stock 7 | The Requiem Red,£22.65,One,In stock 8 | The Dirty Little Secrets of Getting Your Dream Job,£33.34,Four,In stock 9 | "The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull",£17.93,Three,In stock 10 | The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics,£22.60,Four,In stock 11 | The Black Maria,£52.15,One,In stock 12 | "Starving Hearts (Triangular Trade Trilogy, #1)",£13.99,Two,In stock 13 | Shakespeare's Sonnets,£20.66,Four,In stock 14 | Set Me Free,£17.46,Five,In stock 15 | Scott Pilgrim's Precious Little Life (Scott Pilgrim #1),£52.29,Five,In stock 16 | Rip it Up and Start Again,£35.02,Five,In stock 17 | "Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991",£57.25,Three,In stock 18 | Olio,£23.88,One,In stock 19 | Mesaerion: The Best Science Fiction Stories 1800-1849,£37.59,One,In stock 20 | Libertarianism for Beginners,£51.33,Two,In stock 21 | It's Only the Himalayas,£45.17,Two,In stock 22 | The Nameless City (The Nameless City #1),£38.16,Four,In stock 23 | The Murder That Never Was (Forensic Instincts #5),£54.11,Three,In stock 24 | The Most Perfect Thing: Inside (and Outside) a Bird's Egg,£42.96,Four,In stock 25 | "The Mindfulness and Acceptance Workbook for Anxiety: A Guide to Breaking Free from Anxiety, Phobias, and Worry Using Acceptance and Commitment Therapy",£23.89,Four,In stock 26 | The Life-Changing Magic of Tidying Up: The Japanese Art of Decluttering and Organizing,£16.77,Three,In stock 27 | "The Inefficiency Assassin: Time Management Tactics for Working Smarter, Not Longer",£20.59,Five,In stock 28 | The Gutsy Girl: Escapades for Your Life of Epic Adventure,£37.13,One,In stock 29 | The Electric Pencil: Drawings from Inside State Hospital No. 3,£56.06,One,In stock 30 | The Death of Humanity: and the Case for Life,£58.11,Four,In stock 31 | "The Bulletproof Diet: Lose up to a Pound a Day, Reclaim Energy and Focus, Upgrade Your Life",£49.05,Three,In stock 32 | The Art Forger,£40.76,Three,In stock 33 | The Age of Genius: The Seventeenth Century and the Birth of the Modern Mind,£19.73,One,In stock 34 | The Activist's Tao Te Ching: Ancient Advice for a Modern Revolution,£32.24,Five,In stock 35 | Spark Joy: An Illustrated Master Class on the Art of Organizing and Tidying Up,£41.83,Four,In stock 36 | Soul Reader,£39.58,Two,In stock 37 | Security,£39.25,Two,In stock 38 | "Saga, Volume 6 (Saga (Collected Editions) #6)",£25.02,Three,In stock 39 | "Saga, Volume 5 (Saga (Collected Editions) #5)",£51.04,Two,In stock 40 | Reskilling America: Learning to Labor in the Twenty-First Century,£19.83,Two,In stock 41 | "Rat Queens, Vol. 3: Demons (Rat Queens (Collected Editions) #11-15)",£50.40,Three,In stock 42 | In Her Wake,£12.84,One,In stock 43 | How Music Works,£37.32,Two,In stock 44 | "Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More",£30.52,Three,In stock 45 | Chase Me (Paris Nights #2),£25.27,Five,In stock 46 | Black Dust,£34.53,Five,In stock 47 | Birdsong: A Story in Pictures,£54.64,Three,In stock 48 | America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana,£22.50,Three,In stock 49 | Aladdin and His Wonderful Lamp,£53.13,Three,In stock 50 | Worlds Elsewhere: Journeys Around Shakespeare’s Globe,£40.30,Five,In stock 51 | Wall and Piece,£44.18,Four,In stock 52 | The Four Agreements: A Practical Guide to Personal Freedom,£17.66,Five,In stock 53 | The Five Love Languages: How to Express Heartfelt Commitment to Your Mate,£31.05,Three,In stock 54 | The Elephant Tree,£23.82,Five,In stock 55 | The Bear and the Piano,£36.89,One,In stock 56 | Sophie's World,£15.94,Five,In stock 57 | Slow States of Collapse: Poems,£57.31,Three,In stock 58 | Reasons to Stay Alive,£26.41,Two,In stock 59 | Private Paris (Private #10),£47.61,Five,In stock 60 | #HigherSelfie: Wake Up Your Life. Free Your Soul. Find Your Tribe.,£23.11,Five,In stock 61 | Without Borders (Wanderlove #1),£45.07,Two,In stock 62 | When We Collided,£31.77,One,In stock 63 | "We Love You, Charlie Freeman",£50.27,Five,In stock 64 | Untitled Collection: Sabbath Poems 2014,£14.27,Four,In stock 65 | "Unseen City: The Majesty of Pigeons, the Discreet Charm of Snails & Other Wonders of the Urban Wilderness",£44.18,Four,In stock 66 | Unicorn Tracks,£18.78,Three,In stock 67 | "Unbound: How Eight Technologies Made Us Human, Transformed Society, and Brought Our World to the Brink",£25.52,One,In stock 68 | Tsubasa: WoRLD CHRoNiCLE 2 (Tsubasa WoRLD CHRoNiCLE #2),£16.28,One,In stock 69 | "Princess Jellyfish 2-in-1 Omnibus, Vol. 01 (Princess Jellyfish 2-in-1 Omnibus #1)",£13.61,Five,In stock 70 | Princess Between Worlds (Wide-Awake Princess #5),£13.34,Five,In stock 71 | "Pop Gun War, Volume 1: Gift",£18.97,One,In stock 72 | "Political Suicide: Missteps, Peccadilloes, Bad Calls, Backroom Hijinx, Sordid Pasts, Rotten Breaks, and Just Plain Dumb Mistakes in the Annals of American Politics",£36.28,Two,In stock 73 | Patience,£10.16,Three,In stock 74 | "Outcast, Vol. 1: A Darkness Surrounds Him (Outcast #1)",£15.44,Four,In stock 75 | orange: The Complete Collection 1 (orange: The Complete Collection #1),£48.41,One,In stock 76 | Online Marketing for Busy Authors: A Step-By-Step Guide,£46.35,One,In stock 77 | On a Midnight Clear,£14.07,Three,In stock 78 | Obsidian (Lux #1),£14.86,Two,In stock 79 | My Paris Kitchen: Recipes and Stories,£33.37,Two,In stock 80 | Masks and Shadows,£56.40,Two,In stock 81 | "Mama Tried: Traditional Italian Cooking for the Screwed, Crude, Vegan, and Tattooed",£14.02,Four,In stock 82 | "Lumberjanes, Vol. 2: Friendship to the Max (Lumberjanes #5-8)",£46.91,Two,In stock 83 | "Lumberjanes, Vol. 1: Beware the Kitten Holy (Lumberjanes #1-4)",£45.61,Three,In stock 84 | Penny Maybe,£33.29,Three,In stock 85 | Maude (1883-1993):She Grew Up with the country,£18.02,Two,In stock 86 | "In a Dark, Dark Wood",£19.63,One,In stock 87 | Behind Closed Doors,£52.22,Four,In stock 88 | You can't bury them all: Poems,£33.63,Two,In stock 89 | Throwing Rocks at the Google Bus: How Growth Became the Enemy of Prosperity,£31.12,Three,In stock 90 | This One Summer,£19.49,Four,In stock 91 | Thirst,£17.27,Five,In stock 92 | The Torch Is Passed: A Harding Family Story,£19.09,One,In stock 93 | The Secret of Dreadwillow Carse,£56.13,One,In stock 94 | "The Pioneer Woman Cooks: Dinnertime: Comfort Classics, Freezer Food, 16-Minute Meals, and Other Delicious Ways to Solve Supper!",£56.41,One,In stock 95 | The Past Never Ends,£56.50,Four,In stock 96 | The Natural History of Us (The Fine Art of Pretending #2),£45.22,Three,In stock 97 | Lumberjanes Vol. 3: A Terrible Plan (Lumberjanes #9-12),£19.92,Two,In stock 98 | "Layered: Baking, Building, and Styling Spectacular Cakes",£40.11,One,In stock 99 | Judo: Seven Steps to Black Belt (an Introductory Guide for Beginners),£53.90,Two,In stock 100 | Join,£35.67,Five,In stock 101 | In the Country We Love: My Family Divided,£22.00,Four,In stock 102 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T082239.069202.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 08:22:40 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 08:22:40 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Star': 'Three', 'Stock': 'In stock'} 5 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Star': 'One', 'Stock': 'In stock'} 7 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Star': 'One', 'Stock': 'In stock'} 9 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Star': 'Four', 'Stock': 'In stock'} 11 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Star': 'Five', 'Stock': 'In stock'} 13 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Star': 'One', 'Stock': 'In stock'} 15 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Star': 'Four', 'Stock': 'In stock'} 17 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Star': 'Three', 'Stock': 'In stock'} 19 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Star': 'Four', 'Stock': 'In stock'} 21 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Star': 'One', 'Stock': 'In stock'} 23 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Star': 'Two', 'Stock': 'In stock'} 25 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Star': 'Four', 'Stock': 'In stock'} 27 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Star': 'Five', 'Stock': 'In stock'} 29 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Star': 'Five', 'Stock': 'In stock'} 31 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Star': 'Five', 'Stock': 'In stock'} 33 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Star': 'Three', 'Stock': 'In stock'} 35 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Star': 'One', 'Stock': 'In stock'} 37 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Star': 'One', 'Stock': 'In stock'} 39 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Star': 'Two', 'Stock': 'In stock'} 41 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Star': 'Two', 'Stock': 'In stock'} 43 | 2019-10-07 08:22:40 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Star': 'One', 'Stock': 'In stock'} 46 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Star': 'Two', 'Stock': 'In stock'} 48 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Star': 'Three', 'Stock': 'In stock'} 50 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Star': 'Five', 'Stock': 'In stock'} 52 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Star': 'Five', 'Stock': 'In stock'} 54 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Star': 'Three', 'Stock': 'In stock'} 56 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Star': 'Three', 'Stock': 'In stock'} 58 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Star': 'Three', 'Stock': 'In stock'} 60 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Star': 'Five', 'Stock': 'In stock'} 62 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Star': 'Four', 'Stock': 'In stock'} 64 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Star': 'Five', 'Stock': 'In stock'} 66 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Star': 'Three', 'Stock': 'In stock'} 68 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Star': 'Five', 'Stock': 'In stock'} 70 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Star': 'One', 'Stock': 'In stock'} 72 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Star': 'Five', 'Stock': 'In stock'} 74 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Star': 'Three', 'Stock': 'In stock'} 76 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Star': 'Two', 'Stock': 'In stock'} 78 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Star': 'One', 'Stock': 'In stock'} 80 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Star': 'Four', 'Stock': 'In stock'} 82 | 2019-10-07 08:22:40 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Star': 'Two', 'Stock': 'In stock'} 84 | 2019-10-07 08:22:40 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 08:22:40 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.464772, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 12, 22, 40, 603823), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 48943104, 100 | 'memusage/startup': 48934912, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 12, 22, 39, 139051)} 110 | 2019-10-07 08:22:40 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T082701.718961.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 08:27:02 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 08:27:02 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Star': 'Three', 'Stock': 'In stock'} 5 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Star': 'One', 'Stock': 'In stock'} 7 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Star': 'One', 'Stock': 'In stock'} 9 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Star': 'Four', 'Stock': 'In stock'} 11 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Star': 'Five', 'Stock': 'In stock'} 13 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Star': 'One', 'Stock': 'In stock'} 15 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Star': 'Four', 'Stock': 'In stock'} 17 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Star': 'Three', 'Stock': 'In stock'} 19 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Star': 'Four', 'Stock': 'In stock'} 21 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Star': 'One', 'Stock': 'In stock'} 23 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Star': 'Two', 'Stock': 'In stock'} 25 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Star': 'Four', 'Stock': 'In stock'} 27 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Star': 'Five', 'Stock': 'In stock'} 29 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Star': 'Five', 'Stock': 'In stock'} 31 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Star': 'Five', 'Stock': 'In stock'} 33 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Star': 'Three', 'Stock': 'In stock'} 35 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Star': 'One', 'Stock': 'In stock'} 37 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Star': 'One', 'Stock': 'In stock'} 39 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Star': 'Two', 'Stock': 'In stock'} 41 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Star': 'Two', 'Stock': 'In stock'} 43 | 2019-10-07 08:27:02 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Star': 'One', 'Stock': 'In stock'} 46 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Star': 'Two', 'Stock': 'In stock'} 48 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Star': 'Three', 'Stock': 'In stock'} 50 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Star': 'Five', 'Stock': 'In stock'} 52 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Star': 'Five', 'Stock': 'In stock'} 54 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Star': 'Three', 'Stock': 'In stock'} 56 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Star': 'Three', 'Stock': 'In stock'} 58 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Star': 'Three', 'Stock': 'In stock'} 60 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Star': 'Five', 'Stock': 'In stock'} 62 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Star': 'Four', 'Stock': 'In stock'} 64 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Star': 'Five', 'Stock': 'In stock'} 66 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Star': 'Three', 'Stock': 'In stock'} 68 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Star': 'Five', 'Stock': 'In stock'} 70 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Star': 'One', 'Stock': 'In stock'} 72 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Star': 'Five', 'Stock': 'In stock'} 74 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Star': 'Three', 'Stock': 'In stock'} 76 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Star': 'Two', 'Stock': 'In stock'} 78 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Star': 'One', 'Stock': 'In stock'} 80 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Star': 'Four', 'Stock': 'In stock'} 82 | 2019-10-07 08:27:02 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Star': 'Two', 'Stock': 'In stock'} 84 | 2019-10-07 08:27:02 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 08:27:02 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.661318, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 12, 27, 2, 391773), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 51507200, 100 | 'memusage/startup': 51507200, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 12, 27, 1, 730455)} 110 | 2019-10-07 08:27:02 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T083304.294844.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 08:33:04 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 08:33:05 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 08:33:05 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 08:33:05 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 08:33:05 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 08:33:05 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.026372, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 12, 33, 5, 396453), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 48943104, 100 | 'memusage/startup': 48934912, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 12, 33, 4, 370081)} 110 | 2019-10-07 08:33:05 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T083621.056639.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 08:36:21 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 08:36:21 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 08:36:21 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 08:36:21 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 08:36:21 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 08:36:21 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.806945, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 12, 36, 21, 874219), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 51585024, 100 | 'memusage/startup': 51585024, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 12, 36, 21, 67274)} 110 | 2019-10-07 08:36:21 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T090621.997313.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 09:06:22 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 09:06:22 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 09:06:22 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 09:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 09:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 09:06:23 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 09:06:23 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.999746, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 13, 6, 23, 8205), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 53121024, 100 | 'memusage/startup': 53121024, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 13, 6, 22, 8459)} 110 | 2019-10-07 09:06:23 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T090641.534036.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 09:06:41 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 09:06:41 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 09:06:42 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 09:06:42 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 09:06:42 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 09:06:42 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.699336, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 13, 6, 42, 242078), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 53837824, 100 | 'memusage/startup': 53837824, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 13, 6, 41, 542742)} 110 | 2019-10-07 09:06:42 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T100621.938827.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:06:22 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:06:22 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 10:06:22 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 10:06:22 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 10:06:22 [scrapy] DEBUG: Crawled (200) (referer: None) 45 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 46 | {'Title': 'Slow States of Collapse: Poems', 'Price': '£57.31', 'Stock': 'In stock', 'Star': 3} 47 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 48 | {'Title': 'Reasons to Stay Alive', 'Price': '£26.41', 'Stock': 'In stock', 'Star': 2} 49 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 50 | {'Title': 'Private Paris (Private #10)', 'Price': '£47.61', 'Stock': 'In stock', 'Star': 5} 51 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 52 | {'Title': '#HigherSelfie: Wake Up Your Life. Free Your Soul. Find Your Tribe.', 'Price': '£23.11', 'Stock': 'In stock', 'Star': 5} 53 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 54 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 55 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 56 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 57 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 58 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 59 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 60 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 61 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 62 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 63 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 64 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 65 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 66 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 67 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 68 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 69 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 70 | {'Title': 'Without Borders (Wanderlove #1)', 'Price': '£45.07', 'Stock': 'In stock', 'Star': 2} 71 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 72 | {'Title': 'When We Collided', 'Price': '£31.77', 'Stock': 'In stock', 'Star': 1} 73 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 74 | {'Title': 'We Love You, Charlie Freeman', 'Price': '£50.27', 'Stock': 'In stock', 'Star': 5} 75 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 76 | {'Title': 'Untitled Collection: Sabbath Poems 2014', 'Price': '£14.27', 'Stock': 'In stock', 'Star': 4} 77 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 78 | {'Title': 'Unseen City: The Majesty of Pigeons, the Discreet Charm of Snails & Other Wonders of the Urban Wilderness', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 79 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 80 | {'Title': 'Unicorn Tracks', 'Price': '£18.78', 'Stock': 'In stock', 'Star': 3} 81 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 82 | {'Title': 'Unbound: How Eight Technologies Made Us Human, Transformed Society, and Brought Our World to the Brink', 'Price': '£25.52', 'Stock': 'In stock', 'Star': 1} 83 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 84 | {'Title': 'Tsubasa: WoRLD CHRoNiCLE 2 (Tsubasa WoRLD CHRoNiCLE #2)', 'Price': '£16.28', 'Stock': 'In stock', 'Star': 1} 85 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 86 | {'Title': 'Throwing Rocks at the Google Bus: How Growth Became the Enemy of Prosperity', 'Price': '£31.12', 'Stock': 'In stock', 'Star': 3} 87 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 88 | {'Title': 'This One Summer', 'Price': '£19.49', 'Stock': 'In stock', 'Star': 4} 89 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 90 | {'Title': 'Thirst', 'Price': '£17.27', 'Stock': 'In stock', 'Star': 5} 91 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 92 | {'Title': 'The Torch Is Passed: A Harding Family Story', 'Price': '£19.09', 'Stock': 'In stock', 'Star': 1} 93 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 94 | {'Title': 'The Secret of Dreadwillow Carse', 'Price': '£56.13', 'Stock': 'In stock', 'Star': 1} 95 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 96 | {'Title': 'The Pioneer Woman Cooks: Dinnertime: Comfort Classics, Freezer Food, 16-Minute Meals, and Other Delicious Ways to Solve Supper!', 'Price': '£56.41', 'Stock': 'In stock', 'Star': 1} 97 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 98 | {'Title': 'The Past Never Ends', 'Price': '£56.50', 'Stock': 'In stock', 'Star': 4} 99 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 100 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 101 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 102 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 103 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 104 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 105 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 106 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 107 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 108 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 109 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 110 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 111 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 112 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 113 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 114 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 115 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 116 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 117 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 118 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 119 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 120 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 121 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 122 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 123 | 2019-10-07 10:06:23 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 124 | {'Title': 'The Natural History of Us (The Fine Art of Pretending #2)', 'Price': '£45.22', 'Stock': 'In stock', 'Star': 3} 125 | 2019-10-07 10:06:23 [scrapy] INFO: Closing spider (finished) 126 | 2019-10-07 10:06:23 [scrapy] INFO: Dumping Scrapy stats: 127 | {'downloader/request_bytes': 941, 128 | 'downloader/request_count': 4, 129 | 'downloader/request_method_count/GET': 4, 130 | 'downloader/response_bytes': 18378, 131 | 'downloader/response_count': 4, 132 | 'downloader/response_status_count/200': 3, 133 | 'downloader/response_status_count/404': 1, 134 | 'elapsed_time_seconds': 1.090344, 135 | 'finish_reason': 'finished', 136 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 6, 23, 106939), 137 | 'item_scraped_count': 60, 138 | 'log_count/DEBUG': 64, 139 | 'log_count/INFO': 7, 140 | 'memusage/max': 48861184, 141 | 'memusage/startup': 48852992, 142 | 'response_received_count': 4, 143 | 'robotstxt/request_count': 1, 144 | 'robotstxt/response_count': 1, 145 | 'robotstxt/response_status_count/404': 1, 146 | 'scheduler/dequeued': 3, 147 | 'scheduler/dequeued/memory': 3, 148 | 'scheduler/enqueued': 3, 149 | 'scheduler/enqueued/memory': 3, 150 | 'start_time': datetime.datetime(2019, 10, 7, 14, 6, 22, 16595)} 151 | 2019-10-07 10:06:23 [scrapy] INFO: Spider closed (finished) 152 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T100630.053647.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:06:30 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:06:30 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 10:06:30 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 10:06:30 [scrapy] DEBUG: Crawled (200) (referer: None) 45 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 46 | {'Title': 'Slow States of Collapse: Poems', 'Price': '£57.31', 'Stock': 'In stock', 'Star': 3} 47 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 48 | {'Title': 'Reasons to Stay Alive', 'Price': '£26.41', 'Stock': 'In stock', 'Star': 2} 49 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 50 | {'Title': 'Private Paris (Private #10)', 'Price': '£47.61', 'Stock': 'In stock', 'Star': 5} 51 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 52 | {'Title': '#HigherSelfie: Wake Up Your Life. Free Your Soul. Find Your Tribe.', 'Price': '£23.11', 'Stock': 'In stock', 'Star': 5} 53 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 54 | {'Title': 'Without Borders (Wanderlove #1)', 'Price': '£45.07', 'Stock': 'In stock', 'Star': 2} 55 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 56 | {'Title': 'When We Collided', 'Price': '£31.77', 'Stock': 'In stock', 'Star': 1} 57 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 58 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 59 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 60 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 61 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 62 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 63 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 64 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 65 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 66 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 67 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 68 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 69 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 70 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 71 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 72 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 73 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 74 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 75 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 76 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 77 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 78 | {'Title': 'We Love You, Charlie Freeman', 'Price': '£50.27', 'Stock': 'In stock', 'Star': 5} 79 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 80 | {'Title': 'Untitled Collection: Sabbath Poems 2014', 'Price': '£14.27', 'Stock': 'In stock', 'Star': 4} 81 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 82 | {'Title': 'Unseen City: The Majesty of Pigeons, the Discreet Charm of Snails & Other Wonders of the Urban Wilderness', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 83 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 84 | {'Title': 'Unicorn Tracks', 'Price': '£18.78', 'Stock': 'In stock', 'Star': 3} 85 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 86 | {'Title': 'Unbound: How Eight Technologies Made Us Human, Transformed Society, and Brought Our World to the Brink', 'Price': '£25.52', 'Stock': 'In stock', 'Star': 1} 87 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 88 | {'Title': 'Tsubasa: WoRLD CHRoNiCLE 2 (Tsubasa WoRLD CHRoNiCLE #2)', 'Price': '£16.28', 'Stock': 'In stock', 'Star': 1} 89 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 90 | {'Title': 'Throwing Rocks at the Google Bus: How Growth Became the Enemy of Prosperity', 'Price': '£31.12', 'Stock': 'In stock', 'Star': 3} 91 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 92 | {'Title': 'This One Summer', 'Price': '£19.49', 'Stock': 'In stock', 'Star': 4} 93 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 94 | {'Title': 'Thirst', 'Price': '£17.27', 'Stock': 'In stock', 'Star': 5} 95 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 96 | {'Title': 'The Torch Is Passed: A Harding Family Story', 'Price': '£19.09', 'Stock': 'In stock', 'Star': 1} 97 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 98 | {'Title': 'The Secret of Dreadwillow Carse', 'Price': '£56.13', 'Stock': 'In stock', 'Star': 1} 99 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 100 | {'Title': 'The Pioneer Woman Cooks: Dinnertime: Comfort Classics, Freezer Food, 16-Minute Meals, and Other Delicious Ways to Solve Supper!', 'Price': '£56.41', 'Stock': 'In stock', 'Star': 1} 101 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 102 | {'Title': 'The Past Never Ends', 'Price': '£56.50', 'Stock': 'In stock', 'Star': 4} 103 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-3.html> 104 | {'Title': 'The Natural History of Us (The Fine Art of Pretending #2)', 'Price': '£45.22', 'Stock': 'In stock', 'Star': 3} 105 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 106 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 107 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 108 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 109 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 110 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 111 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 112 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 113 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 114 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 115 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 116 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 117 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 118 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 119 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 120 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 121 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 122 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 123 | 2019-10-07 10:06:30 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 124 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 125 | 2019-10-07 10:06:30 [scrapy] INFO: Closing spider (finished) 126 | 2019-10-07 10:06:30 [scrapy] INFO: Dumping Scrapy stats: 127 | {'downloader/request_bytes': 941, 128 | 'downloader/request_count': 4, 129 | 'downloader/request_method_count/GET': 4, 130 | 'downloader/response_bytes': 18378, 131 | 'downloader/response_count': 4, 132 | 'downloader/response_status_count/200': 3, 133 | 'downloader/response_status_count/404': 1, 134 | 'elapsed_time_seconds': 0.839578, 135 | 'finish_reason': 'finished', 136 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 6, 30, 902488), 137 | 'item_scraped_count': 60, 138 | 'log_count/DEBUG': 64, 139 | 'log_count/INFO': 7, 140 | 'memusage/max': 52350976, 141 | 'memusage/startup': 52350976, 142 | 'response_received_count': 4, 143 | 'robotstxt/request_count': 1, 144 | 'robotstxt/response_count': 1, 145 | 'robotstxt/response_status_count/404': 1, 146 | 'scheduler/dequeued': 3, 147 | 'scheduler/dequeued/memory': 3, 148 | 'scheduler/enqueued': 3, 149 | 'scheduler/enqueued/memory': 3, 150 | 'start_time': datetime.datetime(2019, 10, 7, 14, 6, 30, 62910)} 151 | 2019-10-07 10:06:30 [scrapy] INFO: Spider closed (finished) 152 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T101707.474919.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:17:08 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:17:08 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 10:17:08 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 10:17:08 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 10:17:09 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 10:17:09 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 10:17:09 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.492462, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 17, 9, 43866), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 49504256, 100 | 'memusage/startup': 49504256, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 14, 17, 7, 551404)} 110 | 2019-10-07 10:17:09 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T102610.866422.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:26:11 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:26:11 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:26:11 [scrapy] DEBUG: Crawled (200) (referer: None) 4 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 5 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 6 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 7 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 8 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 9 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 10 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 11 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 12 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 13 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 14 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 15 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 16 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 17 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 18 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 19 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 20 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 21 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 22 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 23 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 24 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 25 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 26 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 27 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 28 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 29 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 30 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 31 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 32 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 33 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 34 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 35 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 36 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 37 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 38 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 39 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 40 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 41 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 42 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 43 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 44 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 45 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 46 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 47 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 48 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 49 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 50 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 51 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 53 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 55 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 57 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 58 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 59 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 60 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 61 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 62 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 63 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 64 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 10:26:11 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 10:26:11 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 10:26:11 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.950009, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 26, 11, 825039), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 52023296, 100 | 'memusage/startup': 52023296, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 14, 26, 10, 875030)} 110 | 2019-10-07 10:26:11 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T102624.737591.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:26:25 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:26:25 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:26:25 [scrapy] DEBUG: Crawled (200) (referer: None) 4 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 5 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 6 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 7 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 8 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 9 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 10 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 11 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 12 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 13 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 14 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 15 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 16 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 17 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 18 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 19 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 20 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 21 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 22 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 23 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 24 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 25 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 26 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 27 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 28 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 29 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 30 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 31 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 32 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 33 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 34 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 35 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 36 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 37 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 38 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 39 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 40 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 41 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 42 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 43 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 44 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 46 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 48 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 50 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 54 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 60 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 61 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 62 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 10:26:25 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 10:26:25 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 10:26:25 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.706597, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 26, 25, 455510), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 53805056, 100 | 'memusage/startup': 53805056, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 14, 26, 24, 748913)} 110 | 2019-10-07 10:26:25 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T102659.069092.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:26:59 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:26:59 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 10:26:59 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 10:26:59 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 10:26:59 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 10:26:59 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.711374, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 26, 59, 790196), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 54489088, 100 | 'memusage/startup': 54489088, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 14, 26, 59, 78822)} 110 | 2019-10-07 10:26:59 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T103856.640909.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 10:38:57 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 10:38:57 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 10:38:57 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 10:38:57 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 10:38:58 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 10:38:58 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 10:38:58 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.308668, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 14, 38, 58, 104919), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 92061696, 100 | 'memusage/startup': 92057600, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 14, 38, 56, 796251)} 110 | 2019-10-07 10:38:58 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T111119.847865.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 11:11:20 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 11:11:20 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 11:11:20 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 11:11:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 11:11:20 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 11:11:20 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.023924, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 15, 11, 20, 963254), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 91963392, 100 | 'memusage/startup': 91963392, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 15, 11, 19, 939330)} 110 | 2019-10-07 11:11:20 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T111215.651508.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 11:12:15 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 11:12:16 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 11:12:16 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 11:12:16 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 11:12:16 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 11:12:16 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.685223, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 15, 12, 16, 345349), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 94396416, 100 | 'memusage/startup': 94396416, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 15, 12, 15, 660126)} 110 | 2019-10-07 11:12:16 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T111238.643612.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 11:12:38 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 11:12:39 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 11:12:39 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 11:12:39 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 11:12:39 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 11:12:39 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.678962, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 15, 12, 39, 331841), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 95334400, 100 | 'memusage/startup': 95334400, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 15, 12, 38, 652879)} 110 | 2019-10-07 11:12:39 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T111248.305023.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 11:12:48 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 11:12:48 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 11:12:48 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 11:12:48 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 11:12:49 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 11:12:49 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 11:12:49 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.743234, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 15, 12, 49, 58166), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 96309248, 100 | 'memusage/startup': 96309248, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 15, 12, 48, 314932)} 110 | 2019-10-07 11:12:49 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T120325.525823.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 12:03:26 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 12:03:26 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 12:03:26 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 12:03:26 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 12:03:26 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 12:03:26 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.049994, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 16, 3, 26, 669570), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 91983872, 100 | 'memusage/startup': 91983872, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 16, 3, 25, 619576)} 110 | 2019-10-07 12:03:26 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T120419.884232.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 12:04:20 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 12:04:20 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 12:04:20 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 12:04:20 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 12:04:20 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 12:04:20 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 0.678251, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 16, 4, 20, 573528), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 94306304, 100 | 'memusage/startup': 94306304, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 16, 4, 19, 895277)} 110 | 2019-10-07 12:04:20 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/logs/books/2019-10-07T120501.620502.log: -------------------------------------------------------------------------------- 1 | 2019-10-07 12:05:02 [scrapy] DEBUG: Crawled (404) (referer: None) 2 | 2019-10-07 12:05:03 [scrapy] DEBUG: Crawled (200) (referer: None) 3 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 4 | {'Title': 'A Light in the Attic', 'Price': '£51.77', 'Stock': 'In stock', 'Star': 3} 5 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 6 | {'Title': 'Tipping the Velvet', 'Price': '£53.74', 'Stock': 'In stock', 'Star': 1} 7 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 8 | {'Title': 'Soumission', 'Price': '£50.10', 'Stock': 'In stock', 'Star': 1} 9 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 10 | {'Title': 'Sharp Objects', 'Price': '£47.82', 'Stock': 'In stock', 'Star': 4} 11 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 12 | {'Title': 'Sapiens: A Brief History of Humankind', 'Price': '£54.23', 'Stock': 'In stock', 'Star': 5} 13 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 14 | {'Title': 'The Requiem Red', 'Price': '£22.65', 'Stock': 'In stock', 'Star': 1} 15 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 16 | {'Title': 'The Dirty Little Secrets of Getting Your Dream Job', 'Price': '£33.34', 'Stock': 'In stock', 'Star': 4} 17 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 18 | {'Title': 'The Coming Woman: A Novel Based on the Life of the Infamous Feminist, Victoria Woodhull', 'Price': '£17.93', 'Stock': 'In stock', 'Star': 3} 19 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 20 | {'Title': 'The Boys in the Boat: Nine Americans and Their Epic Quest for Gold at the 1936 Berlin Olympics', 'Price': '£22.60', 'Stock': 'In stock', 'Star': 4} 21 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 22 | {'Title': 'The Black Maria', 'Price': '£52.15', 'Stock': 'In stock', 'Star': 1} 23 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 24 | {'Title': 'Starving Hearts (Triangular Trade Trilogy, #1)', 'Price': '£13.99', 'Stock': 'In stock', 'Star': 2} 25 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 26 | {'Title': "Shakespeare's Sonnets", 'Price': '£20.66', 'Stock': 'In stock', 'Star': 4} 27 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 28 | {'Title': 'Set Me Free', 'Price': '£17.46', 'Stock': 'In stock', 'Star': 5} 29 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 30 | {'Title': "Scott Pilgrim's Precious Little Life (Scott Pilgrim #1)", 'Price': '£52.29', 'Stock': 'In stock', 'Star': 5} 31 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 32 | {'Title': 'Rip it Up and Start Again', 'Price': '£35.02', 'Stock': 'In stock', 'Star': 5} 33 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 34 | {'Title': 'Our Band Could Be Your Life: Scenes from the American Indie Underground, 1981-1991', 'Price': '£57.25', 'Stock': 'In stock', 'Star': 3} 35 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 36 | {'Title': 'Olio', 'Price': '£23.88', 'Stock': 'In stock', 'Star': 1} 37 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 38 | {'Title': 'Mesaerion: The Best Science Fiction Stories 1800-1849', 'Price': '£37.59', 'Stock': 'In stock', 'Star': 1} 39 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 40 | {'Title': 'Libertarianism for Beginners', 'Price': '£51.33', 'Stock': 'In stock', 'Star': 2} 41 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-1.html> 42 | {'Title': "It's Only the Himalayas", 'Price': '£45.17', 'Stock': 'In stock', 'Star': 2} 43 | 2019-10-07 12:05:03 [scrapy] DEBUG: Crawled (200) (referer: None) 44 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 45 | {'Title': 'In Her Wake', 'Price': '£12.84', 'Stock': 'In stock', 'Star': 1} 46 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 47 | {'Title': 'How Music Works', 'Price': '£37.32', 'Stock': 'In stock', 'Star': 2} 48 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 49 | {'Title': 'Foolproof Preserving: A Guide to Small Batch Jams, Jellies, Pickles, Condiments, and More: A Foolproof Guide to Making Small Batch Jams, Jellies, Pickles, Condiments, and More', 'Price': '£30.52', 'Stock': 'In stock', 'Star': 3} 50 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 51 | {'Title': 'Chase Me (Paris Nights #2)', 'Price': '£25.27', 'Stock': 'In stock', 'Star': 5} 52 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 53 | {'Title': 'Black Dust', 'Price': '£34.53', 'Stock': 'In stock', 'Star': 5} 54 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 55 | {'Title': 'Birdsong: A Story in Pictures', 'Price': '£54.64', 'Stock': 'In stock', 'Star': 3} 56 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 57 | {'Title': "America's Cradle of Quarterbacks: Western Pennsylvania's Football Factory from Johnny Unitas to Joe Montana", 'Price': '£22.50', 'Stock': 'In stock', 'Star': 3} 58 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 59 | {'Title': 'Aladdin and His Wonderful Lamp', 'Price': '£53.13', 'Stock': 'In stock', 'Star': 3} 60 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 61 | {'Title': 'Worlds Elsewhere: Journeys Around Shakespeare’s Globe', 'Price': '£40.30', 'Stock': 'In stock', 'Star': 5} 62 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 63 | {'Title': 'Wall and Piece', 'Price': '£44.18', 'Stock': 'In stock', 'Star': 4} 64 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 65 | {'Title': 'The Four Agreements: A Practical Guide to Personal Freedom', 'Price': '£17.66', 'Stock': 'In stock', 'Star': 5} 66 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 67 | {'Title': 'The Five Love Languages: How to Express Heartfelt Commitment to Your Mate', 'Price': '£31.05', 'Stock': 'In stock', 'Star': 3} 68 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 69 | {'Title': 'The Elephant Tree', 'Price': '£23.82', 'Stock': 'In stock', 'Star': 5} 70 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 71 | {'Title': 'The Bear and the Piano', 'Price': '£36.89', 'Stock': 'In stock', 'Star': 1} 72 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 73 | {'Title': "Sophie's World", 'Price': '£15.94', 'Stock': 'In stock', 'Star': 5} 74 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 75 | {'Title': 'Penny Maybe', 'Price': '£33.29', 'Stock': 'In stock', 'Star': 3} 76 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 77 | {'Title': 'Maude (1883-1993):She Grew Up with the country', 'Price': '£18.02', 'Stock': 'In stock', 'Star': 2} 78 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 79 | {'Title': 'In a Dark, Dark Wood', 'Price': '£19.63', 'Stock': 'In stock', 'Star': 1} 80 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 81 | {'Title': 'Behind Closed Doors', 'Price': '£52.22', 'Stock': 'In stock', 'Star': 4} 82 | 2019-10-07 12:05:03 [scrapy] DEBUG: Scraped from <200 http://books.toscrape.com/catalogue/page-2.html> 83 | {'Title': "You can't bury them all: Poems", 'Price': '£33.63', 'Stock': 'In stock', 'Star': 2} 84 | 2019-10-07 12:05:03 [scrapy] INFO: Closing spider (finished) 85 | 2019-10-07 12:05:03 [scrapy] INFO: Dumping Scrapy stats: 86 | {'downloader/request_bytes': 703, 87 | 'downloader/request_count': 3, 88 | 'downloader/request_method_count/GET': 3, 89 | 'downloader/response_bytes': 12169, 90 | 'downloader/response_count': 3, 91 | 'downloader/response_status_count/200': 2, 92 | 'downloader/response_status_count/404': 1, 93 | 'elapsed_time_seconds': 1.734534, 94 | 'finish_reason': 'finished', 95 | 'finish_time': datetime.datetime(2019, 10, 7, 16, 5, 3, 365426), 96 | 'item_scraped_count': 40, 97 | 'log_count/DEBUG': 43, 98 | 'log_count/INFO': 7, 99 | 'memusage/max': 95719424, 100 | 'memusage/startup': 95719424, 101 | 'response_received_count': 3, 102 | 'robotstxt/request_count': 1, 103 | 'robotstxt/response_count': 1, 104 | 'robotstxt/response_status_count/404': 1, 105 | 'scheduler/dequeued': 2, 106 | 'scheduler/dequeued/memory': 2, 107 | 'scheduler/enqueued': 2, 108 | 'scheduler/enqueued/memory': 2, 109 | 'start_time': datetime.datetime(2019, 10, 7, 16, 5, 1, 630892)} 110 | 2019-10-07 12:05:03 [scrapy] INFO: Spider closed (finished) 111 | -------------------------------------------------------------------------------- /MyScraper/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = MyScraper.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = MyScraper 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asn1crypto==0.24.0 2 | attrs==19.1.0 3 | Automat==0.7.0 4 | certifi==2019.9.11 5 | cffi==1.12.3 6 | chardet==3.0.4 7 | Click==7.0 8 | constantly==15.1.0 9 | cryptography==2.7 10 | cssselect==1.1.0 11 | demjson==2.2.4 12 | Flask==1.1.1 13 | hyperlink==19.0.0 14 | idna==2.8 15 | incremental==17.5.0 16 | itsdangerous==1.1.0 17 | Jinja2==2.10.3 18 | lxml==4.4.1 19 | MarkupSafe==1.1.1 20 | numpy==1.17.2 21 | pandas==0.25.1 22 | parsel==1.5.2 23 | pyasn1==0.4.7 24 | pyasn1-modules==0.2.6 25 | pycparser==2.19 26 | PyDispatcher==2.0.5 27 | PyHamcrest==1.9.0 28 | pyOpenSSL==19.0.0 29 | python-dateutil==2.8.0 30 | pytz==2019.3 31 | queuelib==1.5.0 32 | requests==2.22.0 33 | Scrapy==1.7.3 34 | scrapyrt==0.11.0 35 | service-identity==18.1.0 36 | six==1.12.0 37 | Twisted==19.7.0 38 | urllib3==1.25.6 39 | w3lib==1.21.0 40 | Werkzeug==0.16.0 41 | zope.interface==4.6.0 42 | -------------------------------------------------------------------------------- /webapp/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/webapp/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /webapp/__pycache__/app.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/satssehgal/scrapyAPI/cc0fdb04668f25ce941f584f780dca960cf60a8a/webapp/__pycache__/app.cpython-37.pyc -------------------------------------------------------------------------------- /webapp/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | import json 3 | import requests 4 | import pandas as pd 5 | 6 | from flask import Flask, request, Response, render_template, session, redirect, url_for 7 | 8 | app = Flask(__name__) 9 | 10 | @app.route('/') 11 | def scrape(): 12 | 13 | params = { 14 | 'spider_name': 'books', 15 | 'start_requests': True, 16 | 17 | } 18 | response = requests.get('http://localhost:9080/crawl.json', params) 19 | data = json.loads(response.text) 20 | df=pd.DataFrame(data=data['items'], columns=['Title','Price','Stock','Star']) 21 | return render_template('simple.html', tables=[df.to_html(classes='data', index=False)], titles=df.columns.values) 22 | 23 | if __name__ == '__main__': 24 | app.run(debug=True, port=1234) -------------------------------------------------------------------------------- /webapp/form.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
11 |

Bot

12 |

Pages

13 |

14 |
15 | 16 | -------------------------------------------------------------------------------- /webapp/static/style.css: -------------------------------------------------------------------------------- 1 | body { font-family: "Lucida Sans Unicode", "Lucida Grande", sans-serif;} 2 | a, h1, h2 { color: #377ba8; } 3 | h1, h2 { margin: 0; } 4 | h1 { border-bottom: 2px solid #eee; } 5 | h2 { font-size: 1.2em; } 6 | 7 | table.dataframe, .dataframe th, .dataframe td { 8 | border: none; 9 | border-bottom: 1px solid #C8C8C8; 10 | border-collapse: collapse; 11 | text-align:left; 12 | padding: 10px; 13 | margin-bottom: 40px; 14 | font-size: 0.9em; 15 | } 16 | 17 | .male th { 18 | background-color: #add8e6; 19 | color: white; 20 | } 21 | 22 | .female th { 23 | background-color: #77dd77; 24 | color: white; 25 | } 26 | 27 | tr:nth-child(odd) { background-color:#eee; } 28 | tr:nth-child(even) { background-color:#fff; } 29 | 30 | tr:hover { background-color: #ffff99;} -------------------------------------------------------------------------------- /webapp/templates/simple.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% block head %} 5 | 6 | 7 | 8 | 9 | {% block title %}{% endblock %} Scraped Data 10 | {% endblock %} 11 | 12 | 13 |

Scraped Data

14 |
15 | {% for table in tables %} 16 | {{ table|safe }} 17 | {% endfor %} 18 |
19 | 20 | --------------------------------------------------------------------------------