├── Media ├── AddPageNumb.JPG ├── ConditionNew.JPG ├── CopyPasteURL.JPG ├── EbayGenericSearch.JPG ├── EbayWebsite.JPG ├── EnterNumberOfHits.JPG ├── FindNumberOfHits.JPG ├── Results.JPG └── ShowOnlySold.JPG ├── README.md └── ebay_webscraper.py /Media/AddPageNumb.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/AddPageNumb.JPG -------------------------------------------------------------------------------- /Media/ConditionNew.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/ConditionNew.JPG -------------------------------------------------------------------------------- /Media/CopyPasteURL.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/CopyPasteURL.JPG -------------------------------------------------------------------------------- /Media/EbayGenericSearch.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/EbayGenericSearch.JPG -------------------------------------------------------------------------------- /Media/EbayWebsite.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/EbayWebsite.JPG -------------------------------------------------------------------------------- /Media/EnterNumberOfHits.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/EnterNumberOfHits.JPG -------------------------------------------------------------------------------- /Media/FindNumberOfHits.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/FindNumberOfHits.JPG -------------------------------------------------------------------------------- /Media/Results.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/Results.JPG -------------------------------------------------------------------------------- /Media/ShowOnlySold.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lub34/Ebay-Webscraper-for-Getting-Average-Product-Price/5e5108a62f61e74975ed540cc8fdb5cac8e2250f/Media/ShowOnlySold.JPG -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ebay-Webscraper-for-Getting-Average-Product-Price 2 | 3 | The code in this repo is used to determine the average price of an item on Ebay given a valid search URL and the number of hits for said search. It is written in Python and requires the `requests` and `BeautifulSoup` modules to run. To install these Python modules, I either recommend installing [Anaconda](https://docs.anaconda.com/anaconda/install/) (a Python environment manager that comes with these modules) or using the following pip install lines below: 4 | 5 | [pip installation for requests module](https://www.agiratech.com/install-requests-library-in-python) 6 | 7 | [pip installation for BeautifulSoup module](https://www.tutorialspoint.com/beautiful_soup/beautiful_soup_installation.htm) 8 | 9 | **WHEN INSTALLING THE MODULES, BE MINDFUL WHETHER YOU'D PREFER THE PYTHON 2.x OR PYTHON 3.x VERSION(S)!!!** 10 | 11 | At the moment, this code only works for searches of items marked with the 'New' condition on Ebay. The hope is for an update to come out around Christmas 2021 to improve the robustness of this code. Until then, hopefully what is currently available is still of use to you. 12 | 13 | Also, shout out to Red Eyed Coder Club on YouTube, as [this](https://www.youtube.com/watch?v=m4hEAhHHykI) video of his greatly helped out with this project! 14 | -------------------------------------------------------------------------------- /ebay_webscraper.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | # TODO: 5 | # 1. Make a request to the ebay.com and get a page 6 | # 2. Collect data from detail page 7 | # 3. Collect all links to detail pages of each product 8 | # 4. Write scaped data to a csv file 9 | 10 | def get_page(url): 11 | response = requests.get(url) 12 | 13 | # If error in reaching Ebay via provided url, throw a custom error message 14 | if not response.ok: 15 | print('Server responded:', response.status_code) 16 | else: 17 | soup = BeautifulSoup(response.text, 'lxml') 18 | 19 | return soup 20 | 21 | # Returns a dictionary containing the title, price, currency, and number of item sold given an item's url link 22 | def get_detail_data(soup): 23 | # Get title of item 24 | try: 25 | title = soup.find('h1', id='itemTitle').text 26 | title = title.lstrip("Details about \xa0") 27 | except: 28 | title = '' 29 | 30 | # Get price of item 31 | try: 32 | p = soup.find('span', id='prcIsum').text.strip() 33 | currency, price = p.split(' ') 34 | except: 35 | currency = '' 36 | price = '' 37 | 38 | # Get # of items sold (note, this is inconsistent) 39 | try: 40 | sold = soup.find('span', class_='vi-qtyS').find('a').text.strip().split(' ')[0].replace('\xa0', '') 41 | except: 42 | sold = '' 43 | 44 | data = { 45 | 'title': title, 46 | 'price': price, 47 | 'currency': currency, 48 | 'total_sold': sold 49 | } 50 | 51 | return data 52 | 53 | def get_index_data(soup): 54 | try: 55 | links = soup.find_all('a', class_='s-item__link') 56 | except: 57 | links = [] 58 | 59 | urls = [item.get('href') for item in links] 60 | urls = urls[1:] 61 | 62 | return urls 63 | 64 | def main(): 65 | # Copy paste a search URL from Ebay into the single quotes below: 66 | url = 'https://www.ebay.com/sch/i.html?_from=R40&_nkw=lego+star+wars+republic+gunship+set+7676&_sacat=0&rt=nc&LH_ItemCondition=3&_pgn=1' # <-- ENTER SEARCH URL HERE 67 | numberOfHits = 15 # <-- ENTER NUMBER OF HITS FROM EBAY HERE TO HELP MINIMIZE ERROR 68 | sum = 0.0 # Will store sum of Ebay hits 69 | numberOfValidItems = 0 # Counts number of hits listed in US currency 70 | average = 0.0 # Will store average value of Ebay hits 71 | 72 | products = get_index_data(get_page(url)) 73 | 74 | for link in products[0:numberOfHits]: 75 | data = get_detail_data(get_page(link)) 76 | print(data) 77 | 78 | if (data['currency'] == 'US'): 79 | numberOfValidItems += 1 80 | sum += float(data['price'][1:]) # Add price of item to sum; the '[1:]' is used to omit the dollar sign from the price listed as a string 81 | 82 | average = sum / numberOfValidItems 83 | print("\nThe average price of this item on Ebay is: $%.2f" % average) # Print average price from relevant hits 84 | 85 | if __name__ == '__main__': 86 | main() 87 | --------------------------------------------------------------------------------