├── .gitignore ├── MarketplaceAPI.py ├── MarketplaceScraper.py ├── README.md ├── docs └── filter-params.txt └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Visual Studio Code 2 | .vscode 3 | *.code-workspace 4 | 5 | # macOS 6 | .DS_Store 7 | 8 | # Virtual Environment 9 | venv/ 10 | 11 | # Development Files 12 | test.py -------------------------------------------------------------------------------- /MarketplaceAPI.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request 2 | import MarketplaceScraper 3 | 4 | API = Flask(__name__) 5 | 6 | 7 | @API.route("/locations", methods=["GET"]) 8 | def locations(): 9 | response = {} 10 | 11 | # Required parameters provided by the user 12 | locationQuery = request.args.get("locationQuery") 13 | 14 | if (locationQuery): 15 | status, error, data = MarketplaceScraper.getLocations( 16 | locationQuery=locationQuery) 17 | else: 18 | status = "Failure" 19 | error["source"] = "User" 20 | error["message"] = "Missing required parameter" 21 | data = {} 22 | 23 | response["status"] = status 24 | response["error"] = error 25 | response["data"] = data 26 | 27 | return response 28 | 29 | 30 | @API.route("/search", methods=["GET"]) 31 | def search(): 32 | response = {} 33 | 34 | # Required parameters provided by user 35 | locationLatitude = request.args.get("locationLatitude") 36 | locationLongitude = request.args.get("locationLongitude") 37 | listingQuery = request.args.get("listingQuery") 38 | 39 | if (locationLatitude and locationLongitude and listingQuery): 40 | status, error, data = MarketplaceScraper.getListings( 41 | locationLatitude=locationLatitude, locationLongitude=locationLongitude, listingQuery=listingQuery) 42 | else: 43 | status = "Failure" 44 | error["source"] = "User" 45 | error["message"] = "Missing required parameter(s)" 46 | data = {} 47 | 48 | response["status"] = status 49 | response["error"] = error 50 | response["data"] = data 51 | 52 | return response 53 | -------------------------------------------------------------------------------- /MarketplaceScraper.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import copy 4 | 5 | GRAPHQL_URL = "https://www.facebook.com/api/graphql/" 6 | GRAPHQL_HEADERS = { 7 | "sec-fetch-site": "same-origin", 8 | "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36" 9 | } 10 | 11 | 12 | def getLocations(locationQuery): 13 | data = {} 14 | 15 | requestPayload = { 16 | "variables": """{"params": {"caller": "MARKETPLACE", "page_category": ["CITY", "SUBCITY", "NEIGHBORHOOD","POSTAL_CODE"], "query": "%s"}}""" % (locationQuery), 17 | "doc_id": "5585904654783609" 18 | } 19 | 20 | status, error, facebookResponse = getFacebookResponse(requestPayload) 21 | 22 | if (status == "Success"): 23 | data["locations"] = [] # Create a locations object within data 24 | facebookResponseJSON = json.loads(facebookResponse.text) 25 | 26 | # Get location names and their ID from the facebook response 27 | for location in facebookResponseJSON["data"]["city_street_search"]["street_results"]["edges"]: 28 | locationName = location["node"]["subtitle"].split(" \u00b7")[0] 29 | 30 | # Refine location name if it is too general 31 | if (locationName == "City"): 32 | locationName = location["node"]["single_line_address"] 33 | 34 | locationLatitude = location["node"]["location"]["latitude"] 35 | locationLongitude = location["node"]["location"]["longitude"] 36 | 37 | # Add the location to the list of locations 38 | data["locations"].append({ 39 | "name": locationName, 40 | "latitude": str(locationLatitude), 41 | "longitude": str(locationLongitude) 42 | }) 43 | 44 | return (status, error, data) 45 | 46 | 47 | def getListings(locationLatitude, locationLongitude, listingQuery, numPageResults=1): 48 | data = {} 49 | 50 | rawPageResults = [] # Un-parsed list of JSON results from each page 51 | 52 | requestPayload = { 53 | "variables": """{"count":24, "params":{"bqf":{"callsite":"COMMERCE_MKTPLACE_WWW","query":"%s"},"browse_request_params":{"commerce_enable_local_pickup":true,"commerce_enable_shipping":true,"commerce_search_and_rp_available":true,"commerce_search_and_rp_condition":null,"commerce_search_and_rp_ctime_days":null,"filter_location_latitude":%s,"filter_location_longitude":%s,"filter_price_lower_bound":0,"filter_price_upper_bound":214748364700,"filter_radius_km":16},"custom_request_params":{"surface":"SEARCH"}}}""" % (listingQuery, locationLatitude, locationLongitude), 54 | "doc_id": "7111939778879383" 55 | } 56 | 57 | status, error, facebookResponse = getFacebookResponse(requestPayload) 58 | 59 | if (status == "Success"): 60 | facebookResponseJSON = json.loads(facebookResponse.text) 61 | rawPageResults.append(facebookResponseJSON) 62 | 63 | # Retrieve subsequent page results if numPageResults > 1 64 | for _ in range(1, numPageResults): 65 | pageInfo = facebookResponseJSON["data"]["marketplace_search"]["feed_units"]["page_info"] 66 | 67 | # If a next page of results exists 68 | if (pageInfo["has_next_page"]): 69 | cursor = facebookResponseJSON["data"]["marketplace_search"]["feed_units"]["page_info"]["end_cursor"] 70 | 71 | # Make a copy of the original request payload 72 | requestPayloadCopy = copy.copy(requestPayload) 73 | 74 | # Insert the cursor object into the variables object of the request payload copy 75 | requestPayloadCopy["variables"] = requestPayloadCopy["variables"].split( 76 | ) 77 | requestPayloadCopy["variables"].insert( 78 | 1, """"cursor":'{}',""".format(cursor)) 79 | requestPayloadCopy["variables"] = "".join( 80 | requestPayloadCopy["variables"]) 81 | 82 | status, error, facebookResponse = getFacebookResponse( 83 | requestPayloadCopy) 84 | 85 | if (status == "Success"): 86 | facebookResponseJSON = json.loads(facebookResponse.text) 87 | rawPageResults.append(facebookResponseJSON) 88 | else: 89 | return (status, error, data) 90 | else: 91 | return (status, error, data) 92 | 93 | # Parse the raw page results and set as the value of listingPages 94 | data["listingPages"] = parsePageResults(rawPageResults) 95 | return (status, error, data) 96 | 97 | 98 | # Helper function 99 | def getFacebookResponse(requestPayload): 100 | status = "Success" 101 | error = {} 102 | 103 | # Try making post request to Facebook, excpet return 104 | try: 105 | facebookResponse = requests.post( 106 | GRAPHQL_URL, headers=GRAPHQL_HEADERS, data=requestPayload) 107 | except requests.exceptions.RequestException as requestError: 108 | status = "Failure" 109 | error["source"] = "Request" 110 | error["message"] = str(requestError) 111 | facebookResponse = None 112 | return (status, error, facebookResponse) 113 | 114 | if (facebookResponse.status_code == 200): 115 | facebookResponseJSON = json.loads(facebookResponse.text) 116 | 117 | if (facebookResponseJSON.get("errors")): 118 | status = "Failure" 119 | error["source"] = "Facebook" 120 | error["message"] = facebookResponseJSON["errors"][0]["message"] 121 | else: 122 | status = "Failure" 123 | error["source"] = "Facebook" 124 | error["message"] = "Status code {}".format( 125 | facebookResponse.status_code) 126 | 127 | return (status, error, facebookResponse) 128 | 129 | 130 | # Helper function 131 | def parsePageResults(rawPageResults): 132 | listingPages = [] 133 | 134 | pageIndex = 0 135 | for rawPageResult in rawPageResults: 136 | 137 | # Create a new listings object within the listingPages array 138 | listingPages.append({"listings": []}) 139 | 140 | for listing in rawPageResult["data"]["marketplace_search"]["feed_units"]["edges"]: 141 | 142 | # If object is a listing 143 | if (listing["node"]["__typename"] == "MarketplaceFeedListingStoryObject"): 144 | listingID = listing["node"]["listing"]["id"] 145 | listingName = listing["node"]["listing"]["marketplace_listing_title"] 146 | listingCurrentPrice = listing["node"]["listing"]["listing_price"]["formatted_amount"] 147 | 148 | # If listing has a previous price 149 | if (listing["node"]["listing"]["strikethrough_price"]): 150 | listingPreviousPrice = listing["node"]["listing"]["strikethrough_price"]["formatted_amount"] 151 | else: 152 | listingPreviousPrice = "" 153 | 154 | listingSaleIsPending = listing["node"]["listing"]["is_pending"] 155 | listingPrimaryPhotoURL = listing["node"]["listing"]["primary_listing_photo"]["image"]["uri"] 156 | sellerName = listing["node"]["listing"]["marketplace_listing_seller"]["name"] 157 | sellerLocation = listing["node"]["listing"]["location"]["reverse_geocode"]["city_page"]["display_name"] 158 | sellerType = listing["node"]["listing"]["marketplace_listing_seller"]["__typename"] 159 | 160 | # Add the listing to its corresponding page 161 | listingPages[pageIndex]["listings"].append({ 162 | "id": listingID, 163 | "name": listingName, 164 | "currentPrice": listingCurrentPrice, 165 | "previousPrice": listingPreviousPrice, 166 | "saleIsPending": str(listingSaleIsPending).lower(), 167 | "primaryPhotoURL": listingPrimaryPhotoURL, 168 | "sellerName": sellerName, 169 | "sellerLocation": sellerLocation, 170 | "sellerType": sellerType 171 | }) 172 | 173 | pageIndex += 1 174 | 175 | return listingPages 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Marketplace API 2 | 3 | An easy-to-use Facebook Marketplace API, that functions without the need to log into a Facebook account. Wraps the Facebook GraphQL API, allowing for quick and easy retrieval of Facebook Marketplace listings and other relevant Marketplace data. 4 |

5 | ## Responses 6 | **All** endpoints will return a JSON response in the following format: 7 | ```js 8 | { 9 | "status": String, 10 | "error": { 11 | "source": String, 12 | "message": String 13 | }, 14 | "data": Array 15 | } 16 | ``` 17 | ```status```: Indicates whether a request was a success or failure. Successful requests will have a status of "Success" and failed requests will have a status of "Failure". 18 |
19 | 20 | ```error```: A request error (will be empty if no error exists). 21 | - ```error.source```: Indicates the party responsible for an error. Server-side Facebook errors will have a source of "Facebook" and errors caused by the user will have a source of "User". 22 | - ```error.message```: A detailed description of the request error. 23 |
24 | 25 | ```data```: A list of JSON objects representing the information an endpoint retrieved (will be empty if an error exists). 26 |

27 | ## Endpoints 28 | - ```/locations``` 29 |
30 | *Response:* 31 |
32 | Locations which are exact, or close matches, to the search query provided. Latitude and longitude coordinates for a location are required to find Marketplace listings in a targeted area. 33 |

34 | Example: 35 | ```json 36 | { 37 | "status": "Success", 38 | "error": {}, 39 | "data": { 40 | "locations": [ 41 | { 42 | "name": "Houston, Texas", 43 | "latitude": "29.7602", 44 | "longitude": "-95.3694" 45 | }, 46 | { 47 | "name": "Downtown Houston, TX", 48 | "latitude": "29.758767", 49 | "longitude": "-95.361523" 50 | }, 51 | { 52 | "name": "Houston, Mississippi", 53 | "latitude": "33.8981", 54 | "longitude": "-89.0017" 55 | }, 56 | { 57 | "name": "Houston, Alaska", 58 | "latitude": "61.6083", 59 | "longitude": "-149.774" 60 | } 61 | ] 62 | } 63 | } 64 | ``` 65 |
66 | 67 | *Required Parameters:* 68 |
69 | ```js 70 | { 71 | // A location in which to find the latitude and longitude coordinates 72 | "locationQuery": String 73 | } 74 | ``` 75 | --- 76 | - ```/search``` 77 |
78 | *Response:* 79 |
80 | Listings which are exact, or close matches, to the listing query and optional filter(s) provided. 81 |

82 | Example (number of pages/listings displayed and listing's primary photo URL have been removed to shorten this example): 83 | ```json 84 | { 85 | "status": "Success", 86 | "error": {}, 87 | "data": { 88 | "listingPages": [ 89 | { 90 | "listings": [ 91 | { 92 | "id": "4720490308074106", 93 | "name": "Small sectional couch", 94 | "currentPrice": "$150", 95 | "previousPrice": "", 96 | "saleIsPending": "false", 97 | "primaryPhotoURL": "Removed to shorten example", 98 | "sellerName": "Cory Yount", 99 | "sellerLocation": "Scottsdale, Arizona", 100 | "sellerType": "User" 101 | }, 102 | { 103 | "id": "296832592544544", 104 | "name": "Sectional sofa couch", 105 | "currentPrice": "$400", 106 | "previousPrice": "", 107 | "saleIsPending": "false", 108 | "primaryPhotoURL": "Removed to shorten example", 109 | "sellerName": "Alexis Brown", 110 | "sellerLocation": "Scottsdale, Arizona", 111 | "sellerType": "User" 112 | }, 113 | { 114 | "id": "261980506019699", 115 | "name": "Living spaces couch", 116 | "currentPrice": "$600", 117 | "previousPrice": "", 118 | "saleIsPending": "false", 119 | "primaryPhotoURL": "Removed to shorten example", 120 | "sellerName": "Chelsea Markley", 121 | "sellerLocation": "Scottsdale, Arizona", 122 | "sellerType": "User" 123 | }, 124 | { 125 | "id": "683280016149318", 126 | "name": "Beige couch", 127 | "currentPrice": "$100", 128 | "previousPrice": "", 129 | "saleIsPending": "false", 130 | "primaryPhotoURL": "Removed to shorten example", 131 | "sellerName": "Sarah Wilke", 132 | "sellerLocation": "Phoenix, Arizona", 133 | "sellerType": "User" 134 | }, 135 | { 136 | "id": "545545826911162", 137 | "name": "BRAND NEW gray L shaped couch with reversible chaise!", 138 | "currentPrice": "$450", 139 | "previousPrice": "$480", 140 | "saleIsPending": "false", 141 | "primaryPhotoURL": "Removed to shorten example", 142 | "sellerName": "Jamie Clark Hopkins", 143 | "sellerLocation": "Paradise Valley, Arizona", 144 | "sellerType": "User" 145 | }, 146 | { 147 | "id": "321297783315616", 148 | "name": "Leather Couch Set", 149 | "currentPrice": "$150", 150 | "previousPrice": "", 151 | "saleIsPending": "false", 152 | "primaryPhotoURL": "Removed to shorten example", 153 | "sellerName": "Samantha Crosner", 154 | "sellerLocation": "Scottsdale, Arizona", 155 | "sellerType": "User" 156 | } 157 | ] 158 | } 159 | ] 160 | } 161 | } 162 | ``` 163 |
164 | 165 | *Required Parameters:* 166 |
167 | ```js 168 | { 169 | // The latitude coordinate of the search location 170 | "locationLatitude": String, 171 | // The longitude coordinate of the search location 172 | "locationLongitude": String, 173 | // Keywords for which listings to retrieve 174 | "listingQuery": String 175 | } 176 | ``` -------------------------------------------------------------------------------- /docs/filter-params.txt: -------------------------------------------------------------------------------- 1 | Price minimum -> filter_price_lower_bound (int; default = 0) 2 | Price maximum -> filter_price_upper_bound (int; default = 214748364700) 3 | 4 | Delivery method: All -> 5 | Delivery method: Local pickup -> commerce_enable_local_pickup (boolean: true) 6 | Delivery method: Shipping -> commerce_enable_shipping (boolean: true) 7 | 8 | Item condition: All -> 9 | Item condition: New -> commerce_search_and_rp_condition (null or "new") 10 | Item condition: Used - > commerce_search_and_rp_condition (null or "used;open_box_new;refurbished;used_good;used_like_new;used_fair") 11 | 12 | Date listed: All -> 13 | Date listed: Last 24 hours -> commerce_search_and_rp_ctime_days (string: "19062;19061") 14 | 15 | Availability: Available -> commerce_search_and_rp_available (boolean: true) 16 | Availability: Sold -> commerce_search_and_rp_available (boolean: false) 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | autopep8==1.6.0 3 | beautifulsoup4==4.10.0 4 | bs4==0.0.1 5 | certifi==2021.10.8 6 | charset-normalizer==2.0.12 7 | click==8.0.4 8 | cssselect==1.1.0 9 | fake-useragent==0.1.11 10 | Flask==2.0.3 11 | idna==3.3 12 | importlib-metadata==4.11.2 13 | itsdangerous==2.1.0 14 | Jinja2==3.0.3 15 | lxml==4.8.0 16 | MarkupSafe==2.1.0 17 | parse==1.19.0 18 | pycodestyle==2.8.0 19 | pyee==8.2.2 20 | pyppeteer==1.0.2 21 | pyquery==1.4.3 22 | requests==2.27.1 23 | six==1.16.0 24 | soupsieve==2.3.1 25 | toml==0.10.2 26 | tqdm==4.63.0 27 | urllib3==1.26.8 28 | w3lib==1.22.0 29 | websockets==10.2 30 | Werkzeug==2.0.3 31 | zipp==3.7.0 32 | --------------------------------------------------------------------------------