├── .gitignore
├── Analyzer.py
├── Books.py
├── Browser.py
├── LICENSE.md
├── Readme.md
├── Reviews.py
├── Sample.py
├── Tools.py
├── Writer.py
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | #
2 | # Igonre all reviews output files and folder
3 | #
4 | *.csv
5 | lists/
6 | reviews/
7 | *Reviews/
8 | Dialects/
9 |
10 | #
11 | # Ignore PyCharm and Python folders
12 | #
13 | .idea/
14 | __pycache__/
15 |
16 | #
17 | # Ignore Chrome Driver
18 | #
19 | chromedriver*
--------------------------------------------------------------------------------
/Analyzer.py:
--------------------------------------------------------------------------------
1 | import operator
2 |
3 |
4 | def output(key, value):
5 | value_type = "" if value == int(value) else ".2f"
6 | print(f"{key:<25}{value:{value_type}}")
7 |
8 |
9 | input_lines = open("reviews.csv", "r").readlines()
10 | output("Number of reviews:", len(input_lines))
11 |
12 | users = {}
13 | books = {}
14 | tokens = []
15 | token_count = 0
16 | sentence_count = 0
17 |
18 | for line in input_lines:
19 | cells = line.split('\t')
20 | user = cells[1]
21 | users[user] = users[user] + 1 if user in users else 1
22 | book = cells[2]
23 | books[book] = books[book] + 1 if book in books else 1
24 | line_tokens = cells[6].split()
25 | tokens.append(len(line_tokens))
26 | token_count += len(line_tokens)
27 | for token in line_tokens:
28 | if '.' in token:
29 | sentence_count += 1
30 |
31 | users = sorted(users.items(), key=operator.itemgetter(1))
32 | output("Number of users:", len(users))
33 | output("Avg. reviews per user:", len(input_lines) / len(users))
34 | output("Max reviews per user:", users[-1][1])
35 | output("Median reviews per user:", users[len(users) // 2][1])
36 | output("Min reviews per user:", users[1][1])
37 |
38 | books = sorted(books.items(), key=operator.itemgetter(1))
39 | output("Number of books:", len(books))
40 | output("Avg. reviews per book:", len(input_lines) / len(books))
41 | output("Max reviews per book:", books[-1][1])
42 | output("Median reviews per book:", books[len(books) // 2][1])
43 | output("Min reviews per book:", books[1][1])
44 | print(books[:5])
45 | tokens = sorted(tokens)
46 | output("Number of tokens:", token_count)
47 | output("Avg. tokens per review:", token_count / len(tokens))
48 | output("Max tokens per review:", tokens[-1])
49 | output("Median tokens per review:", tokens[len(tokens) // 2])
50 | output("Min tokens per review:", tokens[1])
51 |
52 | output("Number of sentences:", sentence_count)
53 |
--------------------------------------------------------------------------------
/Books.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | # import needed libraries
4 | from Writer import Writer
5 | from Browser import Browser
6 | from bs4 import BeautifulSoup
7 | from Tools import id_from_url, read_books, get_digits
8 |
9 |
10 | # A class to Search then Scrape lists and books from GoodReads.com
11 | class Books:
12 | def __init__(self, path=None, arabic=True):
13 | self.arabic = arabic
14 | # Browsing and writing managers
15 | self.br = Browser()
16 | self.wr = Writer(path) if path else Writer()
17 | # An array for scrapped books
18 | self._books_ids = []
19 |
20 | # Append an external books ids array to local array
21 | def append_books(self, books_ids):
22 | # Loop through sent books ids
23 | for book_id in books_ids:
24 | # Only append id if it's not stored already
25 | if book_id not in self._books_ids:
26 | self._books_ids.append(book_id)
27 |
28 | # Scrape books and write them to a file (browse is: list, lists, author or shelf)
29 | def output_books(self, keyword=None, browse="list", file_name="books"):
30 | self.wr.open(file_name, "w+")
31 | # Get books if keyword is provided, otherwise output stored books
32 | books_ids = self.get_books(keyword, browse) if keyword else self._books_ids
33 | # Loop through book ids and write them
34 | for book_id in books_ids:
35 | self.wr.write(book_id)
36 | self.wr.close()
37 |
38 | def output_books_editions(self, books_ids=None, file_name="editions"):
39 | skip = len(read_books(file_name))
40 | self.wr.open(file_name, "a+")
41 | # Loop through book ids and write their editions id
42 | for book_id in books_ids[skip:] or self._books_ids[skip:]:
43 | editions_id = self.get_book_editions_id(book_id)
44 | # Editions id is None when page refuses to load
45 | if editions_id is None: return self.wr.close()
46 | # Write editions id to file if it loads correctly
47 | self.wr.write(editions_id or "-"*7)
48 | # Display book id and editions id
49 | print(f"Book ID:\t{book_id:<15}Book Editions ID:\t{editions_id or ''}")
50 | self.wr.close()
51 | return True
52 |
53 | def output_books_edition_by_language(self, editions_ids, lang="Arabic", file_name="ara_books"):
54 | skip = len(read_books(file_name))
55 | self.wr.open(file_name, "a+")
56 | # Loop through book ids and write their editions id
57 | for editions_id in editions_ids[skip:]:
58 | books_ids = self.get_book_edition_by_language(editions_id, lang) if editions_id.isdigit() else ''
59 | # Editions id is None when page refuses to load
60 | if books_ids is None: return self.wr.close()
61 | # Write editions id to file if it loads correctly
62 | self.wr.write(books_ids or "-"*7)
63 | # Display book id and editions id
64 | print(f"Book Editions ID:\t{editions_id:<15}Books IDs:\t{books_ids or ''}")
65 | self.wr.close()
66 | # Open a new file to move done list to it
67 | self.wr.open(file_name + "_list")
68 | # Loop through previously scraped editions ids
69 | for line in read_books(file_name):
70 | # If line isn't empty
71 | if line != "-"*7:
72 | # Write each book edition id in a separate line
73 | [self.wr.write(id_) for id_ in line.split(',')]
74 | self.wr.close()
75 | return True
76 |
77 | # Main function to scrape books ids
78 | def get_books(self, keyword, browse="list"):
79 | # Get lists in search list if searching
80 | if browse == "lists":
81 | keywords = self._get_lists(keyword.replace(' ', '+'))
82 | browse = "list"
83 | # Otherwise, it's a single "list" or "shelf"
84 | else:
85 | keywords = [
86 | str(key) for key in (
87 | keyword if isinstance(keyword, list) else [keyword]
88 | )]
89 | try:
90 | # Loop through all lists
91 | for keyword in keywords:
92 | # Open each list url
93 | self.br.open_page(keyword, browse)
94 | # Scrape pages until there's no next page
95 | while True:
96 | self._scrape_list("book", self._books_ids)
97 | if not self.br.goto_next_page():
98 | break
99 | except Exception as e:
100 | print("Couldn't go to next page:", e)
101 | finally:
102 | return self._books_ids
103 |
104 | def get_book_editions_id(self, book_id):
105 | self.br.open("/book/show/", book_id)
106 | return self.br.editions_id()
107 |
108 | def get_book_edition_by_language(self, editions_id, lang):
109 | self.br.open_book_editions(editions_id)
110 | soup = BeautifulSoup(self.br.page_source, "lxml").find(class_="workEditions")
111 | if not soup: return None
112 | editions = []
113 | for details in soup.find_all(class_="editionData"):
114 | language, rating = [row.find(class_="dataValue") for row in details.find_all(class_="dataRow")[-3:-1]]
115 | if language.text.strip() == lang:
116 | reviewers = get_digits(rating.find("span").text)
117 | if reviewers > 50:
118 | editions.append(id_from_url.match(details.find(class_="bookTitle")["href"]).group(1))
119 | return ','.join(editions)
120 |
121 | # Main function to scrape lists ids
122 | def _get_lists(self, keyword):
123 | lists = []
124 | # Open GoodReads' lists search url
125 | self.br.open_list_search(keyword)
126 | # Scrape all result pages
127 | while True:
128 | self._scrape_list("list", lists)
129 | # Go to next page if there's one, otherwise break
130 | if not self.br.goto_next_page():
131 | break
132 | return lists
133 |
134 | # Scrape a single search results page
135 | def _scrape_list(self, title, array):
136 | soup = BeautifulSoup(self.br.page_source, "lxml").find(class_="tableList")
137 | if not soup: return None
138 | for book in soup.find_all("tr"):
139 | if self.arabic or get_digits(book.find(class_="minirating").text.split("—")[1]) > 1000:
140 | try: # Get id from url
141 | id_ = id_from_url.match(book.find(class_=title + "Title")["href"]).group(1)
142 | except Exception:
143 | print("Couldn't extract Book Id from URL")
144 | continue
145 | # Extract and store unique id from link
146 | if id_ not in array:
147 | array.append(id_)
148 | print(f"{title.capitalize()} {id_:<10}count:\t{len(array)}")
149 |
--------------------------------------------------------------------------------
/Browser.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | # Download link: https://sites.google.com/a/chromium.org/chromedriver/downloads
4 | from selenium.webdriver import Chrome
5 | from selenium.webdriver.common.by import By
6 | from selenium.webdriver.support.ui import WebDriverWait
7 | from selenium.webdriver.support import expected_conditions as ec
8 | from selenium.common.exceptions import (
9 | TimeoutException, NoSuchElementException, StaleElementReferenceException, WebDriverException
10 | )
11 | from Tools import write_books, read_books, id_from_url
12 |
13 |
14 | class Browser(Chrome):
15 | OPTIONS = {"goog:chromeOptions": {
16 | # Disable images loading
17 | "prefs": {"profile.managed_default_content_settings.images": 2},
18 | # Disable Chrome's GUI
19 | "args": ["--headless", "--disable-gpu"]
20 | }}
21 |
22 | def __init__(self, edition_reviews=False):
23 | Chrome.__init__(self, desired_capabilities=self.OPTIONS)
24 | # Set page loading timeout to 30 seconds
25 | self.set_page_load_timeout(30)
26 | self.edition_reviews = edition_reviews
27 | # Initialize browsing counters
28 | self.rating = self.sort = self.fails = None
29 |
30 | # Starts browser window
31 | def start(self):
32 | self.start_session(self.OPTIONS)
33 |
34 | # Login to Goodreads.com
35 | def login(self, email, password):
36 | # Open goodreads.com main page
37 | self.open()
38 | # Fill login form and submit it
39 | self.find_element_by_id("userSignInFormEmail").send_keys(email)
40 | self.find_element_by_id("user_password").send_keys(password)
41 | self.find_element_by_class_name("gr-button--signIn").click()
42 |
43 | # General shortcut to open a GoodReads page
44 | def open(self, sub_url='', keyword='', options=''):
45 | # Try to open url until it succeeds
46 | while True:
47 | try:
48 | self.get(f"https://www.goodreads.com{sub_url}{keyword}{options}")
49 | break
50 | # On connection timeout, loop again
51 | except TimeoutException:
52 | print("Reloading page...")
53 |
54 | def open_book_editions(self, book_id):
55 | self.open(f"/work/editions/{book_id}?expanded=true&sort=title")
56 |
57 | # Shortcut to open GoodReads book page
58 | def open_book_page(self, book_id):
59 | self.sort = 0
60 | self.rating = 5
61 | self.open("/book/show/", book_id, f"?text_only=true&rating={self.rating}")
62 | # Get book id from URL
63 | url_id = id_from_url.match(self.current_url).group(1)
64 | # If a book id redirect happened
65 | if book_id != url_id:
66 | # Update new book id in books.txt
67 | ids = read_books()
68 | ids[ids.index(book_id)] = url_id
69 | write_books(ids)
70 | raise ConnectionResetError(f"Redirect book id from {book_id} to {url_id}")
71 | # If only this editions's reviews are required
72 | if self.edition_reviews:
73 | # Make sure to load them before scraping first page
74 | self.switch_reviews_mode(book_id, same_mode=True)
75 | self.are_reviews_loaded()
76 | # Otherwise continue as normal
77 | else: print(f"Rating: {self.rating} Stars, Sorted: {self._SORTS[self.sort].capitalize()}")
78 |
79 | # Shortcut to open GoodReads books list or shelf
80 | def open_page(self, keyword, browse):
81 | # URL contains "show" unless it's an author page
82 | method = "show" if browse != "author" else "list"
83 | self.open(f"/{browse}/{method}/{keyword}")
84 |
85 | # Shortcut to open a GoodReads search page for books lists
86 | def open_list_search(self, keyword):
87 | self.open("/search?q=", keyword, "&search_type=lists")
88 |
89 | # Check if there's a next page
90 | def goto_next_page(self):
91 | try: # Try to find the next button
92 | next_page = self.find_element_by_class_name("next_page")
93 | # Check if button is click-able (i.e. it's anchor tag)
94 | if next_page.tag_name == "a":
95 | # Click the next page button
96 | next_page.click()
97 | return True
98 | return False
99 | # Return none if there isn't one
100 | except NoSuchElementException:
101 | print("WARNING: There is no next page!")
102 | return None
103 | except WebDriverException:
104 | print("WARNING: Retrying to goto next page!")
105 | self.implicitly_wait(1)
106 | return self.goto_next_page()
107 |
108 | def switch_reviews_mode(self, book_id, only_default=False, same_mode=False):
109 | if not same_mode:
110 | self.sort += 1
111 | if self.sort > 2 or only_default:
112 | self.sort = 0
113 | self.rating -= 1
114 | if self.rating < 1:
115 | return False
116 | print(f"Rating: {self.rating} Stars, Sorted: {self._SORTS[self.sort].capitalize()}")
117 | self.execute_script(
118 | 'document.getElementById("reviews").insertAdjacentHTML("beforeend", \'Switch Mode\');' +
122 | f'document.getElementById("switch{self.rating}{self.sort}").click()'
123 | )
124 | # self.find_element_by_id(f"switch{self.rating}{self.sort}").click()
125 | return True
126 |
127 | # Return whether reviews were loaded
128 | def are_reviews_loaded(self):
129 | try: # Add a dummy "loading" tag to DOM
130 | self.execute_script(
131 | 'document.getElementById("reviews").'
132 | 'insertAdjacentHTML("beforeend", \'loading
\');'
133 | )
134 | # Let the driver wait until the the dummy tag has disappeared
135 | WebDriverWait(self, 12).until(ec.invisibility_of_element_located((By.ID, "load_reviews")))
136 | self.fails = 0
137 | # Return true if reviews are loaded and they're more that 0, otherwise return false
138 | return len(self.find_element_by_id("bookReviews").find_elements_by_class_name("review")) > 0
139 | except (TimeoutException, StaleElementReferenceException):
140 | print("WARNING: Reviews Loading Timeout!")
141 | self.fails += 1
142 | # If reviews loading fails 3 times, raise an error
143 | if not self.edition_reviews and self.fails == 3:
144 | raise ConnectionError
145 | return False
146 |
147 | def editions_id(self):
148 | try: # To find a the parent tag (to check whether page loaded)
149 | editions = self.find_element_by_class_name("otherEditions")
150 | except (NoSuchElementException, TimeoutException):
151 | return None
152 | try: # To find child link tag (to check whether it has other editions)
153 | return id_from_url.match(
154 | editions.find_element_by_tag_name("a").get_attribute("href")
155 | ).group(1)
156 | except NoSuchElementException:
157 | return False
158 |
159 | _SORTS = ["default", "newest", "oldest"]
160 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | GNU GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 | Preamble
9 |
10 | The GNU General Public License is a free, copyleft license for
11 | software and other kinds of works.
12 |
13 | The licenses for most software and other practical works are designed
14 | to take away your freedom to share and change the works. By contrast,
15 | the GNU General Public License is intended to guarantee your freedom to
16 | share and change all versions of a program--to make sure it remains free
17 | software for all its users. We, the Free Software Foundation, use the
18 | GNU General Public License for most of our software; it applies also to
19 | any other work released this way by its authors. You can apply it to
20 | your programs, too.
21 |
22 | When we speak of free software, we are referring to freedom, not
23 | price. Our General Public Licenses are designed to make sure that you
24 | have the freedom to distribute copies of free software (and charge for
25 | them if you wish), that you receive source code or can get it if you
26 | want it, that you can change the software or use pieces of it in new
27 | free programs, and that you know you can do these things.
28 |
29 | To protect your rights, we need to prevent others from denying you
30 | these rights or asking you to surrender the rights. Therefore, you have
31 | certain responsibilities if you distribute copies of the software, or if
32 | you modify it: responsibilities to respect the freedom of others.
33 |
34 | For example, if you distribute copies of such a program, whether
35 | gratis or for a fee, you must pass on to the recipients the same
36 | freedoms that you received. You must make sure that they, too, receive
37 | or can get the source code. And you must show them these terms so they
38 | know their rights.
39 |
40 | Developers that use the GNU GPL protect your rights with two steps:
41 | (1) assert copyright on the software, and (2) offer you this License
42 | giving you legal permission to copy, distribute and/or modify it.
43 |
44 | For the developers' and authors' protection, the GPL clearly explains
45 | that there is no warranty for this free software. For both users' and
46 | authors' sake, the GPL requires that modified versions be marked as
47 | changed, so that their problems will not be attributed erroneously to
48 | authors of previous versions.
49 |
50 | Some devices are designed to deny users access to install or run
51 | modified versions of the software inside them, although the manufacturer
52 | can do so. This is fundamentally incompatible with the aim of
53 | protecting users' freedom to change the software. The systematic
54 | pattern of such abuse occurs in the area of products for individuals to
55 | use, which is precisely where it is most unacceptable. Therefore, we
56 | have designed this version of the GPL to prohibit the practice for those
57 | products. If such problems arise substantially in other domains, we
58 | stand ready to extend this provision to those domains in future versions
59 | of the GPL, as needed to protect the freedom of users.
60 |
61 | Finally, every program is threatened constantly by software patents.
62 | States should not allow patents to restrict development and use of
63 | software on general-purpose computers, but in those that do, we wish to
64 | avoid the special danger that patents applied to a free program could
65 | make it effectively proprietary. To prevent this, the GPL assures that
66 | patents cannot be used to render the program non-free.
67 |
68 | The precise terms and conditions for copying, distribution and
69 | modification follow.
70 |
71 | TERMS AND CONDITIONS
72 |
73 | 0. Definitions.
74 |
75 | "This License" refers to version 3 of the GNU General Public License.
76 |
77 | "Copyright" also means copyright-like laws that apply to other kinds of
78 | works, such as semiconductor masks.
79 |
80 | "The Program" refers to any copyrightable work licensed under this
81 | License. Each licensee is addressed as "you". "Licensees" and
82 | "recipients" may be individuals or organizations.
83 |
84 | To "modify" a work means to copy from or adapt all or part of the work
85 | in a fashion requiring copyright permission, other than the making of an
86 | exact copy. The resulting work is called a "modified version" of the
87 | earlier work or a work "based on" the earlier work.
88 |
89 | A "covered work" means either the unmodified Program or a work based
90 | on the Program.
91 |
92 | To "propagate" a work means to do anything with it that, without
93 | permission, would make you directly or secondarily liable for
94 | infringement under applicable copyright law, except executing it on a
95 | computer or modifying a private copy. Propagation includes copying,
96 | distribution (with or without modification), making available to the
97 | public, and in some countries other activities as well.
98 |
99 | To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies. Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 |
103 | An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License. If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 |
112 | 1. Source Code.
113 |
114 | The "source code" for a work means the preferred form of the work
115 | for making modifications to it. "Object code" means any non-source
116 | form of a work.
117 |
118 | A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 |
123 | The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form. A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 |
134 | The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities. However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work. For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 |
147 | The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 |
151 | The Corresponding Source for a work in source code form is that
152 | same work.
153 |
154 | 2. Basic Permissions.
155 |
156 | All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met. This License explicitly affirms your unlimited
159 | permission to run the unmodified Program. The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work. This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 |
164 | You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force. You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright. Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 |
175 | Conveying under any other circumstances is permitted solely under
176 | the conditions stated below. Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 |
179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 |
181 | No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 |
187 | When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 |
195 | 4. Conveying Verbatim Copies.
196 |
197 | You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 |
205 | You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 |
208 | 5. Conveying Modified Source Versions.
209 |
210 | You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 |
214 | a) The work must carry prominent notices stating that you modified
215 | it, and giving a relevant date.
216 |
217 | b) The work must carry prominent notices stating that it is
218 | released under this License and any conditions added under section
219 | 7. This requirement modifies the requirement in section 4 to
220 | "keep intact all notices".
221 |
222 | c) You must license the entire work, as a whole, under this
223 | License to anyone who comes into possession of a copy. This
224 | License will therefore apply, along with any applicable section 7
225 | additional terms, to the whole of the work, and all its parts,
226 | regardless of how they are packaged. This License gives no
227 | permission to license the work in any other way, but it does not
228 | invalidate such permission if you have separately received it.
229 |
230 | d) If the work has interactive user interfaces, each must display
231 | Appropriate Legal Notices; however, if the Program has interactive
232 | interfaces that do not display Appropriate Legal Notices, your
233 | work need not make them do so.
234 |
235 | A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit. Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 |
245 | 6. Conveying Non-Source Forms.
246 |
247 | You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 |
252 | a) Convey the object code in, or embodied in, a physical product
253 | (including a physical distribution medium), accompanied by the
254 | Corresponding Source fixed on a durable physical medium
255 | customarily used for software interchange.
256 |
257 | b) Convey the object code in, or embodied in, a physical product
258 | (including a physical distribution medium), accompanied by a
259 | written offer, valid for at least three years and valid for as
260 | long as you offer spare parts or customer support for that product
261 | model, to give anyone who possesses the object code either (1) a
262 | copy of the Corresponding Source for all the software in the
263 | product that is covered by this License, on a durable physical
264 | medium customarily used for software interchange, for a price no
265 | more than your reasonable cost of physically performing this
266 | conveying of source, or (2) access to copy the
267 | Corresponding Source from a network server at no charge.
268 |
269 | c) Convey individual copies of the object code with a copy of the
270 | written offer to provide the Corresponding Source. This
271 | alternative is allowed only occasionally and noncommercially, and
272 | only if you received the object code with such an offer, in accord
273 | with subsection 6b.
274 |
275 | d) Convey the object code by offering access from a designated
276 | place (gratis or for a charge), and offer equivalent access to the
277 | Corresponding Source in the same way through the same place at no
278 | further charge. You need not require recipients to copy the
279 | Corresponding Source along with the object code. If the place to
280 | copy the object code is a network server, the Corresponding Source
281 | may be on a different server (operated by you or a third party)
282 | that supports equivalent copying facilities, provided you maintain
283 | clear directions next to the object code saying where to find the
284 | Corresponding Source. Regardless of what server hosts the
285 | Corresponding Source, you remain obligated to ensure that it is
286 | available for as long as needed to satisfy these requirements.
287 |
288 | e) Convey the object code using peer-to-peer transmission, provided
289 | you inform other peers where the object code and Corresponding
290 | Source of the work are being offered to the general public at no
291 | charge under subsection 6d.
292 |
293 | A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 |
297 | A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling. In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage. For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product. A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 |
310 | "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source. The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 |
318 | If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information. But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 |
329 | The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed. Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 |
337 | Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 |
343 | 7. Additional Terms.
344 |
345 | "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law. If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 |
354 | When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it. (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.) You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 |
361 | Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 |
365 | a) Disclaiming warranty or limiting liability differently from the
366 | terms of sections 15 and 16 of this License; or
367 |
368 | b) Requiring preservation of specified reasonable legal notices or
369 | author attributions in that material or in the Appropriate Legal
370 | Notices displayed by works containing it; or
371 |
372 | c) Prohibiting misrepresentation of the origin of that material, or
373 | requiring that modified versions of such material be marked in
374 | reasonable ways as different from the original version; or
375 |
376 | d) Limiting the use for publicity purposes of names of licensors or
377 | authors of the material; or
378 |
379 | e) Declining to grant rights under trademark law for use of some
380 | trade names, trademarks, or service marks; or
381 |
382 | f) Requiring indemnification of licensors and authors of that
383 | material by anyone who conveys the material (or modified versions of
384 | it) with contractual assumptions of liability to the recipient, for
385 | any liability that these contractual assumptions directly impose on
386 | those licensors and authors.
387 |
388 | All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10. If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term. If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 |
398 | If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 |
403 | Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 |
407 | 8. Termination.
408 |
409 | You may not propagate or modify a covered work except as expressly
410 | provided under this License. Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 |
415 | However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 |
422 | Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 |
429 | Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License. If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 |
435 | 9. Acceptance Not Required for Having Copies.
436 |
437 | You are not required to accept this License in order to receive or
438 | run a copy of the Program. Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance. However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work. These actions infringe copyright if you do
443 | not accept this License. Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 |
446 | 10. Automatic Licensing of Downstream Recipients.
447 |
448 | Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License. You are not responsible
451 | for enforcing compliance by third parties with this License.
452 |
453 | An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations. If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 |
463 | You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License. For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 |
471 | 11. Patents.
472 |
473 | A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based. The
475 | work thus licensed is called the contributor's "contributor version".
476 |
477 | A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version. For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 |
487 | Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 |
492 | In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement). To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 |
499 | If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients. "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 |
513 | If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 |
521 | A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License. You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 |
536 | Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 |
540 | 12. No Surrender of Others' Freedom.
541 |
542 | If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License. If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all. For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 |
552 | 13. Use with the GNU Affero General Public License.
553 |
554 | Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work. The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 |
563 | 14. Revised Versions of this License.
564 |
565 | The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time. Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 |
570 | Each version is given a distinguishing version number. If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation. If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 |
579 | If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 |
584 | Later license versions may give you additional or different
585 | permissions. However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 |
589 | 15. Disclaimer of Warranty.
590 |
591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 |
600 | 16. Limitation of Liability.
601 |
602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 |
612 | 17. Interpretation of Sections 15 and 16.
613 |
614 | If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 |
621 | END OF TERMS AND CONDITIONS
622 |
623 | How to Apply These Terms to Your New Programs
624 |
625 | If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 |
629 | To do so, attach the following notices to the program. It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 |
634 |
635 | Copyright (C)
636 |
637 | This program is free software: you can redistribute it and/or modify
638 | it under the terms of the GNU General Public License as published by
639 | the Free Software Foundation, either version 3 of the License, or
640 | (at your option) any later version.
641 |
642 | This program is distributed in the hope that it will be useful,
643 | but WITHOUT ANY WARRANTY; without even the implied warranty of
644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645 | GNU General Public License for more details.
646 |
647 | You should have received a copy of the GNU General Public License
648 | along with this program. If not, see .
649 |
650 | Also add information on how to contact you by electronic and paper mail.
651 |
652 | If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 |
655 | Copyright (C)
656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 | This is free software, and you are welcome to redistribute it
658 | under certain conditions; type `show c' for details.
659 |
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License. Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 |
664 | You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | .
668 |
669 | The GNU General Public License does not permit incorporating your program
670 | into proprietary programs. If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library. If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License. But first, please read
674 | .
675 |
--------------------------------------------------------------------------------
/Readme.md:
--------------------------------------------------------------------------------
1 | # GoodReads Reviews Scraper
2 |
3 | This is a python 3 web scraping script to get books reviews from goodreads.com,
4 | using the web browser automation tool Selenium, and BeautifulSoup for pulling data out of HTML.
5 | I used it to scrape around 700k Arabic reviews in 2018 (Arabic reviews are fewer than English ones).
6 |
7 | ### Papers
8 | We experimented on the collected data and published details about it in two research papers:
9 | - [BRAD 1.0: Book reviews in Arabic dataset (IEEE)](https://ieeexplore.ieee.org/abstract/document/7945800).
10 | - [An Annotated Huge Dataset for Standard and Colloquial Arabic Reviews for Subjective Sentiment Analysis (Elsevier)](https://www.sciencedirect.com/science/article/pii/S1877050918321781).
11 |
12 | ### Contents
13 |
14 | - Analyzer.py: short script to display some statistics about scraped books reviews
15 | - Books.py: class to scrape books ids from goodreads list or lists search or shelf
16 | - Browser.py: subclass of Chrome WebDriver class that's specialized for GoodrReads browsing
17 | - Reviews.py: class to scrape reviews from goodreads books using books ids
18 | - Sample.py: sample script showing a complete use of the scraper with error handling
19 | - Tools.py: set of function tools that are used in other scripts
20 | - Writer.py: class to write scraped reviews to files
21 | - requirements.txt: list of Required Python modules to be installed
22 |
23 | ### Requirements
24 |
25 | To install requirements listed in requirements.txt, you'll need to run this (depends on your os):
26 | ```bash
27 | pip install -r requirements.txt
28 | ```
29 | Also, as this is using Selenium to control the Chrome Browser,
30 | so you'll need to download its driver for your specific os from
31 | [here](https://sites.google.com/a/chromium.org/chromedriver/downloads).
32 |
33 | ### Documentation
34 |
35 | - A Books object (from Books.py) represents the books that are needed to be scraped.
36 | class Books() doesn't take any arguments
37 |
38 | Notes for the next two methods:
39 | browse could be one of the following:
40 | "shelf", "author", "lists" or "list" (by default)
41 | the keyword could be the id of a "shelf", an "author" or a "list"
42 | or it could be the search keyword in case you're searching for "lists"
43 |
44 | - Books.get_books(keyword, browse="list")
45 | Scrapes books ids an returns an array of them.
46 |
47 | - Books.output_books(keyword=None, browse="list", file_name="books")
48 | Scrapes books ids an writes them to a file set by
49 | sending file_name value without extension if none
50 | is sent, it'll write them to books.txt file by default
51 |
52 | - Books.append_books(books_ids)
53 | Append an external books ids array to class storage
54 | (Hint: it accepts what Books.get_books() returns)
55 |
56 | - A Reviews object (from Reviews.py) represents the scraped reviews.
57 | class Reviews(lang="ar")
58 | lang is the language of reviews to look for / scrape, it could be: ([ISO 639-1 codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes))
59 |
60 | af, ar, bg, bn, ca, cs, cy, da, de, el, en, es, et, fa, fi, fr, gu, he,
61 | hi, hr, hu, id, it, ja, kn, ko, lt, lv, mk, ml, mr, ne, nl, no, pa, pl,
62 | pt, ro, ru, sk, sl, so, sq, sv, sw, ta, te, th, tl, tr, uk, ur, vi, zh-cn, zh-tw
63 |
64 | - Reviews.output_book_reviews(book_id)
65 | Scrapes a book reviews and writes them to a file.
66 |
67 | - Reviews.output_books_reviews(self, books_ids, consider_previous=True)
68 | Scrapes reviews of an array of books and writes them to a file.
69 | consider_previous is set to whether to consider the books that have
70 | been already scraped or whether to delete them an start over
71 |
72 | - Reviews.wr.read_books(file_name="books")
73 | Reads books ids from file and returns them.
74 |
75 | - Tools module methods:
76 | - Manager.count_files_lines()
77 | Returns and prints the total sum of scraped books lines
78 |
79 | - Manager.delete_repeated_reviews()
80 | Returns unique reviews ids and delete all repeated ones and prints info
81 |
82 | - Manager.combine_reviews()
83 | Writes a single "reviews.txt" file containing all reviews
84 |
85 | - Manager.split_reviews(n)
86 | splits the combined "reviews.txt" file into n smaller files
87 |
88 | **Browser and Writer classes are only for implementation inside Books and Reviews Classes**
89 |
90 | ### Demo
91 |
92 | Import needed modules:
93 | ```python
94 | from Books import Books
95 | from Reviews import Reviews
96 | from Tools import *
97 | ```
98 |
99 | Scrape books ids from books shelved as "arabic":
100 | ```python
101 | b = Books()
102 | books_ids = b.get_books("arabic", "shelf")
103 | ```
104 |
105 | Scrape books reviews and write them to a file:
106 | ```python
107 | r = Reviews("ar")
108 | r.output_books_reviews(books_ids)
109 | ```
110 |
111 | Filter Reviews then combine them:
112 | ```python
113 | delete_repeated_reviews()
114 | combine_reviews()
115 | ```
116 |
117 | **A more comprehensive example can be found in Sample.py**
118 |
119 | ### Resources
120 |
121 | > - [Selenium](http://www.seleniumhq.org/)
122 | > - [LangDetect](https://github.com/Mimino666/langdetect)
123 | > - [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/)
124 |
125 | ### Reference
126 |
127 | Omar Einea
128 |
129 | Supervised by Dr. Ashraf Elnagar.
130 |
131 | University of Sharjah, United Arab Emarites, July 2016
132 |
133 | ### License
134 |
135 | Copyright (C) 2019 by Omar Einea.
136 |
137 | This is an open source tool licensed under GPL v3.0. Copy of the license can be found
138 | [here](https://github.com/OmarEinea/GoodReads/blob/master/LICENSE.md).
139 |
--------------------------------------------------------------------------------
/Reviews.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | from Tools import SafeThread
4 | from bs4 import BeautifulSoup
5 | from langdetect import detect
6 | from Browser import Browser
7 | from Writer import Writer
8 |
9 |
10 | # A class to Scrape books Reviews from GoodReads.com
11 | class Reviews:
12 | def __init__(self, path=None, lang="ar", edition_reviews=False):
13 | # Language of reviews to be scraped
14 | self._lang = lang
15 | # Instantiate browsing and writing managers
16 | self.wr = Writer(path) if path else Writer()
17 | self.br = Browser(edition_reviews)
18 | # Initialize an empty threads list
19 | self._threads = []
20 | # Counter for reviews from different languages
21 | self._invalid = None
22 |
23 | def start(self):
24 | self.br.start()
25 |
26 | # Scrape and write books' reviews to separate files
27 | def output_books_reviews(self, books_ids, consider_previous=True):
28 | if consider_previous:
29 | # Don't loop through already scraped books
30 | self.wr.consider_written_files(books_ids)
31 | # Show how many books are going to be scraped
32 | print(f"Scraping {len(books_ids)} Books")
33 | # Loop through book ids in array and scrape books
34 | for book_id in books_ids:
35 | self.output_book_reviews(book_id)
36 |
37 | # Scrape and write one book's reviews to a file
38 | def output_book_reviews(self, book_id):
39 | self._threads.clear()
40 | # Open book file and page by its Id
41 | self.br.open_book_page(book_id)
42 | self.wr.open_book_file(book_id)
43 | # Reset invalid reviews counter and page counter
44 | self._invalid = 0
45 | # Scrape book meta data in first line
46 | self.run(self._scrape_book_meta, [book_id])
47 | # Scrape first page of the book anyway
48 | self.run(self._scrape_book_reviews)
49 | no_next_page = False
50 | try: # Scrape the remaining pages
51 | while self._invalid < 60:
52 | # Go to next page if there's one
53 | in_next_page = self.br.goto_next_page()
54 | if no_next_page or not in_next_page:
55 | no_next_page = False
56 | # Switch to a different reviews mode
57 | if not self.br.switch_reviews_mode(book_id, in_next_page is None):
58 | # Break after switching to all modes
59 | break
60 | # Wait until requested book reviews are loaded
61 | if self.br.are_reviews_loaded():
62 | # Scrape loaded book reviews
63 | self.run(self._scrape_book_reviews)
64 | else: no_next_page = True
65 | finally:
66 | # Wait until all threads are done
67 | [thread.join() for thread in self._threads]
68 | # Finalize file name and close it
69 | self.wr.close_book_file()
70 |
71 | # Scrape and write book and author data
72 | def _scrape_book_meta(self, html, book_id):
73 | # Create soup object and store book meta section of the page in soup
74 | soup = BeautifulSoup(html, "lxml").find(id="metacol")
75 | # If book is not found
76 | if not soup:
77 | print(f"*Book ID:\t{book_id:<15}Not Found!")
78 | # Close file and raise an error
79 | self.wr.close_book_file()
80 | raise FileNotFoundError
81 | # Get book title and remove spaces from it
82 | title = soup.find(id="bookTitle").get_text(". ", strip=True)
83 | # Get average rating of the book out of five
84 | rating = soup.find(class_="average").get_text()
85 | # Store author data section
86 | author = soup.find(class_="authorName")
87 | # Get author id from url
88 | id_ = author.get("href")[38:].split(".")[0]
89 | # Get author name
90 | name = author.find().get_text()
91 | # Write scraped meta data to file's first line
92 | self.wr.write_book_meta(book_id, title, rating, id_, name)
93 | # Display book id and title
94 | print(f"*Book ID:\t{book_id:<15}Title:\t{title}")
95 |
96 | # Scrape a single page's reviews
97 | def _scrape_book_reviews(self, html):
98 | # Store reviews section of the page in soup
99 | soup = BeautifulSoup(html, "lxml").find(id="bookReviews")
100 | # Loop through reviews individually
101 | for review in soup.find_all(class_="review"):
102 | try: # Get user / reviewer id
103 | user_id = review.find(class_="user").get("href")[11:].split("-")[0]
104 | # Get rating out of five stars
105 | stars = len(review.find(class_="staticStars").find_all(class_="p10"))
106 | # Get full review text even the hidden parts, and remove spaces and newlines
107 | comment = review.find(class_="readable").find_all("span")[-1].get_text(". ", strip=True)
108 | # Detect which language the review is in
109 | if detect(comment) != self._lang:
110 | # Count it as a different language review
111 | self._invalid += 1
112 | continue
113 | # Get review date
114 | date = review.find(class_="reviewDate").get_text()
115 | # Skip the rest if one of the above is missing
116 | except Exception:
117 | # Count it as an invalid review
118 | self._invalid += 2
119 | continue
120 | # If it's not a strike, reset the counter
121 | self._invalid = 0
122 | # Get review ID
123 | review_id = review.get("id")[7:]
124 | # Write the scraped review to the file
125 | self.wr.write_review(review_id, user_id, date, stars, comment)
126 | # Add review id to ids
127 | print(f"Added ID:\t{review_id}")
128 | return True
129 |
130 | # Starts a scraping process on a new thread
131 | def run(self, method, args=[]):
132 | # Create a thread and add it to threads list then start it
133 | self._threads.append(SafeThread(target=method, args=[self.br.page_source] + args))
134 | self._threads[-1].start()
135 |
136 | def reset(self):
137 | self.stop()
138 | self.start()
139 | print("Restarted Reviews")
140 |
141 | def stop(self):
142 | self.br.close()
143 | self.wr.delete_file()
144 |
145 | def close(self):
146 | self.br.quit()
147 | self.wr.close()
148 | self._threads.clear()
149 | print("Closed Reviews")
150 |
--------------------------------------------------------------------------------
/Sample.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | from Books import Books
4 | from Reviews import Reviews
5 | from Tools import read_books
6 | from time import sleep
7 | # from Tools import *
8 |
9 | if __name__ == '__main__':
10 |
11 | # >>> Create books object and get all books shelved as "arabic"
12 | b = Books(arabic=False)
13 | # b.append_books(read_books())
14 | b.output_books(2522)
15 | while not b.output_books_editions(read_books(), file_name="editions"):
16 | print("Refreshing after a while")
17 | sleep(50)
18 | while not b.output_books_edition_by_language(read_books("editions"), file_name="ara_books"):
19 | print("Refreshing after a while")
20 | sleep(50)
21 | # >>> Create Reviews object and scrape all reviews from the books list
22 | r = Reviews(edition_reviews=True)
23 | while True:
24 | try:
25 | r.output_books_reviews(read_books("ara_books_list"))
26 | break
27 | # If an error occurs
28 | except Exception as e:
29 | print("ERROR:", str(e))
30 | r.reset()
31 | r.close()
32 | print("Done!")
33 | # delete_repeated_reviews()
34 | # combine_reviews()
35 | # split_reviews(5)
36 |
--------------------------------------------------------------------------------
/Tools.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | from threading import Thread
4 | import re, os
5 |
6 | id_from_url = re.compile(r"^.*/([0-9]+).*$")
7 | # Root path of reviews
8 | path = "./BooksReviews/"
9 |
10 |
11 | class SafeThread(Thread):
12 | def run(self):
13 | try:
14 | Thread.run(self)
15 | except AttributeError:
16 | Thread.join(self)
17 | raise AttributeError
18 |
19 |
20 | def read_books(file_name="books"):
21 | try: # Try reading the file
22 | with open(file_name + ".txt") as file:
23 | return file.read().splitlines()
24 | # If it's not there, return an empty list
25 | except FileNotFoundError:
26 | return []
27 |
28 |
29 | def write_books(books_ids, file_name="books"):
30 | with open(file_name + ".txt", "w") as file:
31 | file.write("\n".join(books_ids))
32 |
33 |
34 | def combine_files(output, *files, **options):
35 | ids = set()
36 | add_id = ids.add
37 | intersection = options.get("intersection")
38 | write = open(output, 'w+', encoding='utf-8').write
39 | for index, file in enumerate(files[::-1]):
40 | for line in open(file, encoding='utf-8').readlines():
41 | id_ = line.split('\t', 1)[0]
42 | if id_ not in ids:
43 | add_id(id_)
44 | if not intersection or index > 0:
45 | write(line)
46 |
47 |
48 | # Combine all scraped reviews in one file
49 | def combine_reviews(path=path):
50 | # Declare arrays and pointers to their add functions
51 | files, ids = [], set()
52 | append, add_id = files.append, ids.add
53 | # Loop through all files in path
54 | for file in os.listdir(path):
55 | # If file is complete
56 | if file[0] == 'C':
57 | # Read file lines and store them
58 | lines = open(path + file, encoding='utf-8').readlines()
59 | append((len(lines) - 1, lines))
60 | # Combine books titles in this file
61 | write_book = open(path + "books.csv", "w+", encoding='utf-8').write
62 | # Combine reviews in this file
63 | write_review = open(path + "reviews.csv", "w+", encoding='utf-8').write
64 | # Sort files from largest to smallest and loop through them
65 | for file in sorted(files, reverse=True):
66 | reviews = file[1]
67 | # Loop through all file lines
68 | for i in range(len(reviews)):
69 | reviews[i] = reviews[i].split('\t', 2)
70 | # If review is book's description (i.e. third cell is rating not date)
71 | if reviews[i][2][0].isdigit():
72 | # Store book description line index
73 | book_index = i
74 | # Copy book description line index and write it to books.csv
75 | book = reviews[book_index][:]
76 | write_book('\t'.join(book))
77 | # Split the rest of its cells
78 | book[2:] = book[2].split('\t')
79 | book_id, author_id = book[0], book[3]
80 | # Delete it from file lines (keeping reviews only)
81 | del reviews[book_index]
82 | # Loop through the reviews
83 | for review in reviews:
84 | # Make sure review id isn't repeated
85 | id_ = review[0]
86 | if id_ not in ids:
87 | # Add it to array and write it to file
88 | add_id(id_)
89 | write_review('\t'.join([id_, review[1], book_id, author_id, review[2].replace("\u2028", ". ")]))
90 |
91 |
92 | # Split the reviews from one file into n files
93 | def split_reviews(n):
94 | # Store lines from reviews file
95 | lines = open("reviews.csv", encoding="utf-8").readlines()
96 | # Make n number of steps in loop
97 | n = int(len(lines) / n)
98 | # Loop n times
99 | for i in range(0, len(lines), n):
100 | write = open("reviews" + str(int(i / n + 1)) + ".csv", "w+", encoding="utf-8").write
101 | # Loop through chunks of reviews file
102 | for line in lines[i:i + n]:
103 | write(line)
104 |
105 |
106 | # Counter for total lines written
107 | def count_files_lines(from_file=None):
108 | total = 0
109 | # If counting specific set of books
110 | if from_file:
111 | files = set()
112 | # Add specified lines in file to an array
113 | for file in open("./" + from_file + ".txt").readlines():
114 | files.add("C_" + file.strip('\n') + ".txt")
115 | else:
116 | # Otherwise, store all files in path to array
117 | files = os.listdir(path)
118 | # Loop through all files
119 | for file in files:
120 | # If file is complete
121 | if file[0] == 'C':
122 | # Open file and add numbers of lines in file to total
123 | total = len(open(path + file, 'r').readlines()) - 1
124 | # Display and return total count
125 | print("Total Count:\t" + str(total))
126 | return total
127 |
128 |
129 | def get_empty_files():
130 | write_empty = open("empty.txt", "w+").write
131 | for file in os.listdir(path):
132 | # Only count completed files
133 | if file[0] == 'E':
134 | write_empty(file[2:-4] + '\n')
135 |
136 |
137 | def compare_two_files(file1, file2):
138 | with open(file1, encoding='utf-8') as file1, open(file2, encoding='utf-8') as file2:
139 | reviews1 = file1.readlines()
140 | reviews2 = file2.readlines()
141 | reviews_ids = set(review.split('\t', 1)[0] for review in reviews1 + reviews2)
142 | print("Total Reviews:", len(reviews_ids))
143 | print("Repeated Reviews:", len(reviews1) + len(reviews2) - len(reviews_ids))
144 | print("Unique Reviews in First File:", len(reviews_ids) - len(reviews2))
145 | print("Unique Reviews in Second File:", len(reviews_ids) - len(reviews1))
146 |
147 |
148 | def get_digits(text):
149 | return int(''.join(char for char in text if char.isdigit()))
150 |
151 |
152 | def fix_invalid_tabs(file):
153 | lines = open(file, encoding="utf-8").readlines()
154 | write = open(file[:-4] + "_fixed" + file[-4:], "w+", encoding="utf-8").write
155 | count = 0
156 | for line in lines:
157 | parts = line.split('\t')
158 | if len(parts) >= 7:
159 | write('\t'.join(parts[:6] + [' '.join(parts[6:])]))
160 | if len(parts) > 7:
161 | count += 1
162 | print("Fixed", count, "reviews")
163 |
164 |
165 | def count_invalid(file):
166 | count = 0
167 | for line in open(file, encoding="utf-8").readlines():
168 | length = len(line.split('\t'))
169 | if length != 7:
170 | print(length)
171 | print("Number of invalid reviews:", count)
172 |
--------------------------------------------------------------------------------
/Writer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | from Tools import path as reviews_path
4 | import codecs, os
5 |
6 |
7 | # A class to Write Books and Reviews from GoodReads.com to files
8 | class Writer:
9 | def __init__(self, path=reviews_path):
10 | # File to write book reviews in
11 | self._file = None
12 | # Path to write files in
13 | self._path = path
14 | # Output file format
15 | self._format = ".txt"
16 | # Flag whether file is empty
17 | self._empty = True
18 |
19 | # Path setter
20 | def set_path(self, path):
21 | self._path = path
22 |
23 | # File format setter
24 | def set_format(self, format_):
25 | self._format = format_
26 |
27 | # Discard empty and complete files
28 | def consider_written_files(self, array):
29 | self._prepare_path()
30 | # Loop through files in the chosen path
31 | for file in os.listdir(self._path):
32 | # If file starts with C_ or E_
33 | if not file[0].isdigit():
34 | # Try to remove it from array
35 | while file[2:-4] in array:
36 | array.remove(file[2:-4])
37 | if os.path.exists("empty.txt"):
38 | for file in open("empty.txt", "r").readlines():
39 | if file in array:
40 | array.remove(file)
41 |
42 | # Create folder if it isn't already there
43 | def _prepare_path(self):
44 | if not os.path.exists(self._path):
45 | os.makedirs(self._path)
46 |
47 | # General shortcut to open a file for writing
48 | def open(self, name, key="w", path="./"):
49 | self._file = codecs.open(f"{path}{name}{self._format}", key, "utf-8")
50 |
51 | # Open file to write book reviews
52 | def open_book_file(self, name):
53 | self._prepare_path()
54 | # Delete file if it already exists
55 | file_name = f"{self._path}{name}{self._format}"
56 | if os.path.exists(file_name):
57 | os.remove(file_name)
58 | # Open the book reviews file with utf-8 encoding
59 | self.open(name, "a+", self._path)
60 | # Consider file empty at the beginning
61 | self._empty = True
62 |
63 | # General shortcut to write a string to file
64 | def write(self, string):
65 | self._file.write(string + "\n")
66 |
67 | # Write review to file
68 | def write_review(self, review_id, user_id, date, stars, comment):
69 | self.write("\t".join([review_id, user_id, date, str(stars), comment]))
70 | # File isn't empty anymore
71 | self._empty = False
72 |
73 | # Write book meta data to file
74 | def write_book_meta(self, book_id, title, rating, author_id, name):
75 | self.write("\t".join([str(book_id), title, rating, author_id, name]))
76 |
77 | # General shortcut to close the file
78 | def close(self):
79 | if self._file:
80 | self._file.close()
81 |
82 | # Flag file as empty or complete and close it
83 | def close_book_file(self):
84 | # Close file
85 | self.close()
86 | # If no reviews were added mark file as empty, otherwise mark it as complete
87 | name, i = self._file.name, len(self._path)
88 | os.replace(name, name[:i] + ("C_", "E_")[self._empty] + name[i:])
89 |
90 | def delete_file(self):
91 | self.close()
92 | os.remove(self._file.name)
93 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | beautifulsoup4==4.7.1
2 | langdetect==1.0.7
3 | selenium==3.141.0
4 | lxml==4.6.2
5 |
--------------------------------------------------------------------------------