 Ed Luff 💻 |
253 |  Diretnan Domnan 🚇 ⚠️ 🔧 💻 |
254 |  MeNsaaH 🚇 ⚠️ 🔧 💻 |
255 |  Aditya Pal ⚠️ 💻 📖 |
256 |  Avinash Reddy 🐛 |
257 |  David Onuh 💻 ⚠️ |
258 |  Panagiotis Simakis 💻 ⚠️ |
259 |
 reiarthur 💻 |
262 |  Ashokkumar TA 💻 |
263 |  Andreas Teuber 💻 |
264 |  mi096684 🐛 |
265 |  devajithvs 💻 |
266 |  Geg Zakaryan 💻 🐛 |
267 |  Hakan Boğan 🐛 |
268 |
 NicKoehler 🐛 💻 |
271 |  ChrisLin 🐛 💻 |
272 |  Pietro 💻 🐛 |
273 |
30 | :type single_result: `bs4.element.ResultSet`
31 | :return: parsed title, link and description of single result
32 | :rtype: dict
33 | """
34 | rdict = SearchItem()
35 | h3_tag = single_result.find('h3')
36 | link_tag = h3_tag.find('a')
37 | if return_type in (ReturnType.FULL, return_type.TITLE):
38 | # Get the text and link
39 | rdict["titles"] = link_tag.text
40 |
41 | if return_type in (ReturnType.FULL, ReturnType.LINK):
42 | rdict["links"] = link_tag.get("href")
43 |
44 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
45 | caption = single_result.find('div', class_='compText aAbs')
46 | desc = caption.find('p', class_='lh-16')
47 | rdict["descriptions"] = desc.text
48 |
49 | return rdict
50 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/ask.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for ask search results
3 | """
4 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
5 |
6 |
7 | class Search(BaseSearch):
8 | """
9 | Searches Ask for string
10 | """
11 | name = "Ask"
12 |
13 | search_url = "https://www.ask.com/web?"
14 |
15 | summary = "\t Formerly known as Ask Jeeves, Ask.com receives approximately 0.42% of the search"\
16 | " share. ASK is based on a question/answer format where most questions are answered by "\
17 | "other users or are in the form of polls.\nIt also has the general search functionality "\
18 | "but the results returned lack quality compared to Google or even Bing and Yahoo."
19 |
20 | def get_params(self, query=None, page=None, offset=None, **kwargs):
21 | params = {}
22 | params["o"] = 0
23 | params["l"] = "dir"
24 | params["qo"] = "pagination"
25 | params["q"] = query
26 | params["qsrc"] = 998
27 | params["page"] = page
28 | return params
29 |
30 | def parse_soup(self, soup):
31 | """
32 | Parses Ask Search Soup for results
33 | """
34 | # find all class_='PartialSearchResults-item' => each result
35 | return soup.find_all('div', class_="PartialSearchResults-item")
36 |
37 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
38 | """
39 | Parses the source code to return
40 |
41 | :param single_result: single result found in
42 | :type single_result: `bs4.element.ResultSet`
43 | :return: parsed title, link and description of single result
44 | :rtype: str, str, str
45 | """
46 |
47 | rdict = SearchItem()
48 | if return_type in (ReturnType.FULL, return_type.TITLE):
49 | rdict["titles"] = single_result.find('a').text
50 |
51 | if return_type in (ReturnType.FULL, return_type.TITLE):
52 | rdict["links"] = single_result.a["href"]
53 |
54 | if return_type in (ReturnType.FULL, return_type.TITLE):
55 | rdict["descriptions"] = single_result.find(
56 | 'p', class_="PartialSearchResults-item-abstract").text
57 |
58 | return rdict
59 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/baidu.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for Baidu search results
3 | """
4 |
5 | import re
6 |
7 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
8 |
9 |
10 | class Search(BaseSearch):
11 | """
12 | Searches Baidu for string
13 | """
14 | name = "Baidu"
15 | search_url = "https://www.baidu.com/s?"
16 | summary = "\tBaidu, Inc. is a Chinese multinational technology company specializing in"\
17 | " Internet-related services and products and artificial intelligence (AI), headquartered"\
18 | " in Beijing's Haidian District.\n\tIt is one of the largest AI and internet"\
19 | " companies in the world.\n\tBaidu offers various services, including a"\
20 | " Chinese search engine, as well as a mapping service called Baidu Maps."
21 |
22 | """Override get_search_url"""
23 |
24 | def get_params(self, query=None, page=None, offset=None, **kwargs):
25 | params = {}
26 | params["wd"] = query
27 | params["pn"] = (page - 1) * 10
28 | params["oq"] = query
29 | return params
30 |
31 | def parse_soup(self, soup):
32 | """
33 | Parses Baidu for a search query
34 | """
35 |
36 | # Baidu search can be made deterministic via an id
37 | # Hence, a regex is used to match all eligible ids
38 |
39 | return soup.find_all('div', {'id': re.compile(r"^\d{1,2}")}, class_="c-container")
40 |
41 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
42 | """
43 | Parses the source code to return
44 |
45 | :param single_result: single result found in div with a numeric id
46 | :type single_result: `bs4.element.Tag`
47 | :return: parsed title, link and description of single result
48 | :rtype: dict
49 | """
50 | rdict = SearchItem()
51 | if return_type in (ReturnType.FULL, return_type.TITLE):
52 | h3_tag = single_result.find('h3')
53 |
54 | # sometimes h3 tag is not found
55 | if h3_tag:
56 | rdict["title"] = h3_tag.text
57 |
58 | if return_type in (ReturnType.FULL, ReturnType.LINK):
59 | link_tag = single_result.find('a')
60 | # Get the text and link
61 | rdict["links"] = link_tag.get('href')
62 |
63 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
64 | desc = single_result.find('div', class_='c-abstract')
65 | rdict["descriptions"] = desc if desc else ''
66 | return rdict
67 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/bing.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for Bing search results
3 | """
4 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
5 |
6 |
7 | class Search(BaseSearch):
8 | """
9 | Searches Bing for string
10 | """
11 | name = "Bing"
12 | search_url = "https://www.bing.com/search?"
13 | summary = "\tBing is Microsoft’s attempt to challenge Google in search, but despite their "\
14 | "efforts they still did not manage to convince users that their search engine can be"\
15 | " an alternative to Google.\n\tTheir search engine market share is constantly below "\
16 | "10%, even though Bing is the default search engine on Windows PCs."
17 |
18 | def get_params(self, query=None, page=None, offset=None, **kwargs):
19 | params = {}
20 | params["q"] = query
21 | params["offset"] = 0
22 | params["first"] = offset
23 | params["count"] = 10
24 | params["FORM"] = "PERE"
25 | return params
26 |
27 | def parse_soup(self, soup):
28 | """
29 | Parses Bing for a search query.
30 | """
31 | # find all li tags
32 | return soup.find_all('li', class_='b_algo')
33 |
34 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
35 | """
36 | Parses the source code to return
37 |
38 | :param single_result: single result found in
39 | :type single_result: `bs4.element.ResultSet`
40 | :return: parsed title, link and description of single result
41 | :rtype: dict
42 | """
43 | rdict = SearchItem()
44 | h2_tag = single_result.find('h2')
45 | link_tag = h2_tag.find('a')
46 |
47 | if return_type in (ReturnType.FULL, return_type.TITLE):
48 | rdict["titles"] = link_tag.text
49 |
50 | if return_type in (ReturnType.FULL, return_type.LINK):
51 | link = link_tag.get('href')
52 | rdict["links"] = link
53 |
54 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
55 | caption = single_result.find('div', class_='b_caption')
56 | desc = caption.find('p')
57 | rdict["descriptions"] = desc.text
58 |
59 | return rdict
60 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/coursera.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for coursera search results
3 | """
4 |
5 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
6 | from urllib.parse import urljoin
7 |
8 |
9 | class Search(BaseSearch):
10 | """
11 | Searches Coursera for string
12 | """
13 | name = "Coursera"
14 | search_url = "https://www.coursera.org/search?"
15 | summary = "\tCoursera is an American online learning platform founded by Stanford professors Andrew Ng and " \
16 | "Daphne Koller that offers massive open online courses, specializations, and degrees."
17 |
18 | def get_params(self, query=None, page=None, offset=None, **kwargs):
19 | params = {}
20 | params["query"] =query
21 | params["page"] = page
22 | return params
23 |
24 | def parse_soup(self, soup):
25 | """
26 | Parses Coursera Search Soup for results
27 | """
28 | # find all class_='gs_r gs_or gs_scl' => each result
29 | return soup.find_all('li', class_='ais-InfiniteHits-item')
30 |
31 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
32 | """
33 | Parses the source code to return
34 |
35 | :param single_result: single result found in
36 | :type single_result: `bs4.element.ResultSet`
37 | :return: parsed title, link, description, file link, result type of single result
38 | :rtype: dict
39 | """
40 | rdict = SearchItem()
41 |
42 | if return_type in (ReturnType.FULL, return_type.LINK):
43 | link = single_result.find('a', class_='rc-DesktopSearchCard anchor-wrapper').get('href')
44 |
45 | rdict["links"] = urljoin('https://www.coursera.org', link)
46 |
47 | if return_type in (ReturnType.FULL, return_type.TITLE):
48 | title = single_result.find('h2', class_="card-title").text
49 | rdict["titles"] = title
50 |
51 | if return_type in (ReturnType.FULL,):
52 | partner_elem = single_result.find('span', class_='partner-name')
53 | partner = ''
54 | if partner_elem:
55 | partner = partner_elem.text
56 |
57 | rating_avg_elem = single_result.find('span', class_='ratings-text')
58 | rating_avg = None
59 | if rating_avg_elem:
60 | rating_avg = float(rating_avg_elem.text)
61 |
62 | enrollment_elem = single_result.find('span', class_='enrollment-number')
63 | enrolment_number = None
64 |
65 | if enrollment_elem:
66 | enr_cl_txt = enrollment_elem.text.lower().replace(',', '').replace('.', '')\
67 | .replace('m', '0' * 6).replace('k', '0' * 3)
68 | if enr_cl_txt.isdigit():
69 | enrolment_number = int(enr_cl_txt)
70 |
71 | difficulty_elem = single_result.find('span', class_='difficulty')
72 | difficulty = ''
73 | if difficulty_elem:
74 | difficulty = difficulty_elem.text
75 |
76 | rating_count_elem = single_result.find('span', class_='ratings-count')
77 | rating_count = None
78 | if rating_count_elem:
79 | rating_count_elem = rating_count_elem.find('span')
80 | rating_count_cl = rating_count_elem.text.replace(',', '')
81 | if rating_count_cl.isdigit():
82 | rating_count = int(rating_count_cl)
83 |
84 | rdict.update({
85 | "partners": partner,
86 | "ratings_avg": rating_avg,
87 | "ratings_count": rating_count,
88 | "enrolments_numbers": enrolment_number,
89 | "difficulties": difficulty,
90 | })
91 | return rdict
92 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/duckduckgo.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for DuckDuckGo search results
3 | """
4 | import re
5 |
6 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
7 |
8 |
9 | class Search(BaseSearch):
10 | """
11 | Searches DuckDuckGo for string
12 | """
13 | name = "DuckDuckGo"
14 | base_url = "https://www.duckduckgo.com"
15 | search_url = "https://www.duckduckgo.com/html/?"
16 | summary = "\tHas a number of advantages over the other search engines. \n\tIt has a clean "\
17 | "interface, it does not track users, it is not fully loaded with ads and has a number "\
18 | "of very nice features (only one page of results, you can search directly other web "\
19 | "sites etc).\n\tAccording to DuckDuckGo traffic stats [December, 2018], they are "\
20 | "currently serving more than 30 million searches per day."
21 |
22 | def get_params(self, query=None, page=None, offset=None, **kwargs):
23 | params = {}
24 | params["q"] = query
25 | params["s"] = 0 if (page < 2) else (((page-1) * 50) - 20)
26 | params["dc"] = offset
27 | params["o"] = "json"
28 | params["api"] = "d.js"
29 | return params
30 |
31 | def parse_soup(self, soup):
32 | """
33 | Parses DuckDuckGo Search Soup for a query results
34 | """
35 | # find all div tags
36 | return soup.find_all('div', class_='result')
37 |
38 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
39 | """
40 | Parses the source code to return
41 |
42 | :param single_result: single result found in
43 | :type single_result: `bs4.element.ResultSet`
44 | :return: parsed title, link and description of single result
45 | :rtype: dict
46 | """
47 |
48 | rdict = SearchItem()
49 |
50 | if return_type in (ReturnType.FULL, return_type.TITLE):
51 | h2 = single_result.find(
52 | 'h2', class_="result__title") # pylint: disable=invalid-name
53 | # Get the text and link
54 | rdict["titles"] = h2.text.strip()
55 |
56 | if return_type in (ReturnType.FULL, ReturnType.LINK):
57 | link = None
58 | link_tag = single_result.find('a', class_="result__a")
59 | if link_tag is not None:
60 | rdict["links"] = link_tag.get('href')
61 | else:
62 | rdict['links'] = None
63 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
64 | desc = single_result.find(class_='result__snippet')
65 | if desc is not None:
66 | rdict["descriptions"] = desc.text
67 | else:
68 | rdict["descriptions"] = ""
69 | if rdict['links'] is None:
70 | rdict = None
71 |
72 | return rdict
73 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/github.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for GitHub search results
3 | """
4 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
5 | from search_engine_parser.core.exceptions import IncorrectKeyWord
6 |
7 |
8 | class Search(BaseSearch):
9 | """
10 | Searches GitHub for string
11 | """
12 | name = "GitHub"
13 | base_url = "https://github.com"
14 | search_url = base_url + "/search?"
15 | summary = "\tGitHub is an American company that provides hosting for software development "\
16 | "version control using Git. It is a subsidiary of Microsoft, which acquired the company "\
17 | "in 2018 for $7.5 billion.\n\tIt offers all of the distributed version control and source"\
18 | " code management (SCM) functionality of Git as well as adding its own features."\
19 | "\n\tAs of May 2019, GitHub reports having over 37 million users and more than 100 million"\
20 | " repositories (including at least 28 million public repositories), making it the largest "\
21 | "host of source code in the world."
22 |
23 | def get_params(self, query=None, page=None, offset=None, **kwargs):
24 | params = {}
25 | params["q"] = query
26 | params["p"] = page
27 | params["type"] = kwargs.get("type_", None)
28 | self.type = params["type"]
29 | return params
30 |
31 | def parse_soup(self, soup):
32 | """
33 | Parses GitHub for a search query.
34 | """
35 | allowed_types = (
36 | None,
37 | "Repositories",
38 | "Wikis",
39 | "Users",
40 | "Topics",
41 | "Marketplace",
42 | "RegistryPackages",
43 | "Issues",
44 | "Commits",
45 | "Code")
46 | if self.type not in allowed_types:
47 | raise IncorrectKeyWord(
48 | "No type <{type_}> exists".format(type_=self.type))
49 | # find all li tags
50 | if self.type in (None, "Repositories"):
51 | return soup.find_all('li', class_='repo-list-item')
52 | elif self.type == "RegistryPackages":
53 | return soup.find_all("div", class_='hx_hit-package')
54 | # find all user divs
55 | elif self.type == "Users":
56 | return soup.find_all('div', class_='user-list-item')
57 | elif self.type == "Wikis":
58 | return soup.find_all('div', class_='hx_hit-wiki')
59 | elif self.type == "Topics":
60 | return soup.find_all('div', class_='topic-list-item')
61 | elif self.type == "Issues":
62 | return soup.find_all('div', class_='issue-list-item')
63 | elif self.type == "Marketplace":
64 | return soup.find_all('div', class_='hx_hit-marketplace')
65 | elif self.type == "Commits":
66 | return soup.find_all('div', class_='commits-list-item')
67 |
68 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
69 | """
70 | Parses the source code to return
71 |
72 | :param single_result: single result found in container element
73 | :type single_result: `bs4.element.ResultSet`
74 | :return: parsed title, link and description of single result
75 | :rtype: dict
76 | """
77 | rdict = SearchItem()
78 | if self.type in (None, "Repositories"):
79 | h3 = single_result.find(
80 | 'div', class_='f4') # pylint: disable=invalid-name
81 | link_tag = h3.find('a')
82 | # Get the text and link
83 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
84 | title = link_tag.text
85 | rdict["titles"] = title
86 |
87 | if return_type in (ReturnType.FULL, ReturnType.LINK):
88 | ref_link = link_tag.get('href')
89 | link = self.base_url + ref_link
90 | rdict["links"] = link
91 |
92 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
93 | desc = single_result.find('p', class_="mb-1")
94 | rdict["descriptions"] = getattr(desc, 'text', '')
95 |
96 | if return_type in (ReturnType.FULL,):
97 | stars_and_lang_div = single_result.find(
98 | 'div', class_='d-flex')
99 | lang = stars_and_lang_div.find(
100 | 'span', itemprop="programmingLanguage")
101 | stars = single_result.find('div', class_='mr-3').find(
102 | 'a')
103 | updated_on = single_result.find("relative-time").get("title")
104 | rdict.update({
105 | "stars": "" if not stars else stars.text.strip(),
106 | "languages": lang.text if lang else "",
107 | "updated_on": updated_on,
108 | })
109 |
110 | if self.type == "Users":
111 | title_tag = single_result.find('div', class_='f4')
112 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
113 | title = title_tag.text
114 | rdict["titles"] = title
115 |
116 | if return_type in (ReturnType.FULL, ReturnType.LINK):
117 | ref_link = title_tag.find('a').get('href')
118 | link = self.base_url + ref_link
119 | rdict["links"] = link
120 |
121 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
122 | desc_tag = single_result.find('p', class_='mb-1')
123 | desc = None
124 | if desc_tag:
125 | desc = desc_tag.text.strip(' \n')
126 | rdict["descriptions"] = desc
127 |
128 | if return_type in (ReturnType.FULL, ):
129 | location_div = single_result.find('div', class_='d-flex')
130 | location_and_email = location_div.find_all(
131 | 'div', class_='mr-3')
132 | location = email = None
133 | for single in location_and_email:
134 | if single.get('href') == None:
135 | location = single.text.strip(' \n')
136 | else:
137 | email = single.text
138 |
139 | rdict.update({
140 | "locations": location,
141 | "emails": email,
142 | })
143 |
144 | if self.type == "Wikis":
145 | title_tag = single_result.find('a', class_=None)
146 |
147 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
148 | title = title_tag.get('title')
149 | rdict["title"] = title
150 |
151 | if return_type in (ReturnType.FULL, ReturnType.LINK):
152 | ref_link = title_tag.get('href')
153 | link = self.base_url + ref_link
154 | rdict["links"] = link
155 |
156 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
157 | desc = single_result.find('p', class_="mb1").text
158 | rdict["descriptions"] = desc
159 |
160 | if return_type in (ReturnType.FULL, ):
161 | last_updated = single_result.find(
162 | 'relative-time').get('title')
163 | repository = single_result.find('a', class_='muted-link').text
164 | rdict.update({
165 | "repositories": repository,
166 | "last_updated": last_updated,
167 | })
168 |
169 | if self.type == "Topics":
170 | title_div = single_result.find('div', class_='f4')
171 | title_tag = title_div.find('a', class_=None)
172 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
173 | rdict["titles"] = title_tag.text
174 | if return_type in (ReturnType.FULL, ReturnType.LINK):
175 | ref_link = title_tag.get('href')
176 | link = self.base_url + ref_link
177 | rdict["links"] = link
178 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
179 | desc = None
180 | desc_tag = single_result.find('p', class_=None)
181 | if desc_tag:
182 | desc = desc_tag.text
183 | rdict["descriptions"] = desc
184 |
185 | if self.type == "Marketplace":
186 | title_tag = single_result.find('a', class_='no-underline')
187 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
188 | title = title_tag.get('title')
189 | rdict["titles"] = title_tag.text
190 | if return_type in (ReturnType.FULL, ReturnType.LINK):
191 | link = title_tag.get('href')
192 | rdict["links"] = link
193 |
194 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
195 | desc = None
196 | desc_tag = single_result.find('text-gray')
197 | if desc_tag:
198 | desc = desc_tag.text
199 | rdict["descriptions"] = desc
200 |
201 | if return_type in (ReturnType.FULL, ):
202 | categories = list()
203 | categories_tags = single_result.find_all('a', class_='Label')
204 | if categories_tags:
205 | for i in categories_tags:
206 | categories.append(str(i).strip('\n '))
207 | rdict["categories"] = categories
208 |
209 | if self.type == "RegistryPackages":
210 | title_tag = single_result.find('a', class_='h4')
211 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
212 | title = title_tag.text
213 | rdict["titles"] = title_tag.text
214 |
215 | if return_type in (ReturnType.FULL, ReturnType.LINK):
216 | ref_link = title_tag.get('href')
217 | link = self.base_url + ref_link
218 | rdict["links"] = link
219 |
220 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
221 | desc = single_result.find(
222 | 'p', class_='mb-1').text.strip('\n ')
223 | rdict["descriptions"] = desc
224 |
225 | if self.type == "Issues":
226 | title_tag = single_result.find('a', class_=None)
227 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
228 | title = title_tag.text
229 | rdict["titles"] = title_tag.text
230 |
231 | if return_type in (ReturnType.FULL, ReturnType.LINK):
232 | ref_link = title_tag.get('href')
233 | link = self.base_url + ref_link
234 | rdict["links"] = link
235 |
236 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
237 | desc = single_result.find('p', class_='mb-0').text
238 | rdict["descriptions"] = desc
239 |
240 | if return_type in (ReturnType.FULL, ):
241 | repository = single_result.find(
242 | 'div', class_='ml-1').find('a', 'text-bold').text
243 | opened_by = self.base_url + \
244 | single_result.find(
245 | 'div', class_='mr-3').find('a').get('href')
246 | opened_on = single_result.find('relative-time').get("title")
247 | rdict.update({
248 | "opened_by": opened_by,
249 | "opened_on": opened_on,
250 | "respositories": repository,
251 | })
252 |
253 | if self.type == "Commits":
254 | title_p = single_result.find('div', class_="f4")
255 | title_tag = title_p.find('a')
256 |
257 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
258 | title = title_tag.get('aria-label').strip("\n ")
259 | rdict["titles"] = title_tag.text
260 |
261 | if return_type in (ReturnType.FULL, ReturnType.LINK):
262 | ref_link = title_tag.get('href')
263 | if ref_link.startswith("http"):
264 | link = ref_link
265 | else:
266 | link = self.base_url + ref_link
267 | rdict["links"] = link
268 |
269 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
270 | opened_on = None
271 | author = None
272 | if single_result.find('relative-time'):
273 | opened_on = single_result.find(
274 | 'relative-time').get("title")
275 | desc = None
276 | if single_result.find('a', class_='commit-author'):
277 | author_tag = single_result.find(
278 | 'a', class_='commit-author')
279 | author = author_tag.text
280 | div = single_result.find('div', class_='d-flex')
281 | repo = div.find('a').text
282 | desc = "Committed to {}".format(repo)
283 | rdict["descriptions"] = desc
284 | if return_type == ReturnType.FULL:
285 | rdict.update({
286 | "authors": author,
287 | "opened_on": opened_on,
288 | })
289 | return rdict
290 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/google.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for google search results
3 | """
4 | import sys
5 | from urllib.parse import (
6 | urljoin,
7 | parse_qs,
8 | unquote
9 | )
10 | import urllib.parse as urlparse
11 |
12 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
13 |
14 |
15 | EXTRA_PARAMS = ('hl', 'tbs')
16 |
17 |
18 | class Search(BaseSearch):
19 | """
20 | Searches Google for string
21 | """
22 | name = "Google"
23 | base_url = "https://www.google.com/"
24 | summary = "\tNo need for further introductions. The search engine giant holds the first "\
25 | "place in search with a stunning difference of 65% from second in place Bing.\n"\
26 | "\tAccording to the latest netmarketshare report (November 2018) 73% of searches "\
27 | "were powered by Google and only 7.91% by Bing.\n\tGoogle is also dominating the "\
28 | "mobile/tablet search engine market share with 81%!"
29 |
30 | def __init__(self):
31 | super().__init__()
32 | self.search_url = urljoin(self.base_url, "search")
33 |
34 | def get_params(self, query=None, offset=None, page=None, **kwargs):
35 | params = {}
36 | params["start"] = (page-1) * 10
37 | params["q"] = query
38 | params["gbv"] = 1
39 | # additional parameters will be considered
40 | for param in EXTRA_PARAMS:
41 | if kwargs.get(param):
42 | params[param] = kwargs[param]
43 | return params
44 |
45 | def parse_url(self, url):
46 | return self.clean_url(urljoin(self.base_url, url))
47 |
48 | def parse_soup(self, soup):
49 | """
50 | Parses Google Search Soup for results
51 | """
52 | # find all class_='g' => each result
53 | return soup.find_all('div', class_="Gx5Zad fP1Qef xpd EtOod pkphOe")
54 |
55 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
56 | """
57 | Parses the source code to return
58 |
59 | :param single_result: single result found in
60 | :type single_result: `bs4.element.ResultSet`
61 | :return: parsed title, link and description of single result
62 | :rtype: dict
63 | """
64 | # Some unneeded details shown such as suggestions should be ignore
65 | if (single_result.find("h2", class_="wITvVb") and single_result.find("div", class_="LKSyXe"))\
66 | or single_result.find("div", class_="X7NTVe"):
67 | return
68 |
69 | results = SearchItem()
70 | els = single_result.find_all('div', class_='kCrYT')
71 | if len(els) < 2:
72 | return
73 |
74 | # First div contains title and url
75 | r_elem = els[0]
76 |
77 | # Get the text and link
78 | if return_type in (ReturnType.FULL, ReturnType.TITLE):
79 | link_tag = r_elem.find('a')
80 | if link_tag:
81 | title = link_tag.find('h3').text
82 | else:
83 | r_elem = els[1]
84 | title = r_elem.find('div', class_='BNeawe').text
85 | results['titles'] = title
86 |
87 | if return_type in (ReturnType.FULL, ReturnType.LINK):
88 | link_tag = r_elem.find('a')
89 | if link_tag:
90 | raw_link = link_tag.get('href')
91 | raw_url = urljoin(self.base_url, raw_link)
92 | results['raw_urls'] = raw_url
93 | results['links'] = self.clean_url(raw_url)
94 |
95 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
96 | # Second Div contains Description
97 | desc_tag = els[1]
98 | if return_type in (ReturnType.FULL, ReturnType.LINK) and not results.get('links'):
99 | link_tag = desc_tag.find('a')
100 | if link_tag:
101 | desc_tag = els[0]
102 | raw_link = link_tag.get('href')
103 | raw_url = urljoin(self.base_url, raw_link)
104 | results['raw_urls'] = raw_url
105 | results['links'] = self.clean_url(raw_url)
106 | desc = desc_tag.text
107 | results['descriptions'] = desc
108 | return results
109 |
110 | def clean_url(self, url):
111 | """
112 | Extract clean URL from the SERP URL.
113 |
114 | >clean_url('https://www.google.com/url?q=https://english.stackexchange.com/questions/140710/what-is-the-opposite-of-preaching-to-the-choir&sa=U&ved=2ahUKEwi31MGyzvnuAhXyyDgGHXXACOYQFnoECAkQAg&usg=AOvVaw1GdXON-JIWGu-dGjHfgljl')
115 | https://english.stackexchange.com/questions/140710/what-is-the-opposite-of-preaching-to-the-choir
116 | """
117 | parsed = urlparse.urlparse(url)
118 | url_qs = parse_qs(parsed.query)
119 | if 'q' in url_qs:
120 | return unquote(url_qs['q'][0])
121 | elif 'url' in url_qs:
122 | return unquote(url_qs['url'][0])
123 | # Add more cases here.
124 | return url
125 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/googlenews.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for google news search results
3 | """
4 |
5 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
6 |
7 |
8 | class Search(BaseSearch):
9 | """
10 | Searches Google News for string
11 | """
12 | name = "GoogleNews"
13 | search_url = "https://www.google.com/search?"
14 | summary = "\tGoogle News is a news aggregator app developed by Google. It presents a "\
15 | "continuous, customizable flow of articles organized from thousands of publishers "\
16 | "and magazines. Google News is available as an app on Android, iOS, and the Web. "\
17 | "Google released a beta version in September 2002 and the official app in January 2006."
18 |
19 | def get_params(self, query=None, offset=None, page=None, **kwargs):
20 | params = {}
21 | params["num"] = 10
22 | params["start"] = page
23 | params["q"] = query
24 | params["client"] = "ubuntu"
25 | params["tbm"] = "nws"
26 | return params
27 |
28 | def parse_soup(self, soup):
29 | """
30 | Parses Google News Search Soup for results
31 | """
32 | # find all class_='g' => each result
33 | return soup.find_all('div', class_='g')
34 |
35 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
36 | """
37 | Parses the source code to return
38 |
39 | :param single_result: single result found in
40 | :type single_result: `bs4.element.ResultSet`
41 | :return: parsed title, link, description, imge link, news source, date of single result
42 | :rtype: dict
43 | """
44 | rdict = SearchItem()
45 |
46 | if return_type in (ReturnType.FULL, return_type.TITLE):
47 | title_tag = single_result.find('h3')
48 | title = title_tag.text
49 | rdict["titles"] = title
50 |
51 | if return_type in (ReturnType.FULL, ReturnType.LINK):
52 | link_tag = single_result.find('a')
53 | rdict["links"] = link_tag.get('href')
54 |
55 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
56 | desc_tag = single_result.find('div', class_='st')
57 | rdict["descriptions"] = desc_tag.text
58 |
59 | if return_type in (ReturnType.FULL,):
60 | img_tag = single_result.find('img', class_='th')
61 | news_source_tag = single_result.find('span', class_='e8fRJf')
62 | date_tag = single_result.find('span', class_='f')
63 |
64 | rdict["image_url"] = img_tag.get('src')
65 | rdict["news_source"] = news_source_tag.text
66 | rdict["date"] = date_tag.text
67 | return rdict
68 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/googlescholar.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for google scholar search results
3 | """
4 |
5 | import re
6 |
7 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
8 |
9 |
10 | class Search(BaseSearch):
11 | """
12 | Searches Google Scholar for string
13 | """
14 | name = "GoogleScholar"
15 | search_url = "https://scholar.google.gr/scholar?"
16 | summary = "\tGoogle Scholar is a freely accessible web search engine that indexes the full "\
17 | "text or metadata of scholarly literature across an array of publishing formats and "\
18 | "disciplines."
19 |
20 | def get_params(self, query=None, offset=None, page=None, **kwargs):
21 | params = {}
22 | params["hl"] = "en"
23 | params["start"] = page
24 | params["q"] = query
25 | return params
26 |
27 | def parse_soup(self, soup):
28 | """
29 | Parses Google Scholar Search Soup for results
30 | """
31 | # find all class_='gs_r gs_or gs_scl' => each result
32 | return soup.find_all('div', class_='gs_r gs_or gs_scl')
33 |
34 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
35 | """
36 | Parses the source code to return
37 |
38 | :param single_result: single result found in
39 | :type single_result: `bs4.element.ResultSet`
40 | :return: parsed title, link, description, file link, result type of single result
41 | :rtype: dict
42 | """
43 | rdict = SearchItem()
44 | r_elem = single_result.find('h3', class_='gs_rt')
45 | if return_type in (ReturnType.FULL, ReturnType.LINK):
46 | link_tag = r_elem.find('a')
47 | if link_tag:
48 | raw_link = link_tag.get('href')
49 | else:
50 | raw_link = ''
51 | rdict["links"] = raw_link
52 |
53 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
54 | desc = single_result.find('div', class_='gs_rs')
55 | if desc:
56 | desc = desc.text
57 | else:
58 | desc = ''
59 | rdict["descriptions"] = desc
60 |
61 | if return_type in (ReturnType.FULL, return_type.TITLE):
62 | title = r_elem.text
63 | title = re.sub(r'^[\[\w+\]]+ ', '', title)
64 | rdict["titles"] = title
65 |
66 | if return_type == ReturnType.FULL:
67 | t_elem = single_result.find('span', class_='gs_ct1')
68 | if t_elem:
69 | result_type = t_elem.text
70 | else:
71 | result_type = ''
72 |
73 | f_elem = single_result.find('div', class_='gs_or_ggsm')
74 | if f_elem:
75 | flink_tag = r_elem.find('a')
76 | if flink_tag:
77 | file_link = flink_tag.get('href')
78 | else:
79 | file_link = ''
80 | else:
81 | file_link = ''
82 |
83 | rdict.update({
84 | "result_types": result_type,
85 | "files_links": file_link
86 | })
87 |
88 | return rdict
89 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/myanimelist.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for MyAnimeList search results
3 | """
4 |
5 | import math
6 | import sys
7 |
8 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
9 |
10 |
11 | class Search(BaseSearch):
12 | """
13 | Searches MyAnimeList for string
14 | """
15 | name = "MyAnimeList"
16 |
17 | search_url = "https://myanimelist.net/anime.php?"
18 | summary = "\tMyAnimeList, often abbreviated as MAL, is an anime and manga social"\
19 | "networking and social cataloging application website."\
20 | "\n\tThe site provides its users with a list-like system to organize"\
21 | "and score anime and manga.\n\tIt facilitates finding users who share"\
22 | "similar tastes and provides a large database on anime and manga.\n\tThe"\
23 | "site claims to have 4.4 million anime and 775,000 manga entries."\
24 | "\n\tIn 2015, the site received over 120 million visitors a month."
25 |
26 | def get_params(self, query=None, page=None, offset=None, **kwargs):
27 | params = {}
28 | params["show"] = (math.ceil(page / 5) - 1) * 50
29 | params["q"] = query
30 | return params
31 |
32 | def parse_soup(self, soup):
33 | """
34 | Parses MyAnimeList for a search query
35 | """
36 |
37 | # The data is stored in table so find all table rows
38 | # The first row is table header
39 | res = soup.find('div', class_='js-categories-seasonal js-block-list list')
40 | if res:
41 | return res.find_all('tr')[1:]
42 |
43 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
44 | """
45 | Parses the source code to return
46 |
47 | :param single_result: single result found in div with a numeric id
48 | :type single_result: `bs4.element.Tag`
49 | :return: parsed title, link and description of single result
50 | :rtype: str, str, str
51 | """
52 | rdict = SearchItem()
53 | link_tag = single_result.find('a', class_='fw-b')
54 |
55 | if return_type in (ReturnType.FULL, return_type.TITLE):
56 | title = link_tag.find('strong').text
57 | rdict["titles"] = title
58 |
59 | if return_type in (ReturnType.FULL, ReturnType.LINK):
60 | rdict["links"] = link_tag.get('href')
61 |
62 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
63 | desc = single_result.find('div', class_='pt4').text.strip()
64 | rdict["descriptions"] = desc
65 |
66 | if return_type == ReturnType.FULL:
67 | data = list(single_result.find_all('td', class_='ac'))
68 | animetype = data[0].text.strip()
69 | episodes = data[1].text.strip()
70 | score = data[2].text.strip()
71 |
72 | rdict.update({
73 | "episode_count": episodes,
74 | "animetypes": animetype,
75 | "ratings": score
76 | })
77 | return rdict
78 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/stackoverflow.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for AOL search results
3 | """
4 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
5 |
6 |
7 | class Search(BaseSearch):
8 | """
9 | Searches StackOverflow for string
10 | """
11 | name = "StackOverflow"
12 | base_url = "https://stackoverflow.com"
13 | search_url = base_url + "/search?"
14 | summary = "\tStack Overflow is a question and answer site for professional and enthusiast "\
15 | "programmers.\n\tIt is a privately held website, the flagship site of the Stack "\
16 | "Exchange Network, created in 2008 by Jeff Atwood and Joel Spolsky.\n\tIt features "\
17 | "questions and answers on a wide range of topics in computer programming. It was "\
18 | "created to be a more open alternative to earlier question and answer sites "\
19 | "such as Experts-Exchange"
20 |
21 | def get_params(self, query=None, offset=None, page=None, **kwargs):
22 | params = {}
23 | params["page"] = page
24 | params["q"] = query
25 | params["pagesize"] = 15
26 | return params
27 |
28 | def parse_soup(self, soup):
29 | """
30 | Parses StackOverflow for a search query
31 | """
32 | # find all divs
33 | return soup.find_all('div', class_='summary')
34 |
35 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
36 | """
37 | Parses the source code to return
38 |
39 | :param single_result: single result found in
40 | :type single_result: `bs4.element.ResultSet`
41 | :return: parsed title, link and description of single result
42 | :rtype: dict
43 | """
44 | rdict = SearchItem()
45 | h3 = single_result.find('h3') # pylint: disable=invalid-name
46 | link_tag = h3.find('a')
47 | if return_type in (ReturnType.FULL, return_type.TITLE):
48 | # Get the text and link
49 | rdict["titles"] = link_tag.text
50 |
51 | if return_type in (ReturnType.FULL, return_type.LINK):
52 | ref_link = link_tag.get('href')
53 | link = self.base_url + ref_link
54 | rdict["links"] = link
55 |
56 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
57 | caption = single_result.find('div', class_='excerpt')
58 | rdict["descriptions"] = caption.text
59 | return rdict
60 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/yahoo.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for Yahoo search results
3 | """
4 | import re
5 |
6 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
7 |
8 |
9 | class Search(BaseSearch):
10 | """
11 | Searches Yahoo for string
12 | """
13 | name = "Yahoo"
14 | search_url = "https://search.yahoo.com/search?"
15 | summary = "\tYahoo is one the most popular email providers and holds the fourth place in "\
16 | "search with 3.90% market share.\n\tFrom October 2011 to October 2015, Yahoo search "\
17 | "was powered exclusively by Bing. \n\tSince October 2015 Yahoo agreed with Google to "\
18 | "provide search-related services and since then the results of Yahoo are powered both "\
19 | "by Google and Bing. \n\tYahoo is also the default search engine for Firefox browsers "\
20 | "in the United States (since 2014)."
21 |
22 | def get_params(self, query=None, page=None, offset=None, **kwargs):
23 | params = {}
24 | params["p"] = query
25 | params["b"] = offset
26 | return params
27 |
28 | def parse_soup(self, soup):
29 | """
30 | Parses Yahoo for a search query
31 | """
32 | # find all divs
33 | return soup.find_all('div', class_='Sr')
34 |
35 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
36 | """
37 | Parses the source code to return
38 |
39 | :param single_result: single result found in
40 | :type single_result: `bs4.element.ResultSet`
41 | :return: parsed title, link and description of single result
42 | :rtype: dict
43 | """
44 | rdict = SearchItem()
45 | h3_tag = single_result.find('h3', class_='title')
46 |
47 | if return_type in (ReturnType.FULL, return_type.TITLE):
48 | title = h3_tag.text
49 | rdict["titles"] = title
50 |
51 | if return_type in (ReturnType.FULL, ReturnType.LINK):
52 | link_tag = h3_tag.find('a')
53 | raw_link = link_tag.get('href')
54 | re_str = re.findall("/RU=(.+)/RK", raw_link)[0]
55 | re_str = re_str.replace("%3a", ":")
56 | link = re_str.replace("%2f", "/")
57 | rdict["links"] = link
58 |
59 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
60 | desc = single_result.find('span', class_='fc-falcon')
61 | rdict["descriptions"] = desc.text
62 |
63 | return rdict
64 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/yandex.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for Yandex search results
3 | """
4 |
5 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
6 |
7 |
8 | class Search(BaseSearch):
9 | """
10 | Searches Yandex for string
11 | """
12 | name = "Yandex"
13 | search_url = "https://yandex.com/search/?"
14 | summary = "\tYandex is the largest technology company in Russia and the"\
15 | " largest search engine on the internet in Russian"\
16 | ", with a market share of over 52%."\
17 | "\n\tThe Yandex.ru home page is the 4th most popular website in Russia."\
18 | "\n\tIt also has the largest market share of any search engine in the Commonwealth"\
19 | " of Independent States and is the 5th largest search engine worldwide"\
20 | " after Google, Baidu, Bing, and Yahoo!"
21 |
22 | def get_params(self, query=None, page=None, offset=None, **kwargs):
23 | params = {}
24 | params["text"] = query
25 | params["p"] = offset
26 | return params
27 |
28 | def parse_soup(self, soup):
29 | """
30 | Parses Yandex for a search query
31 | """
32 | return soup.find_all('li', class_="serp-item")
33 |
34 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
35 | """
36 | Parses the source code to return
37 |
38 | :param single_result: single result found in
39 | :type single_result: `bs4.element.ResultSet`
40 | :return: parsed title, link and description of single result
41 | :rtype: str, str, str
42 | """
43 | rdict = SearchItem()
44 | h3_tag = single_result.find('div', class_="organic__url-text")
45 |
46 | if return_type in (ReturnType.FULL, return_type.TITLE):
47 | # Get the text and link
48 | title = h3_tag.text
49 | # Handle read more type texts
50 | index = title.find("Read more")
51 | if index >= 0:
52 | title = title[0:int(index)]
53 | rdict["titles"] = title
54 |
55 | if return_type in (ReturnType.FULL, ReturnType.LINK):
56 | link_tag = single_result.find('a')
57 | link = link_tag.get('href')
58 | rdict["links"] = link
59 |
60 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
61 | desc = single_result.find('div', class_="organic__content-wrapper")
62 | desc = desc.text
63 | rdict["descriptions"] = desc
64 | return rdict
65 |
--------------------------------------------------------------------------------
/search_engine_parser/core/engines/youtube.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Parser for YouTube search results
3 | """
4 | from search_engine_parser.core.base import BaseSearch, ReturnType, SearchItem
5 |
6 |
7 | class Search(BaseSearch):
8 | """
9 | Searches YouTube for string
10 | """
11 | name = "YouTube"
12 | base_url = "https://youtube.com"
13 | search_url = base_url + "/results?"
14 | summary = "\tYouTube is an American video-sharing website headquartered in San Bruno, "\
15 | "California. Three former PayPal employees—Chad Hurley, Steve Chen, and Jawed "\
16 | "Karim—created the service in February 2005.\n\tGoogle bought the site in November "\
17 | "2006 for US$1.65 billion; YouTube now operates as one of Google's subsidiaries. "\
18 | "As of May 2019, more than 500 hours of video content are uploaded to YouTube every minute"
19 |
20 | def get_params(self, query=None, page=None, offset=None, **kwargs):
21 | params = {}
22 | params["search_query"] = query
23 | return params
24 |
25 | def parse_soup(self, soup):
26 | """
27 | Parses YouTube for a search query.
28 | """
29 | # find all ytd-video-renderer tags
30 | return soup.find_all('div', class_='yt-lockup-content')
31 |
32 | def parse_single_result(self, single_result, return_type=ReturnType.FULL, **kwargs):
33 | """
34 | Parses the source code to return
35 |
36 | :param single_result: single result found in
37 | :type single_result: `bs4.element.ResultSet`
38 | :return: parsed title, link and description of single result
39 | :rtype: dict
40 | """
41 | rdict = SearchItem()
42 | # pylint: disable=too-many-locals
43 | title_tag = single_result.find('a', class_='yt-uix-tile-link')
44 | channel_name = ""
45 |
46 | if return_type in (ReturnType.FULL, return_type.TITLE):
47 | # Get the text and link
48 | rdict["titles"] = title_tag.text
49 |
50 | # try for single videos
51 | try:
52 | if return_type in (ReturnType.FULL, ReturnType.LINK):
53 | ref_link = title_tag.get('href')
54 | link = self.base_url + ref_link
55 | rdict["links"] = link
56 |
57 | if return_type in (ReturnType.FULL, return_type.DESCRIPTION):
58 | desc = single_result.find(
59 | 'div', class_="yt-lockup-description").text
60 | rdict["descriptions"] = desc
61 |
62 | if return_type in (ReturnType.FULL, ):
63 | duration = single_result.find(
64 | 'span', class_='accessible-description').text
65 | ul_tag = single_result.find('ul', class_='yt-lockup-meta-info')
66 |
67 | channel_name = single_result.find(
68 | 'a', class_='yt-uix-sessionlink spf-link').text
69 | views_and_upload_date = ul_tag.find_all('li')
70 | upload_date = views_and_upload_date[0].text
71 | views = views_and_upload_date[1].text
72 | rdict.update({
73 | "channels": channel_name,
74 | "durations": duration,
75 | "views": views,
76 | "upload_dates": upload_date,
77 | })
78 | except BaseException: # pylint: disable=broad-except
79 | link_tags = single_result.find_all(
80 | 'a', class_='yt-uix-sessionlink spf-link')
81 | # TODO Optimize calls here so that we don't assign ref_link and channel_name
82 | # when we don't need them
83 | for i in link_tags:
84 | if i.get("href").startswith("/playlist"):
85 | ref_link = i.get("href")
86 | elif i.get("href").startswith("/user"):
87 | channel_name = i.text
88 | if return_type in (ReturnType.FULL, ReturnType.LINK):
89 | link = self.base_url + ref_link
90 | rdict["links"] = link
91 |
92 | if return_type in (ReturnType.FULL, ReturnType.DESCRIPTION):
93 | desc = single_result.find(
94 | 'span', class_='accessible-description').text
95 | rdict["descriptions"] = desc
96 | if return_type in (ReturnType.FULL,):
97 | rdict.update({
98 | "channels": channel_name,
99 | })
100 | return rdict
101 |
--------------------------------------------------------------------------------
/search_engine_parser/core/exceptions.py:
--------------------------------------------------------------------------------
1 | """@desc
2 | Exceptions
3 | """
4 |
5 |
6 | class NoResultsFound(Exception):
7 | pass
8 |
9 |
10 | class NoResultsOrTrafficError(Exception):
11 | """ When No results is returned or unusual traffic caused app to return empty results """
12 |
13 | class IncorrectKeyWord(Exception):
14 | """ When a wrong keyword argument is passed to the search function """
15 |
--------------------------------------------------------------------------------
/search_engine_parser/core/utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | import random
3 | import pickle
4 | import hashlib
5 | import aiohttp
6 | from fake_useragent import UserAgent
7 |
8 | FILEPATH = os.path.dirname(os.path.abspath(__file__))
9 |
10 | # prevent caching
11 | USER_AGENT_LIST = [
12 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0",
13 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
14 | "Chrome/72.0.3626.121 Safari/537.36",
15 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100 101 Firefox/22.0",
16 | "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0",
17 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.5 (KHTML, like Gecko) "
18 | "Chrome/19.0.1084.46 Safari/536.5",
19 | "Mozilla/5.0 (Windows; Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) "
20 | "Chrome/19.0.1084.46 Safari/536.5",
21 | ]
22 |
23 |
24 | def get_rand_user_agent():
25 | user_agent = random.choice(USER_AGENT_LIST)
26 | try:
27 | user_agent = UserAgent().random
28 | except:
29 | pass
30 | return user_agent
31 |
32 |
33 |
34 | class CacheHandler:
35 | def __init__(self):
36 | self.cache = os.path.join(FILEPATH, "cache")
37 | engine_path = os.path.join(FILEPATH, "engines")
38 | if not os.path.exists(self.cache):
39 | os.makedirs(self.cache)
40 | enginelist = os.listdir(engine_path)
41 | self.engine_cache = {i[:-3]: os.path.join(self.cache, i[:-3]) for i in enginelist if i not in
42 | ("__init__.py")}
43 | for cache in self.engine_cache.values():
44 | if not os.path.exists(cache):
45 | os.makedirs(cache)
46 |
47 | async def get_source(self, engine, url, headers, cache=True,
48 | proxy=None, proxy_auth=None):
49 | """
50 | Retrieves source code of webpage from internet or from cache
51 |
52 | :rtype: str, bool
53 | :param engine: engine of the engine saving
54 | :type engine: str
55 | :param url: URL to pull source code from
56 | :type url: str
57 | :param headers: request headers to make use of
58 | :type headers: dict
59 | :param cache: use cache or not
60 | :type cache: bool
61 | :param proxy: proxy address to make use off
62 | :type proxy: str
63 | :param proxy_auth: (user, password) tuple to authenticate proxy
64 | :type proxy_auth: (str, str)
65 | """
66 | encodedUrl = url.encode("utf-8")
67 | urlhash = hashlib.sha256(encodedUrl).hexdigest()
68 | engine = engine.lower()
69 | cache_path = os.path.join(self.engine_cache[engine], urlhash)
70 | if os.path.exists(cache_path) and cache:
71 | with open(cache_path, 'rb') as stream:
72 | return pickle.load(stream), True
73 | get_vars = { 'url':url, 'headers':headers }
74 | if proxy and proxy_auth:
75 | auth = aiohttp.BasicAuth(*proxy_auth)
76 | get_vars.update({'proxy':proxy, 'proxy_auth': auth})
77 |
78 | async with aiohttp.ClientSession() as session:
79 | async with session.get(**get_vars) as resp:
80 | html = await resp.text()
81 | with open(cache_path, 'wb') as stream:
82 | pickle.dump(str(html), stream)
83 | return str(html), False
84 |
85 | def clear(self, engine=None):
86 | """
87 | Clear the entire cache either by engine name
88 | or just all
89 |
90 | :param engine: engine to clear
91 | """
92 | if not engine:
93 | for engine_cache in self.engine_cache.values():
94 | for root, dirs, files in os.walk(engine_cache):
95 | for f in files:
96 | os.remove(os.path.join(engine_cache, f))
97 | else:
98 | engine_cache = self.engine_cache[engine.lower()]
99 | for _, _, files in os.walk(engine_cache):
100 | for f in files:
101 | os.remove(os.path.join(engine_cache, f))
102 |
--------------------------------------------------------------------------------
/search_engine_parser/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bisohns/search-engine-parser/0c2f4bde7dd21c10e64c9204417d9a228e96c187/search_engine_parser/tests/__init__.py
--------------------------------------------------------------------------------
/search_engine_parser/tests/test_base.py:
--------------------------------------------------------------------------------
1 | import os
2 | import unittest
3 | from importlib import import_module
4 | from urllib.parse import urlparse
5 | from unittest.mock import patch, MagicMock
6 | import vcr
7 | from parameterized import parameterized_class
8 |
9 | from search_engine_parser.core.exceptions import NoResultsOrTrafficError
10 |
11 | SEARCH_ARGS = ('Hello', 1)
12 |
13 |
14 | def get_engines():
15 | """ Returns a list of all engines for tests """
16 | engines = []
17 |
18 | base_dir = os.getcwd()
19 | engines_dir = os.path.join(base_dir, 'search_engine_parser', 'core', 'engines')
20 |
21 | for filename in os.listdir(engines_dir):
22 | if os.path.isfile(os.path.join(engines_dir, filename)) and filename.endswith('.py') \
23 | and filename != '__init__.py':
24 | engine = filename.split('.py')[0]
25 | module = import_module("search_engine_parser.core.engines.{}".format(engine.lower()))
26 | engine_class = getattr(module, "Search")
27 | engines.append([engine, engine_class(),])
28 | return engines
29 |
30 |
31 | def validate_url(url):
32 | """ Checks if a url is valid
33 | urls must contain scheme, netloc and path
34 | """
35 | try:
36 | result = urlparse(url)
37 | return all([result.scheme, result.netloc, result.path])
38 | except BaseException: # pylint: disable=broad-except
39 | print("URL: %s\n" % url)
40 | return False
41 |
42 |
43 | # pylint: disable=no-member
44 | class EngineBaseTest(unittest.TestCase):
45 | """ Testbase for Engines
46 |
47 | provides tests for engine methods
48 | """
49 |
50 | def setUp(self):
51 | from search_engine_parser.core.engines.google import Search # pylint: disable=import-outside-toplevel
52 | self.engine = Search()
53 |
54 | @patch('search_engine_parser.core.engines.google.Search.get_results')
55 | @patch('search_engine_parser.core.engines.google.Search.get_soup')
56 | async def test_urls(self, get_results_mock, get_soup_mock):
57 | """ Test that url updates work fine """
58 | await self.engine.search(query="hello", url="google.com.tr")
59 | first_url = self.engine._parsed_url.geturl()
60 | self.assertTrue(validate_url(first_url))
61 |
62 | self.engine.search(query="World", url="https://google.com.tr")
63 | second_url = self.engine._parsed_url.geturl()
64 | self.assertTrue(validate_url(second_url))
65 |
66 | self.assertNotEqual(second_url, first_url)
67 |
68 | # Test for https://github.com/bisoncorps/search-engine-parser/issues/92
69 | def test_two_queries_different_results(self):
70 | """ Test that url updates work fine """
71 | from search_engine_parser.core.engines.google import Search as GoogleSearch # pylint: disable=import-outside-toplevel
72 | from search_engine_parser.core.engines.yahoo import Search as YahooSearch # pylint: disable=import-outside-toplevel
73 | gengine = GoogleSearch()
74 | yahoo_engine = YahooSearch()
75 | gresults = None
76 | gresults = None
77 | with vcr.use_cassette('fixtures/google-test-diff-synopsis.yaml', record_mode='once'):
78 | gresults = gengine.search(query="What's up from this side")
79 | with vcr.use_cassette('fixtures/yahoo-test-diff-synopsis.yaml', record_mode='once'):
80 | yresults = yahoo_engine.search(query="this is example Bob")
81 | for key in gresults[0]:
82 | self.assertNotEqual(gresults[0].get(key, "GSearch"), yresults[0].get(key, "Ysearch"))
83 |
84 | self.assertNotEqual(gresults, yresults)
85 |
86 | # pylint: disable=no-member
87 | @parameterized_class(('name', 'engine'), get_engines())
88 | class TestScraping(unittest.TestCase):
89 | """ Testbase for Engines
90 |
91 | provides tests for titles, description and return urls
92 | """
93 | engine_class = None
94 |
95 | @classmethod
96 | def setUpClass(cls):
97 | super().setUpClass()
98 |
99 | try:
100 | cls.vcr_search(*SEARCH_ARGS)
101 | except NoResultsOrTrafficError:
102 | raise unittest.SkipTest(
103 | '{} failed due to traffic'.format(
104 | cls.engine))
105 |
106 | @classmethod
107 | def vcr_search(cls, *args, **kwargs):
108 | print(cls.name)
109 | with vcr.use_cassette('fixtures/{}-{}-synopsis.yaml'.format(cls.name, args[0].replace(" ", "-")), record="once"):
110 | cls.results = cls.engine.search(*args, **kwargs)
111 |
112 | @classmethod
113 | def test_cache_used(cls):
114 | """
115 | Test that the cache was used
116 | """
117 | try:
118 | cls.vcr_search(*SEARCH_ARGS, cache=True)
119 | if cls.engine._cache_hit == False:
120 | assert False, "{} cache - unexpected miss".format(
121 | cls.engine.name)
122 | except NoResultsOrTrafficError:
123 | raise unittest.SkipTest(
124 | '{} failed due to traffic'.format(
125 | cls.engine))
126 |
127 | @classmethod
128 | def test_cache_not_used(cls):
129 | """
130 | Test that the cache was used
131 | """
132 | try:
133 | cls.vcr_search(*SEARCH_ARGS, cache=False)
134 | if cls.engine._cache_hit == True:
135 | assert False, "{} cache - unexpected hit".format(
136 | cls.engine.name)
137 | except NoResultsOrTrafficError:
138 | raise unittest.SkipTest(
139 | '{} failed due to traffic'.format(
140 | cls.engine))
141 |
142 | @classmethod
143 | def test_cache_bypassed(cls):
144 | """
145 | Test that cache was bypassed
146 | """
147 | # wrongly set cls.engine._cache_hit
148 | cls.engine._cache_hit = True
149 | try:
150 | cls.vcr_search(*SEARCH_ARGS, cache=False)
151 | if cls.engine._cache_hit == True:
152 | assert False, "{} cache - not bypassed".format(
153 | cls.engine.name)
154 | except NoResultsOrTrafficError:
155 | raise unittest.SkipTest(
156 | '{} failed due to traffic'.format(
157 | cls.engine))
158 |
159 | def test_search_urls(self):
160 | """
161 | Test that the search urls generated are valid
162 | """
163 | self.assertTrue(validate_url(self.engine._parsed_url.geturl()))
164 |
165 | def test_returned_results(self):
166 | """
167 | Test that the returned results have valid data. 8 is just a chosen value as most search
168 | engines return values more than that
169 | """
170 | self.assertTrue(len(self.results['titles']) >= 4)
171 | self.assertTrue(len(self.results['links']) >= 4)
172 | # coursera does not return descriptions for
173 | # Preaching to the choir
174 | if not self.engine.name.lower() == "coursera":
175 | self.assertTrue(len(self.results['descriptions']) >= 4)
176 | else:
177 | self.assertTrue(len(self.results["difficulties"]) >= 4)
178 |
179 | def test_links(self):
180 | for link in self.results['links']:
181 | print("{}:::::{}".format(self.name, link))
182 | # Sometimes googlescholar returns empty links for citation type results
183 | if not link and self.name.lower() == "googlescholar":
184 | continue
185 | self.assertTrue(validate_url(link))
186 |
187 | def test_results_length_are_the_same(self):
188 | """ Tests if returned result items are equal.
189 | :param args: a list/tuple of other keys returned
190 | """
191 | # Different engines have different keys which may be returned or not returned
192 | # So if all keys are not the same length check that the titles and links length are
193 | # the same
194 | default_keys = ["titles", "links"]
195 | default_keys_set = set(map(lambda x: len(self.results[x]), default_keys))
196 |
197 | items = self.results.keys()
198 | items_set = set(map(lambda x: len(self.results[x]), items))
199 |
200 | self.assertTrue(len(items_set) == 1 or len(default_keys_set) == 1)
201 |
--------------------------------------------------------------------------------
/search_engine_parser/tests/test_cli.py:
--------------------------------------------------------------------------------
1 | import os
2 | import unittest
3 | from unittest.mock import patch, MagicMock
4 |
5 | from search_engine_parser.core import cli
6 |
7 | engine_class_mock = MagicMock()
8 | engine_class_mock.name = "Random Engine Name"
9 | engine_class_mock.clear_cache = MagicMock()
10 | engine_class_mock.search = MagicMock()
11 |
12 | class CliTests(unittest.TestCase):
13 |
14 | def setUp(self):
15 | self.parser = cli.create_parser()
16 |
17 | def test_show_summary(self):
18 | args = self.parser.parse_args(["-e", "google", "--show-summary"])
19 | # If it executes properly it should return None
20 | self.assertTrue(cli.main(args) is None)
21 |
22 | @patch('search_engine_parser.core.cli.get_engine_class', return_value=engine_class_mock)
23 | def test_query(self, engine_class):
24 | args = self.parser.parse_args(["-e", "google", "Preach"])
25 | # If it executes properly it should return None
26 | self.assertTrue(cli.main(args) is None)
27 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import re
2 | import setuptools
3 |
4 | REQUIRED_PYTHON = (3, 5)
5 |
6 | # Load requirements
7 | REQUIREMENTS = 'requirements/main.txt'
8 | CLI_REQUIREMENTS = 'requirements/cli.txt'
9 | REQUIREMENTS = [line.strip('\n') for line in open(REQUIREMENTS).readlines()]
10 | CLI_REQUIREMENTS = [line.strip('\n') for line in open(CLI_REQUIREMENTS).readlines()]
11 |
12 | with open("README.md", "r", encoding="utf8") as fh:
13 | LONG_DESCRIPTION = fh.read()
14 |
15 | # Trying to load version directly from `search-engine-parser` module attempts
16 | # to load __init__.py which will try to load other libraries not yet installed
17 | with open("search_engine_parser/__init__.py", "rt", encoding="utf8") as f:
18 | VERSION = re.search(r'__version__ = "(.*?)"', f.read(), re.M).group(1)
19 |
20 | setuptools.setup(
21 | name="search-engine-parser",
22 | version=VERSION,
23 | author='Domnan Diretnan, Mmadu Manasseh',
24 | author_email="diretnandomnan@gmail.com",
25 | description="scrapes search engine pages for query titles, descriptions and links",
26 | url="https://github.com/bisoncorps/search-engine-parser",
27 | project_urls={
28 | "Documentation":"https://search-engine-parser.readthedocs.io/en/latest",
29 | "Source": "https://github.com/bisoncorps/search-engine-parser",
30 | },
31 | packages=setuptools.find_packages(),
32 | install_requires=REQUIREMENTS,
33 | long_description=LONG_DESCRIPTION,
34 | long_description_content_type="text/markdown",
35 | license="MIT",
36 | keywords='\
37 | search-engine \
38 | search \
39 | parser \
40 | google \
41 | yahoo \
42 | bing \
43 | yandex \
44 | stackoverflow \
45 | github \
46 | baidu ',
47 | entry_points={'console_scripts': [
48 | 'pysearch=search_engine_parser.core.cli:runner'
49 | ]},
50 | classifiers=[
51 | "Programming Language :: Python :: 3",
52 | "License :: OSI Approved :: MIT License",
53 | "Operating System :: OS Independent",
54 | ],
55 | package_data={
56 | '': ['*.*'],
57 | 'requirements': ['*.*'],
58 | },
59 | include_package_data=True,
60 | extras_require={
61 | 'cli': CLI_REQUIREMENTS
62 | },
63 | python_requires='>={}.{}'.format(*REQUIRED_PYTHON),
64 | )
65 |
--------------------------------------------------------------------------------