6 |
7 | ### INSTALL
8 |
9 | * from pip
10 | ```
11 | $ pip install startpage-parser
12 | ```
13 | * from source
14 | ```
15 | $ git clone https://github.com/knassar702/startpage-parser
16 | $ cd startpage-parser
17 | $ pip install -e .
18 | ```
19 |
20 | ***
21 |
22 | ```python
23 | >>> from startpage import StartPage
24 | >>> task = StartPage()
25 | >>> task.search("Hello World",page=1) # page = number of pages (page=10 > from page number one to page number ten)
26 | >>> # All results stored in .results
27 | >>> # print(task.results)
28 | >>> # {'page number':"Results"}
29 | >>> print(task.results)
30 | {'1': [{'title': '"Hello, World!" program - Wikipedia', 'link': 'https://en.wikipedia.org/wiki/%22Hello,_World!%22_program', 'description': 'A "'}, {'title': 'Hello World (film) - Wikipedia', 'link': 'https://en.wikipedia.org/wiki/Hello_World_(film)', 'description': Hello World}, {'title': 'hello world - YouTube', 'link': 'https://www.youtube.com/watch?v=Yw6u6YkTgQ4', 'description': '30 Mar 2018 '}, {'title': 'Total immersion, Serious fun! with Hello-World!', 'link': 'https://www.hello-world.com/', 'description': 'Main index for '}, {'title': 'Hello, World! - Learn Python - Free Interactive Python Tutorial', 'link': 'https://www.learnpython.org/en/Hello,_World!', 'description': Hello}, {'title': 'C "Hello, World!" Program', 'link': 'https://www.programiz.com/c-programming/examples/print-sentence', 'description': 'In this example, you will learn to print "'}, {'title': 'C++ "Hello, World!" Program', 'link': 'https://www.programiz.com/cpp-programming/examples/print-sentence', 'description': 'In this example, we will learn to create a simple program named "'}, {'title': "The History of 'Hello, World' - HackerRank Blog", 'link': 'https://blog.hackerrank.com/the-history-of-hello-world/', 'description': '21 Apr 2015 '}, {'title': 'Hello World Studio', 'link': 'https://www.helloworldstudio.org/', 'description': Hello World}, {'title': 'Hello World - Go by Example', 'link': 'https://gobyexample.com/hello-world', 'description': 'To run the program, put the code in '}]}
31 |
32 | ```
33 |
34 | ### examples
35 |
36 | ```python
37 | from startpage import StartPage
38 |
39 | task = StartPage()
40 | task.search("Hello World",page=1)
41 | for page_num,results in task.results.items():
42 | print(f'PAGE: {page_num}\n---------------')
43 | for res in results:
44 | print(f'Title: {res["title"]}\n Description: {res["description"]}\n Link: {res["link"]}\n======')
45 | """
46 | PAGE: 1
47 | ---------------
48 | Title: "Hello, World!" program - Wikipedia
49 | Description: A "
50 | Link: https://en.wikipedia.org/wiki/%22Hello,_World!%22_program
51 | ======
52 | Title: Hello World (film) - Wikipedia
53 | Description: Hello World
54 | Link: https://en.wikipedia.org/wiki/Hello_World_(film)
55 | ======
56 | Title: hello world - YouTube
57 | Description: 30 Mar 2018
58 | Link: https://www.youtube.com/watch?v=Yw6u6YkTgQ4
59 | ======
60 | Title: Total immersion, Serious fun! with Hello-World!
61 | Description: Main index for
62 | Link: https://www.hello-world.com/
63 | ======
64 | Title: Hello, World! - Learn Python - Free Interactive Python Tutorial
65 | Description: Hello
66 | Link: https://www.learnpython.org/en/Hello,_World!
67 | ======
68 | """
69 | ```
70 |
71 | 
72 | 
73 |
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | from .src.startpage import StartPage
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | bs4
2 | requests
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 | with open('README.md') as readme_file:
4 | readme = readme_file.read()
5 |
6 | setup(
7 | name="startpage-parser",
8 | version='1.0.0',
9 | license="GPL-3.0 License",
10 | description='parsing search results from startpage search engine (based on google.com results)',
11 | py_modules=["startpage"],
12 | long_description=readme,
13 | package_dir={'':'src'},
14 | url='https://github.com/knassar702/startpage-parser',
15 | python_requires='>=3.6',
16 | install_requires=['bs4','requests']
17 | )
18 |
--------------------------------------------------------------------------------
/src/startpage.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # coded by : Khaled Nassar @knassar702
3 | # version : 1.0.0
4 | # https://github.com/knassar702/startpage-parser
5 |
6 | import requests
7 | from bs4 import BeautifulSoup
8 |
9 | class StartPage:
10 | def __init__(self):
11 | self.results = {} # {"Text":{"link":"http://google.com","Desctiopns":"blblalbal"}}
12 | def search(self,word,page=int(1)):
13 | page += 1
14 | for page_number in range(1,page): # number of search pages
15 | self.results[str(page_number)] = []
16 | """
17 | send request to startpage.com with search words and number of page
18 | """
19 | try:
20 | req = requests.get(f'https://www.startpage.com/do/search?q={word}&segment=startpage.brave&page={page_number}',headers={"User-agent":'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'}).content.decode()
21 | soup = BeautifulSoup(req, "html.parser")
22 | ancher = soup.find_all('div', {'class': "w-gl__result w-gl__desktop__result"}) # find div with "w-gl__result w-gl__desktop__result" class name
23 | """
24 |
25 | LINK
26 |
TEXT
27 |description
28 | """ 29 | for i in ancher: 30 | text = i.find('h3').contents[0] 31 | link = i.find('a',{'class':'w-gl__result-title result-link'})['href'] 32 | description = i.find('p',{'class':'w-gl__description'}).contents[0] 33 | self.results[str(page_number)].append({'title':text,'link':link,'description':description}) 34 | except Exception as e: 35 | print(e) 36 | """ 37 | s = StartPage() 38 | s.search("Hello World") 39 | print(s.results) 40 | """ 41 | return self.results 42 | --------------------------------------------------------------------------------