├── onionsearch ├── __init__.py └── core.py ├── setup.py ├── .gitignore ├── README.md └── LICENSE /onionsearch/__init__.py: -------------------------------------------------------------------------------- 1 | from onionsearch.core import * 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | 4 | 5 | setup( 6 | name='onionsearch', 7 | version="1.3", 8 | packages=find_packages(), 9 | author="megadose", 10 | install_requires=["requests","argparse","termcolor","tqdm", "html5lib","bs4","PySocks"], 11 | description="OnionSearch is a script that scrapes urls on different .onion search engines.", 12 | include_package_data=True, 13 | url='http://github.com/megadose/OnionSearch', 14 | entry_points = {'console_scripts': ['onionsearch = onionsearch.core:scrape']}, 15 | classifiers=[ 16 | "Programming Language :: Python", 17 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 18 | ], 19 | ) 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OnionSearch 2 | 👋 Hi there! For any professional inquiries or collaborations, please reach out to me at: 3 | megadose@protonmail.com 4 | 5 | 📧 Preferably, use your professional email for correspondence. Let's keep it short and sweet, and all in English! 6 | 7 | ![PyPI](https://img.shields.io/pypi/v/onionsearch) ![PyPI - Week](https://img.shields.io/pypi/dw/onionsearch) ![PyPI - Downloads](https://static.pepy.tech/badge/onionsearch) ![PyPI - License](https://img.shields.io/pypi/l/onionsearch) 8 | #### For BTC Donations : 1FHDM49QfZX6pJmhjLE5tB2K6CaTLMZpXZ 9 | ## Educational purposes only 10 | 11 | OnionSearch is a Python3 script that scrapes urls on different ".onion" search engines. 12 | 13 | ![](https://files.catbox.moe/vguy1e.png) 14 | 15 | ### Demo 16 | 17 | ![](https://github.com/megadose/gif-demo/raw/master/onionsearch.gif) 18 | 19 | 20 | ## 💡 Prerequisite 21 | [Python 3](https://www.python.org/download/releases/3.0/) 22 | 23 | ## 📚 Currently supported Search engines 24 | - ahmia 25 | - darksearchio 26 | - onionland 27 | - notevil 28 | - darksearchenginer 29 | - phobos 30 | - onionsearchserver 31 | - torgle 32 | - onionsearchengine 33 | - tordex 34 | - tor66 35 | - tormax 36 | - haystack 37 | - multivac 38 | - evosearch 39 | - deeplink 40 | 41 | ## 🛠️ Installation 42 | ### With PyPI 43 | 44 | ```pip3 install onionsearch``` 45 | 46 | ### With Github 47 | 48 | ```bash 49 | git clone https://github.com/megadose/OnionSearch.git 50 | cd OnionSearch/ 51 | python3 setup.py install 52 | ``` 53 | 54 | 55 | ## 📈 Usage 56 | 57 | Help: 58 | ``` 59 | usage: onionsearch [-h] [--proxy PROXY] [--output OUTPUT] 60 | [--continuous_write CONTINUOUS_WRITE] [--limit LIMIT] 61 | [--engines [ENGINES [ENGINES ...]]] 62 | [--exclude [EXCLUDE [EXCLUDE ...]]] 63 | [--fields [FIELDS [FIELDS ...]]] 64 | [--field_delimiter FIELD_DELIMITER] [--mp_units MP_UNITS] 65 | search 66 | 67 | positional arguments: 68 | search The search string or phrase 69 | 70 | optional arguments: 71 | -h, --help show this help message and exit 72 | --proxy PROXY Set Tor proxy (default: 127.0.0.1:9050) 73 | --output OUTPUT Output File (default: output_$SEARCH_$DATE.txt), where $SEARCH is replaced by the first chars of the search string and $DATE is replaced by the datetime 74 | --continuous_write CONTINUOUS_WRITE 75 | Write progressively to output file (default: False) 76 | --limit LIMIT Set a max number of pages per engine to load 77 | --engines [ENGINES [ENGINES ...]] 78 | Engines to request (default: full list) 79 | --exclude [EXCLUDE [EXCLUDE ...]] 80 | Engines to exclude (default: none) 81 | --fields [FIELDS [FIELDS ...]] 82 | Fields to output to csv file (default: engine name link), available fields are shown below 83 | --field_delimiter FIELD_DELIMITER 84 | Delimiter for the CSV fields 85 | --mp_units MP_UNITS Number of processing units (default: core number minus 1) 86 | 87 | [...] 88 | ``` 89 | 90 | ### Multi-processing behaviour 91 | 92 | By default, the script will run with the parameter `mp_units = cpu_count() - 1`. It means if you have a machine with 4 cores, 93 | it will run 3 scraping functions in parallel. You can force `mp_units` to any value but it is recommended to leave to default. 94 | You may want to set it to 1 to run all requests sequentially (disabling multi-processing feature). 95 | 96 | Please note that continuous writing to csv file has not been *heavily* tested with multiprocessing feature and therefore 97 | may not work as expected. 98 | 99 | Please also note that the progress bars may not be properly displayed when `mp_units` is greater than 1. 100 | **It does not affect the results**, so don't worry. 101 | 102 | ### Examples 103 | 104 | To request all the engines for the word "computer": 105 | ``` 106 | onionsearch "computer" 107 | ``` 108 | 109 | To request all the engines excepted "Ahmia" and "Candle" for the word "computer": 110 | ``` 111 | onionsearch "computer" --exclude ahmia candle 112 | ``` 113 | 114 | To request only "Tor66", "DeepLink" and "Phobos" for the word "computer": 115 | ``` 116 | onionsearch "computer" --engines tor66 deeplink phobos 117 | ``` 118 | 119 | The same as previously but limiting to 3 the number of pages to load per engine: 120 | ``` 121 | onionsearch "computer" --engines tor66 deeplink phobos --limit 3 122 | ``` 123 | 124 | Please kindly note that the list of supported engines (and their keys) is given in the script help (-h). 125 | 126 | 127 | ### Output 128 | 129 | #### Default output 130 | 131 | By default, the file is written at the end of the process. The file will be csv formatted, containing the following columns: 132 | ``` 133 | "engine","name of the link","url" 134 | ``` 135 | 136 | #### Customizing the output fields 137 | 138 | You can customize what will be flush in the output file by using the parameters `--fields` and `--field_delimiter`. 139 | 140 | `--fields` allows you to add, remove, re-order the output fields. The default mode is show just below. Instead, you can for instance 141 | choose to output: 142 | ``` 143 | "engine","name of the link","url","domain" 144 | ``` 145 | by setting `--fields engine name link domain`. 146 | 147 | Or even, you can choose to output: 148 | ``` 149 | "engine","domain" 150 | ``` 151 | by setting `--fields engine domain`. 152 | 153 | These are examples but there are many possibilities. 154 | 155 | Finally, you can also choose to modify the CSV delimiter (comma by default), for instance: `--field_delimiter ";"`. 156 | 157 | #### Changing filename 158 | 159 | The filename will be set by default to `output_$DATE_$SEARCH.txt`, where $DATE represents the current datetime and $SEARCH the first 160 | characters of the search string. 161 | 162 | You can modify this filename by using `--output` when running the script, for instance: 163 | ``` 164 | onionsearch "computer" --output "\$DATE.csv" 165 | onionsearch "computer" --output output.txt 166 | onionsearch "computer" --output "\$DATE_\$SEARCH.csv" 167 | ... 168 | ``` 169 | (Note that it might be necessary to escape the dollar character.) 170 | 171 | In the csv file produced, the name and url strings are sanitized as much as possible, but there might still be some problems... 172 | 173 | #### Write progressively 174 | 175 | You can choose to progressively write to the output (instead of everything at the end, which would prevent 176 | losing the results if something goes wrong). To do so you have to use `--continuous_write True`, just as is: 177 | ``` 178 | onionsearch "computer" --continuous_write True 179 | ``` 180 | You can then use the `tail -f` (tail follow) Unix command to actively watch or monitor the results of the scraping. 181 | ## Thank you to [Gobarigo](https://github.com/Gobarigo) 182 | ## Thank you [mxrch](https://github.com/mxrch) for this logo 183 | 184 | ## 📝 License 185 | [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.fr.html) 186 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /onionsearch/core.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import csv 5 | import math 6 | import re 7 | import time 8 | from datetime import datetime 9 | from functools import reduce 10 | from random import choice 11 | from multiprocessing import Pool, cpu_count, current_process, freeze_support 12 | from tqdm import tqdm 13 | 14 | import requests 15 | import urllib.parse as urlparse 16 | from urllib.parse import parse_qs 17 | from urllib.parse import quote 18 | from urllib.parse import unquote 19 | from bs4 import BeautifulSoup 20 | from urllib3.exceptions import ProtocolError 21 | 22 | ENGINES = { 23 | "ahmia": "http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion", # Offline? 24 | "darksearchio": "http://darksearch.io", 25 | "onionland": "http://3bbad7fauom4d6sgppalyqddsqbf5u5p56b5k5uk2zxsy3d6ey2jobad.onion", 26 | "notevil": "http://hss3uro2hsxfogfq.onion", # Offline? 27 | "darksearchenginer": "http://l4rsciqnpzdndt2llgjx3luvnxip7vbyj6k6nmdy4xs77tx6gkd24ead.onion", 28 | "phobos": "http://phobosxilamwcg75xt22id7aywkzol6q6rfl2flipcqoc4e4ahima5id.onion", 29 | "onionsearchserver": "http://3fzh7yuupdfyjhwt3ugzqqof6ulbcl27ecev33knxe3u7goi3vfn2qqd.onion", 30 | "torgle": "http://no6m4wzdexe3auiupv2zwif7rm6qwxcyhslkcnzisxgeiw6pvjsgafad.onion", # "torgle" -> "Submarine" 31 | "torgle1": "http://torgle5fj664v7pf.onion", # Offline? 32 | "onionsearchengine": "http://onionf4j3fwqpeo5.onion", # Offline? 33 | "tordex": "http://tordex7iie7z2wcg.onion", # Offline? 34 | "tor66": "http://tor66sewebgixwhcqfnp5inzp5x5uohhdy3kvtnyfxc2e5mxiuh34iid.onion", 35 | "tormax": "http://tormaxunodsbvtgo.onion", # Offline? 36 | "haystack": "http://haystak5njsmn2hqkewecpaxetahtwhsbsa64jom2k22z5afxhnpxfid.onion", 37 | "multivac": "http://multivacigqzqqon.onion", # Offline? 38 | "evosearch": "http://evo7no6twwwrm63c.onion", # Offline? 39 | "deeplink": "http://deeplinkdeatbml7.onion", # Offline? 40 | } 41 | 42 | desktop_agents = [ 43 | 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0', # Tor Browser for Windows and Linux 44 | 'Mozilla/5.0 (Android 10; Mobile; rv:91.0) Gecko/91.0 Firefox/91.0', # Tor Browser for Android 45 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36', 46 | 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36', 47 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 48 | 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36', 49 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) ' 50 | 'AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14', 51 | 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', 52 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) ' 53 | 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', 54 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) ' 55 | 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36', 56 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36', 57 | 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36', 58 | 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0' 59 | ] 60 | 61 | supported_engines = ENGINES 62 | 63 | available_csv_fields = [ 64 | "engine", 65 | "name", 66 | "link", 67 | "domain" 68 | ] 69 | 70 | 71 | def print_epilog(): 72 | epilog = "Available CSV fields: \n\t" 73 | for f in available_csv_fields: 74 | epilog += " {}".format(f) 75 | epilog += "\n" 76 | epilog += "Supported engines: \n\t" 77 | for e in supported_engines.keys(): 78 | epilog += " {}".format(e) 79 | return epilog 80 | 81 | 82 | parser = argparse.ArgumentParser(epilog=print_epilog(), formatter_class=argparse.RawTextHelpFormatter) 83 | parser.add_argument("--proxy", default='localhost:9050', type=str, help="Set Tor proxy (default: 127.0.0.1:9050)") 84 | parser.add_argument("--output", default='output_$SEARCH_$DATE.txt', type=str, 85 | help="Output File (default: output_$SEARCH_$DATE.txt), where $SEARCH is replaced by the first " 86 | "chars of the search string and $DATE is replaced by the datetime") 87 | parser.add_argument("--continuous_write", type=bool, default=False, 88 | help="Write progressively to output file (default: False)") 89 | parser.add_argument("search", type=str, help="The search string or phrase") 90 | parser.add_argument("--limit", type=int, default=0, help="Set a max number of pages per engine to load") 91 | parser.add_argument("--engines", type=str, action='append', help='Engines to request (default: full list)', nargs="*") 92 | parser.add_argument("--exclude", type=str, action='append', help='Engines to exclude (default: none)', nargs="*") 93 | parser.add_argument("--fields", type=str, action='append', 94 | help='Fields to output to csv file (default: engine name link), available fields are shown below', 95 | nargs="*") 96 | parser.add_argument("--field_delimiter", type=str, default=",", help='Delimiter for the CSV fields') 97 | parser.add_argument("--mp_units", type=int, default=(cpu_count() - 1), help="Number of processing units (default: " 98 | "core number minus 1)") 99 | 100 | args = parser.parse_args() 101 | proxies = {'http': 'socks5h://{}'.format(args.proxy), 'https': 'socks5h://{}'.format(args.proxy)} 102 | filename = args.output 103 | field_delim = "," 104 | if args.field_delimiter and len(args.field_delimiter) == 1: 105 | field_delim = args.field_delimiter 106 | 107 | 108 | def random_headers(): 109 | return {'User-Agent': choice(desktop_agents), 110 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'} 111 | 112 | 113 | def clear(toclear): 114 | str = toclear.replace("\n", " ") 115 | str = ' '.join(str.split()) 116 | return str 117 | 118 | 119 | def get_parameter(url, parameter_name): 120 | parsed = urlparse.urlparse(url) 121 | return parse_qs(parsed.query)[parameter_name][0] 122 | 123 | 124 | def get_proc_pos(): 125 | return (current_process()._identity[0]) - 1 126 | 127 | 128 | def get_tqdm_desc(e_name, pos): 129 | return "%20s (#%d)" % (e_name, pos) 130 | 131 | 132 | def ahmia(searchstr): 133 | results = [] 134 | ahmia_url = supported_engines['ahmia'] + "/search/?q={}" 135 | 136 | pos = get_proc_pos() 137 | with tqdm(total=1, initial=0, desc=get_tqdm_desc("Ahmia", pos), position=pos) as progress_bar: 138 | response = requests.get(ahmia_url.format(quote(searchstr)), proxies=proxies, headers=random_headers()) 139 | soup = BeautifulSoup(response.text, 'html5lib') 140 | results = link_finder("ahmia", soup) 141 | progress_bar.update() 142 | 143 | return results 144 | 145 | 146 | 147 | 148 | def darksearchio(searchstr): 149 | results = [] 150 | darksearchio_url = supported_engines['darksearchio'] + "/api/search?query={}&page={}" 151 | max_nb_page = 30 152 | if args.limit != 0: 153 | max_nb_page = args.limit 154 | 155 | with requests.Session() as s: 156 | s.proxies = proxies 157 | s.headers = random_headers() 158 | resp = s.get(darksearchio_url.format(quote(searchstr), 1)) 159 | 160 | page_number = 1 161 | if resp.status_code == 200: 162 | resp = resp.json() 163 | if 'last_page' in resp: 164 | page_number = resp['last_page'] 165 | if page_number > max_nb_page: 166 | page_number = max_nb_page 167 | else: 168 | return 169 | 170 | pos = get_proc_pos() 171 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("DarkSearch (.io)", pos), position=pos) \ 172 | as progress_bar: 173 | 174 | results = link_finder("darksearchio", resp['data']) 175 | progress_bar.update() 176 | 177 | for n in range(2, page_number + 1): 178 | resp = s.get(darksearchio_url.format(quote(searchstr), n)) 179 | if resp.status_code == 200: 180 | resp = resp.json() 181 | results = results + link_finder("darksearchio", resp['data']) 182 | progress_bar.update() 183 | else: 184 | # Current page results will be lost but we will try to continue after a short sleep 185 | time.sleep(1) 186 | 187 | return results 188 | 189 | 190 | def onionland(searchstr): 191 | results = [] 192 | onionlandv3_url = supported_engines['onionland'] + "/search?q={}&page={}" 193 | max_nb_page = 100 194 | if args.limit != 0: 195 | max_nb_page = args.limit 196 | 197 | with requests.Session() as s: 198 | s.proxies = proxies 199 | s.headers = random_headers() 200 | 201 | resp = s.get(onionlandv3_url.format(quote(searchstr), 1)) 202 | soup = BeautifulSoup(resp.text, 'html5lib') 203 | 204 | page_number = 1 205 | for i in soup.find_all('div', attrs={"class": "search-status"}): 206 | approx_re = re.match(r"About ([,0-9]+) result(.*)", 207 | clear(i.find('div', attrs={'class': "col-sm-12"}).get_text())) 208 | if approx_re is not None: 209 | nb_res = int((approx_re.group(1)).replace(",", "")) 210 | results_per_page = 19 211 | page_number = math.ceil(nb_res / results_per_page) 212 | if page_number > max_nb_page: 213 | page_number = max_nb_page 214 | 215 | pos = get_proc_pos() 216 | with tqdm(total=max_nb_page, initial=0, desc=get_tqdm_desc("OnionLand", pos), position=pos) as progress_bar: 217 | 218 | results = link_finder("onionland", soup) 219 | progress_bar.update() 220 | 221 | for n in range(2, page_number + 1): 222 | resp = s.get(onionlandv3_url.format(quote(searchstr), n)) 223 | soup = BeautifulSoup(resp.text, 'html5lib') 224 | ret = link_finder("onionland", soup) 225 | if len(ret) == 0: 226 | break 227 | results = results + ret 228 | progress_bar.update() 229 | 230 | return results 231 | 232 | 233 | def notevil(searchstr): 234 | results = [] 235 | notevil_url1 = supported_engines['notevil'] + "/index.php?q={}" 236 | notevil_url2 = supported_engines['notevil'] + "/index.php?q={}&hostLimit=20&start={}&numRows={}&template=0" 237 | max_nb_page = 20 238 | if args.limit != 0: 239 | max_nb_page = args.limit 240 | 241 | # Do not use requests.Session() here (by experience less results would be got) 242 | req = requests.get(notevil_url1.format(quote(searchstr)), proxies=proxies, headers=random_headers()) 243 | soup = BeautifulSoup(req.text, 'html5lib') 244 | 245 | page_number = 1 246 | last_div = soup.find("div", attrs={"style": "text-align:center"}).find("div", attrs={"style": "text-align:center"}) 247 | if last_div is not None: 248 | for i in last_div.find_all("a"): 249 | page_number = int(i.get_text()) 250 | if page_number > max_nb_page: 251 | page_number = max_nb_page 252 | 253 | pos = get_proc_pos() 254 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("not Evil", pos), position=pos) as progress_bar: 255 | num_rows = 20 256 | results = link_finder("notevil", soup) 257 | progress_bar.update() 258 | 259 | for n in range(2, page_number + 1): 260 | start = (int(n - 1) * num_rows) 261 | req = requests.get(notevil_url2.format(quote(searchstr), start, num_rows), 262 | proxies=proxies, 263 | headers=random_headers()) 264 | soup = BeautifulSoup(req.text, 'html5lib') 265 | results = results + link_finder("notevil", soup) 266 | progress_bar.update() 267 | time.sleep(1) 268 | 269 | return results 270 | 271 | 272 | 273 | 274 | def darksearchenginer(searchstr): 275 | results = [] 276 | darksearchenginer_url = supported_engines['darksearchenginer'] 277 | max_nb_page = 20 278 | if args.limit != 0: 279 | max_nb_page = args.limit 280 | page_number = 1 281 | 282 | with requests.Session() as s: 283 | s.proxies = proxies 284 | s.headers = random_headers() 285 | 286 | # Note that this search engine is very likely to timeout 287 | resp = s.post(darksearchenginer_url, data={"search[keyword]": searchstr, "page": page_number}) 288 | soup = BeautifulSoup(resp.text, 'html5lib') 289 | 290 | pages_input = soup.find_all("input", attrs={"name": "page"}) 291 | for i in pages_input: 292 | page_number = int(i['value']) 293 | if page_number > max_nb_page: 294 | page_number = max_nb_page 295 | 296 | pos = get_proc_pos() 297 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Dark Search Enginer", pos), position=pos) \ 298 | as progress_bar: 299 | 300 | results = link_finder("darksearchenginer", soup) 301 | progress_bar.update() 302 | 303 | for n in range(2, page_number + 1): 304 | resp = s.post(darksearchenginer_url, data={"search[keyword]": searchstr, "page": str(n)}) 305 | soup = BeautifulSoup(resp.text, 'html5lib') 306 | results = results + link_finder("darksearchenginer", soup) 307 | progress_bar.update() 308 | 309 | return results 310 | 311 | 312 | def phobos(searchstr): 313 | results = [] 314 | phobos_url = supported_engines['phobos'] + "/search?query={}&p={}" 315 | max_nb_page = 100 316 | if args.limit != 0: 317 | max_nb_page = args.limit 318 | 319 | with requests.Session() as s: 320 | s.proxies = proxies 321 | s.headers = random_headers() 322 | 323 | resp = s.get(phobos_url.format(quote(searchstr), 1), proxies=proxies, headers=random_headers()) 324 | soup = BeautifulSoup(resp.text, 'html5lib') 325 | 326 | page_number = 1 327 | pages = soup.find("div", attrs={"class": "pages"}).find_all('a') 328 | if pages is not None: 329 | for i in pages: 330 | page_number = int(i.get_text()) 331 | if page_number > max_nb_page: 332 | page_number = max_nb_page 333 | 334 | pos = get_proc_pos() 335 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Phobos", pos), position=pos) as progress_bar: 336 | results = link_finder("phobos", soup) 337 | progress_bar.update() 338 | 339 | for n in range(2, page_number + 1): 340 | resp = s.get(phobos_url.format(quote(searchstr), n), proxies=proxies, headers=random_headers()) 341 | soup = BeautifulSoup(resp.text, 'html5lib') 342 | results = results + link_finder("phobos", soup) 343 | progress_bar.update() 344 | 345 | return results 346 | 347 | 348 | def onionsearchserver(searchstr): 349 | results = [] 350 | onionsearchserver_url1 = supported_engines['onionsearchserver'] + "/oss/" 351 | onionsearchserver_url2 = None 352 | results_per_page = 10 353 | max_nb_page = 100 354 | if args.limit != 0: 355 | max_nb_page = args.limit 356 | 357 | with requests.Session() as s: 358 | s.proxies = proxies 359 | s.headers = random_headers() 360 | 361 | resp = s.get(onionsearchserver_url1) 362 | soup = BeautifulSoup(resp.text, 'html5lib') 363 | for i in soup.find_all('iframe', attrs={"style": "display:none;"}): 364 | onionsearchserver_url2 = i['src'] + "{}&page={}" 365 | 366 | if onionsearchserver_url2 is None: 367 | return results 368 | 369 | resp = s.get(onionsearchserver_url2.format(quote(searchstr), 1)) 370 | soup = BeautifulSoup(resp.text, 'html5lib') 371 | 372 | page_number = 1 373 | pages = soup.find_all("div", attrs={"class": "osscmnrdr ossnumfound"}) 374 | if pages is not None and not str(pages[0].get_text()).startswith("No"): 375 | total_results = float(str.split(clear(pages[0].get_text()))[0]) 376 | page_number = math.ceil(total_results / results_per_page) 377 | if page_number > max_nb_page: 378 | page_number = max_nb_page 379 | 380 | pos = get_proc_pos() 381 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Onion Search Server", pos), position=pos) \ 382 | as progress_bar: 383 | 384 | results = link_finder("onionsearchserver", soup) 385 | progress_bar.update() 386 | 387 | for n in range(2, page_number + 1): 388 | resp = s.get(onionsearchserver_url2.format(quote(searchstr), n)) 389 | soup = BeautifulSoup(resp.text, 'html5lib') 390 | results = results + link_finder("onionsearchserver", soup) 391 | progress_bar.update() 392 | 393 | return results 394 | 395 | 396 | 397 | def torgle(searchstr): 398 | results = [] 399 | torgle_url = supported_engines['torgle'] + "/search.php?term={}" 400 | 401 | pos = get_proc_pos() 402 | with tqdm(total=1, initial=0, desc=get_tqdm_desc("Torgle", pos), position=pos) as progress_bar: 403 | response = requests.get(torgle_url.format(quote(searchstr)), proxies=proxies, headers=random_headers()) 404 | soup = BeautifulSoup(response.text, 'html5lib') 405 | results = link_finder("torgle", soup) 406 | progress_bar.update() 407 | 408 | return results 409 | 410 | 411 | def onionsearchengine(searchstr): 412 | results = [] 413 | onionsearchengine_url = supported_engines['onionsearchengine'] + "/search.php?search={}&submit=Search&page={}" 414 | # same as onionsearchengine_url = "http://5u56fjmxu63xcmbk.onion/search.php?search={}&submit=Search&page={}" 415 | max_nb_page = 100 416 | if args.limit != 0: 417 | max_nb_page = args.limit 418 | 419 | with requests.Session() as s: 420 | s.proxies = proxies 421 | s.headers = random_headers() 422 | 423 | resp = s.get(onionsearchengine_url.format(quote(searchstr), 1)) 424 | soup = BeautifulSoup(resp.text, 'html5lib') 425 | 426 | page_number = 1 427 | approx_re = re.search(r"\s([0-9]+)\sresult[s]?\sfound\s!.*", clear(soup.find('body').get_text())) 428 | if approx_re is not None: 429 | nb_res = int(approx_re.group(1)) 430 | results_per_page = 9 431 | page_number = math.ceil(float(nb_res / results_per_page)) 432 | if page_number > max_nb_page: 433 | page_number = max_nb_page 434 | 435 | pos = get_proc_pos() 436 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Onion Search Engine", pos), position=pos) \ 437 | as progress_bar: 438 | 439 | results = link_finder("onionsearchengine", soup) 440 | progress_bar.update() 441 | 442 | for n in range(2, page_number + 1): 443 | resp = s.get(onionsearchengine_url.format(quote(searchstr), n)) 444 | soup = BeautifulSoup(resp.text, 'html5lib') 445 | results = results + link_finder("onionsearchengine", soup) 446 | progress_bar.update() 447 | 448 | return results 449 | 450 | 451 | def tordex(searchstr): 452 | results = [] 453 | tordex_url = supported_engines['tordex'] + "/search?query={}&page={}" 454 | max_nb_page = 100 455 | if args.limit != 0: 456 | max_nb_page = args.limit 457 | 458 | with requests.Session() as s: 459 | s.proxies = proxies 460 | s.headers = random_headers() 461 | 462 | resp = s.get(tordex_url.format(quote(searchstr), 1)) 463 | soup = BeautifulSoup(resp.text, 'html5lib') 464 | 465 | page_number = 1 466 | pages = soup.find_all("li", attrs={"class": "page-item"}) 467 | if pages is not None: 468 | for i in pages: 469 | if i.get_text() != "...": 470 | page_number = int(i.get_text()) 471 | if page_number > max_nb_page: 472 | page_number = max_nb_page 473 | 474 | pos = get_proc_pos() 475 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Tordex", pos), position=pos) as progress_bar: 476 | 477 | results = link_finder("tordex", soup) 478 | progress_bar.update() 479 | 480 | for n in range(2, page_number + 1): 481 | resp = s.get(tordex_url.format(quote(searchstr), n)) 482 | soup = BeautifulSoup(resp.text, 'html5lib') 483 | results = results + link_finder("tordex", soup) 484 | progress_bar.update() 485 | 486 | return results 487 | 488 | 489 | def tor66(searchstr): 490 | results = [] 491 | tor66_url = supported_engines['tor66'] + "/search?q={}&sorttype=rel&page={}" 492 | max_nb_page = 30 493 | if args.limit != 0: 494 | max_nb_page = args.limit 495 | 496 | with requests.Session() as s: 497 | s.proxies = proxies 498 | s.headers = random_headers() 499 | 500 | resp = s.get(tor66_url.format(quote(searchstr), 1)) 501 | soup = BeautifulSoup(resp.text, 'html5lib') 502 | 503 | page_number = 1 504 | approx_re = re.search(r"\.Onion\ssites\sfound\s:\s([0-9]+)", resp.text) 505 | if approx_re is not None: 506 | nb_res = int(approx_re.group(1)) 507 | results_per_page = 20 508 | page_number = math.ceil(float(nb_res / results_per_page)) 509 | if page_number > max_nb_page: 510 | page_number = max_nb_page 511 | 512 | pos = get_proc_pos() 513 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Tor66", pos), position=pos) as progress_bar: 514 | 515 | results = link_finder("tor66", soup) 516 | progress_bar.update() 517 | 518 | for n in range(2, page_number + 1): 519 | resp = s.get(tor66_url.format(quote(searchstr), n)) 520 | soup = BeautifulSoup(resp.text, 'html5lib') 521 | results = results + link_finder("tor66", soup) 522 | progress_bar.update() 523 | 524 | return results 525 | 526 | 527 | def tormax(searchstr): 528 | results = [] 529 | tormax_url = supported_engines['tormax'] + "/search?q={}" 530 | 531 | pos = get_proc_pos() 532 | with tqdm(total=1, initial=0, desc=get_tqdm_desc("Tormax", pos), position=pos) as progress_bar: 533 | response = requests.get(tormax_url.format(quote(searchstr)), proxies=proxies, headers=random_headers()) 534 | soup = BeautifulSoup(response.text, 'html5lib') 535 | results = link_finder("tormax", soup) 536 | progress_bar.update() 537 | 538 | return results 539 | 540 | 541 | def haystack(searchstr): 542 | results = [] 543 | haystack_url = supported_engines['haystack'] + "/?q={}&offset={}" 544 | # At the 52nd page, it timeouts 100% of the time 545 | max_nb_page = 50 546 | if args.limit != 0: 547 | max_nb_page = args.limit 548 | offset_coeff = 20 549 | 550 | with requests.Session() as s: 551 | s.proxies = proxies 552 | s.headers = random_headers() 553 | 554 | req = s.get(haystack_url.format(quote(searchstr), 0)) 555 | soup = BeautifulSoup(req.text, 'html5lib') 556 | 557 | pos = get_proc_pos() 558 | with tqdm(total=max_nb_page, initial=0, desc=get_tqdm_desc("Haystack", pos), position=pos) as progress_bar: 559 | continue_processing = True 560 | ret = link_finder("haystack", soup) 561 | results = results + ret 562 | progress_bar.update() 563 | if len(ret) == 0: 564 | continue_processing = False 565 | 566 | it = 1 567 | while continue_processing: 568 | offset = int(it * offset_coeff) 569 | req = s.get(haystack_url.format(quote(searchstr), offset)) 570 | soup = BeautifulSoup(req.text, 'html5lib') 571 | ret = link_finder("haystack", soup) 572 | results = results + ret 573 | progress_bar.update() 574 | it += 1 575 | if it >= max_nb_page or len(ret) == 0: 576 | continue_processing = False 577 | 578 | return results 579 | 580 | 581 | def multivac(searchstr): 582 | results = [] 583 | multivac_url = supported_engines['multivac'] + "/?q={}&page={}" 584 | max_nb_page = 10 585 | if args.limit != 0: 586 | max_nb_page = args.limit 587 | 588 | with requests.Session() as s: 589 | s.proxies = proxies 590 | s.headers = random_headers() 591 | 592 | page_to_request = 1 593 | req = s.get(multivac_url.format(quote(searchstr), page_to_request)) 594 | soup = BeautifulSoup(req.text, 'html5lib') 595 | 596 | pos = get_proc_pos() 597 | with tqdm(total=max_nb_page, initial=0, desc=get_tqdm_desc("Multivac", pos), position=pos) as progress_bar: 598 | continue_processing = True 599 | ret = link_finder("multivac", soup) 600 | results = results + ret 601 | progress_bar.update() 602 | if len(ret) == 0 or page_to_request >= max_nb_page: 603 | continue_processing = False 604 | 605 | while continue_processing: 606 | page_to_request += 1 607 | req = s.get(multivac_url.format(quote(searchstr), page_to_request)) 608 | soup = BeautifulSoup(req.text, 'html5lib') 609 | ret = link_finder("multivac", soup) 610 | results = results + ret 611 | progress_bar.update() 612 | if len(ret) == 0 or page_to_request >= max_nb_page: 613 | continue_processing = False 614 | 615 | return results 616 | 617 | 618 | def evosearch(searchstr): 619 | results = [] 620 | evosearch_url = supported_engines['evosearch'] + "/evo/search.php?" \ 621 | "query={}&" \ 622 | "start={}&" \ 623 | "search=1&type=and&mark=bold+text&" \ 624 | "results={}" 625 | results_per_page = 50 626 | max_nb_page = 30 627 | if args.limit != 0: 628 | max_nb_page = args.limit 629 | 630 | with requests.Session() as s: 631 | s.proxies = proxies 632 | s.headers = random_headers() 633 | 634 | req = s.get(evosearch_url.format(quote(searchstr), 1, results_per_page)) 635 | soup = BeautifulSoup(req.text, 'html5lib') 636 | 637 | page_number = 1 638 | i = soup.find("p", attrs={"class": "cntr"}) 639 | if i is not None: 640 | if i.get_text() is not None and "of" in i.get_text(): 641 | nb_res = float(clear(str.split(i.get_text().split("-")[1].split("of")[1])[0])) 642 | page_number = math.ceil(nb_res / results_per_page) 643 | if page_number > max_nb_page: 644 | page_number = max_nb_page 645 | 646 | pos = get_proc_pos() 647 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Evo Search", pos), position=pos) as progress_bar: 648 | results = link_finder("evosearch", soup) 649 | progress_bar.update() 650 | 651 | for n in range(2, page_number + 1): 652 | resp = s.get(evosearch_url.format(quote(searchstr), n, results_per_page)) 653 | soup = BeautifulSoup(resp.text, 'html5lib') 654 | results = results + link_finder("evosearch", soup) 655 | progress_bar.update() 656 | 657 | return results 658 | 659 | 660 | 661 | def deeplink(searchstr): 662 | results = [] 663 | deeplink_url1 = supported_engines['deeplink'] + "/index.php" 664 | deeplink_url2 = supported_engines['deeplink'] + "/?search={}&type=verified" 665 | 666 | with requests.Session() as s: 667 | s.proxies = proxies 668 | s.headers = random_headers() 669 | s.get(deeplink_url1) 670 | 671 | pos = get_proc_pos() 672 | with tqdm(total=1, initial=0, desc=get_tqdm_desc("DeepLink", pos), position=pos) as progress_bar: 673 | response = s.get(deeplink_url2.format(quote(searchstr))) 674 | soup = BeautifulSoup(response.text, 'html5lib') 675 | results = link_finder("deeplink", soup) 676 | progress_bar.update() 677 | 678 | return results 679 | 680 | 681 | 682 | def torgle1(searchstr): 683 | results = [] 684 | torgle1_url = supported_engines['torgle1'] + "/torgle/index-frame.php?query={}&search=1&engine-ver=2&isframe=0{}" 685 | results_per_page = 10 686 | max_nb_page = 30 687 | if args.limit != 0: 688 | max_nb_page = args.limit 689 | 690 | with requests.Session() as s: 691 | s.proxies = proxies 692 | s.headers = random_headers() 693 | 694 | resp = s.get(torgle1_url.format(quote(searchstr), "")) 695 | soup = BeautifulSoup(resp.text, 'html5lib') 696 | 697 | page_number = 1 698 | i = soup.find('div', attrs={"id": "result_report"}) 699 | if i is not None: 700 | if i.get_text() is not None and "of" in i.get_text(): 701 | res_re = re.match(r".*of\s([0-9]+)\s.*", clear(i.get_text())) 702 | total_results = int(res_re.group(1)) 703 | page_number = math.ceil(total_results / results_per_page) 704 | if page_number > max_nb_page: 705 | page_number = max_nb_page 706 | 707 | pos = get_proc_pos() 708 | with tqdm(total=page_number, initial=0, desc=get_tqdm_desc("Torgle 1", pos), position=pos) as progress_bar: 709 | results = link_finder("torgle1", soup) 710 | progress_bar.update() 711 | 712 | for n in range(2, page_number + 1): 713 | start_page_param = "&start={}".format(n) 714 | resp = s.get(torgle1_url.format(quote(searchstr), start_page_param)) 715 | soup = BeautifulSoup(resp.text, 'html5lib') 716 | results = results + link_finder("torgle1", soup) 717 | progress_bar.update() 718 | 719 | return results 720 | 721 | 722 | 723 | 724 | def get_domain_from_url(link): 725 | fqdn_re = r"^[a-z][a-z0-9+\-.]*://([a-z0-9\-._~%!$&'()*+,;=]+@)?([a-z0-9\-._~%]+|\[[a-z0-9\-._~%!$&'()*+,;=:]+\])" 726 | domain_re = re.match(fqdn_re, link) 727 | if domain_re is not None: 728 | if domain_re.lastindex == 2: 729 | return domain_re.group(2) 730 | return None 731 | 732 | 733 | def write_to_csv(csv_writer, fields): 734 | line_to_write = [] 735 | if args.fields and len(args.fields) > 0: 736 | for f in args.fields[0]: 737 | if f in fields: 738 | line_to_write.append(fields[f]) 739 | if f == "domain": 740 | domain = get_domain_from_url(fields['link']) 741 | line_to_write.append(domain) 742 | csv_writer.writerow(line_to_write) 743 | else: 744 | # Default output mode 745 | line_to_write.append(fields['engine']) 746 | line_to_write.append(fields['name']) 747 | line_to_write.append(fields['link']) 748 | csv_writer.writerow(line_to_write) 749 | 750 | 751 | def link_finder(engine_str, data_obj): 752 | global filename 753 | name = "" 754 | link = "" 755 | csv_file = None 756 | found_links = [] 757 | 758 | if args.continuous_write: 759 | csv_file = open(filename, 'a', newline='') 760 | 761 | def add_link(): 762 | found_links.append({"engine": engine_str, "name": name, "link": link}) 763 | 764 | if args.continuous_write and csv_file.writable(): 765 | csv_writer = csv.writer(csv_file, delimiter=field_delim, quoting=csv.QUOTE_ALL) 766 | fields = {"engine": engine_str, "name": name, "link": link} 767 | write_to_csv(csv_writer, fields) 768 | 769 | if engine_str == "ahmia": 770 | for r in data_obj.select('li.result h4'): 771 | name = clear(r.get_text()) 772 | link = r.find('a')['href'].split('redirect_url=')[1] 773 | add_link() 774 | 775 | if engine_str == "darksearchenginer": 776 | for r in data_obj.select('.table-responsive a'): 777 | name = clear(r.get_text()) 778 | link = clear(r['href']) 779 | add_link() 780 | 781 | if engine_str == "darksearchio": 782 | for r in data_obj: 783 | name = clear(r["title"]) 784 | link = clear(r["link"]) 785 | add_link() 786 | 787 | if engine_str == "deeplink": 788 | for tr in data_obj.find_all('tr'): 789 | cels = tr.find_all('td') 790 | if cels is not None and len(cels) == 4: 791 | name = clear(cels[1].get_text()) 792 | link = clear(cels[0].find('a')['href']) 793 | add_link() 794 | 795 | if engine_str == "evosearch": 796 | for r in data_obj.select("#results .title a"): 797 | name = clear(r.get_text()) 798 | link = get_parameter(r['href'], 'url') 799 | add_link() 800 | 801 | 802 | 803 | if engine_str == "haystack": 804 | for r in data_obj.select(".result b a"): 805 | name = clear(r.get_text()) 806 | link = get_parameter(r['href'], 'url') 807 | add_link() 808 | 809 | if engine_str == "multivac": 810 | for r in data_obj.select("dl dt a"): 811 | if r['href'] != "": 812 | name = clear(r.get_text()) 813 | link = clear(r['href']) 814 | add_link() 815 | else: 816 | break 817 | 818 | if engine_str == "notevil": 819 | for r in data_obj.find_all("p"): 820 | r=r.find("a") 821 | name = clear(r.get_text()) 822 | link = unquote(r["href"]).split('./r2d.php?url=')[1].split('&')[0] 823 | add_link() 824 | 825 | 826 | if engine_str == "onionland": 827 | for r in data_obj.select('.result-block .title a'): 828 | if not r['href'].startswith('/ads/'): 829 | name = clear(r.get_text()) 830 | link = unquote(unquote(get_parameter(r['href'], 'l'))) 831 | add_link() 832 | 833 | if engine_str == "onionsearchengine": 834 | for r in data_obj.select("table a b"): 835 | name = clear(r.get_text()) 836 | link = get_parameter(r.parent['href'], 'u') 837 | add_link() 838 | 839 | if engine_str == "onionsearchserver": 840 | for r in data_obj.select('.osscmnrdr.ossfieldrdr1 a'): 841 | name = clear(r.get_text()) 842 | link = clear(r['href']) 843 | add_link() 844 | 845 | if engine_str == "phobos": 846 | for r in data_obj.select('.serp .titles'): 847 | name = clear(r.get_text()) 848 | link = clear(r['href']) 849 | add_link() 850 | 851 | if engine_str == "tor66": 852 | for i in data_obj.find('hr').find_all_next('b'): 853 | if i.find('a'): 854 | name = clear(i.find('a').get_text()) 855 | link = clear(i.find('a')['href']) 856 | add_link() 857 | 858 | 859 | if engine_str == "tordex": 860 | for r in data_obj.select('.container h5 a'): 861 | name = clear(r.get_text()) 862 | link = clear(r['href']) 863 | add_link() 864 | 865 | if engine_str == "torgle": 866 | for i in data_obj.find_all('ul', attrs={"id": "page"}): 867 | for j in i.find_all('a'): 868 | if str(j.get_text()).startswith("http"): 869 | link = clear(j.get_text()) 870 | else: 871 | name = clear(j.get_text()) 872 | add_link() 873 | 874 | if engine_str == "torgle1": 875 | for r in data_obj.select("#results a.title"): 876 | name = clear(r.get_text()) 877 | link = clear(r['href']) 878 | add_link() 879 | 880 | if engine_str == "tormax": 881 | for r in data_obj.find_all("section",attrs={"id":"search-results"})[0].find_all("article"): 882 | name = clear(r.find('a',attrs={"class":"title"}).get_text()) 883 | link = clear(r.find('div',attrs={"class":"url"}).get_text()) 884 | add_link() 885 | 886 | 887 | 888 | if args.continuous_write and not csv_file.closed: 889 | csv_file.close() 890 | 891 | return found_links 892 | 893 | 894 | def run_method(method_name_and_argument): 895 | method_name = method_name_and_argument.split(':')[0] 896 | argument = method_name_and_argument.split(':')[1] 897 | ret = [] 898 | try: 899 | ret = globals()[method_name](argument) 900 | except Exception as e: 901 | print(f'Exception occured: {e}') 902 | return ret 903 | 904 | 905 | def scrape(): 906 | global filename 907 | 908 | start_time = datetime.now() 909 | 910 | # Building the filename 911 | filename = str(filename).replace("$DATE", start_time.strftime("%Y%m%d%H%M%S")) 912 | search = str(args.search).replace(" ", "") 913 | if len(search) > 10: 914 | search = search[0:9] 915 | filename = str(filename).replace("$SEARCH", search) 916 | 917 | func_args = [] 918 | stats_dict = {} 919 | if args.engines and len(args.engines) > 0: 920 | eng = args.engines[0] 921 | for e in eng: 922 | try: 923 | if not (args.exclude and len(args.exclude) > 0 and e in args.exclude[0]): 924 | func_args.append("{}:{}".format(e, args.search)) 925 | stats_dict[e] = 0 926 | except KeyError: 927 | print("Error: search engine {} not in the list of supported engines".format(e)) 928 | else: 929 | for e in supported_engines.keys(): 930 | if not (args.exclude and len(args.exclude) > 0 and e in args.exclude[0]): 931 | func_args.append("{}:{}".format(e, args.search)) 932 | stats_dict[e] = 0 933 | 934 | # Doing multiprocessing 935 | if args.mp_units and args.mp_units > 0: 936 | units = args.mp_units 937 | else: 938 | # Use (cores count - 1), but not less then one, threads 939 | units = max((cpu_count() - 1), 1) 940 | print("search.py started with {} processing units...".format(units)) 941 | freeze_support() 942 | 943 | results = {} 944 | with Pool(units) as p: 945 | results_map = p.map(run_method, func_args) 946 | results = reduce(lambda a, b: a + b if b is not None else a, results_map) 947 | 948 | stop_time = datetime.now() 949 | 950 | if not args.continuous_write: 951 | with open(filename, 'w', newline='') as csv_file: 952 | csv_writer = csv.writer(csv_file, delimiter=field_delim, quoting=csv.QUOTE_ALL) 953 | for r in results: 954 | write_to_csv(csv_writer, r) 955 | 956 | total = 0 957 | print("\nReport:") 958 | print(" Execution time: %s seconds" % (stop_time - start_time)) 959 | print(" Results per engine:") 960 | for r in results: 961 | stats_dict[r['engine']] += 1 962 | for s in stats_dict: 963 | n = stats_dict[s] 964 | print(" {}: {}".format(s, str(n))) 965 | total += n 966 | print(" Total: {} links written to {}".format(str(total), filename)) 967 | --------------------------------------------------------------------------------