├── maryam ├── __init__.py ├── core │ ├── __init__.py │ ├── util │ │ ├── __init__.py │ │ ├── iris │ │ │ ├── __init__.py │ │ │ ├── chart.py │ │ │ ├── meta_search_util.py │ │ │ └── tf_histogram.py │ │ ├── engines │ │ │ ├── __init__.py │ │ │ ├── baidu.py │ │ │ ├── yandex.py │ │ │ ├── bing_images.py │ │ │ └── etools.py │ │ ├── footprint │ │ │ └── __init__.py │ │ ├── helpers │ │ │ ├── __init__.py │ │ │ ├── keywords.py │ │ │ └── reglib.py │ │ ├── osint │ │ │ ├── __init__.py │ │ │ ├── ahmia.py │ │ │ ├── keyserver.py │ │ │ ├── bing_mobile_view.py │ │ │ ├── exalead.py │ │ │ ├── onionland.py │ │ │ ├── virustotal.py │ │ │ ├── urlscan.py │ │ │ ├── numverify.py │ │ │ ├── darksearch.py │ │ │ ├── hunter.py │ │ │ ├── netcraft.py │ │ │ └── reddit.py │ │ └── search │ │ │ ├── __init__.py │ │ │ ├── crt.py │ │ │ ├── discord.py │ │ │ ├── ask.py │ │ │ ├── photon.py │ │ │ ├── dictionary.py │ │ │ ├── scholar.py │ │ │ ├── pubmed.py │ │ │ ├── sepiasearch.py │ │ │ ├── openstreetmap.py │ │ │ ├── wikileaks.py │ │ │ ├── piratebay.py │ │ │ ├── arxiv.py │ │ │ ├── wikipedia.py │ │ │ ├── reddit_pushshift.py │ │ │ └── pastebin.py │ ├── web │ │ └── __init__.py │ └── basedir.py ├── modules │ ├── __init__.py │ ├── search │ │ ├── etools.py │ │ ├── arxiv.py │ │ ├── duckduckgo.py │ │ ├── pubmed.py │ │ ├── startpage.py │ │ ├── scholar.py │ │ ├── piratebay.py │ │ ├── bing.py │ │ ├── google.py │ │ ├── yahoo.py │ │ ├── dictionary.py │ │ ├── sepiasearch.py │ │ ├── wikileaks.py │ │ ├── pastebin.py │ │ ├── crt.py │ │ ├── wikipedia.py │ │ ├── sanctionsearch.py │ │ ├── photon.py │ │ ├── discord.py │ │ ├── youtube.py │ │ ├── linkedin.py │ │ ├── twitter.py │ │ ├── trello.py │ │ ├── spotify.py │ │ ├── quora.py │ │ ├── tiktok.py │ │ └── facebook.py │ ├── osint │ │ ├── suggest.py │ │ ├── phone_number_search.py │ │ ├── onion_search.py │ │ ├── email_pwned.py │ │ ├── article_search.py │ │ ├── image_search.py │ │ ├── crawler.py │ │ ├── docs_search.py │ │ ├── cloud_storage.py │ │ ├── email_search.py │ │ └── social_nets.py │ ├── iris │ │ ├── cluster.py │ │ ├── iris_cluster.py │ │ ├── iris.py │ │ ├── topicmodeling.py │ │ └── sentiment.py │ └── footprint │ │ └── entry_points.py ├── data │ └── stopwords.csv └── __main__.py ├── MANIFEST.in ├── .gitignore ├── maryam.py ├── bin └── maryam ├── requirements ├── .github ├── ISSUE_TEMPLATE │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── CONTRIBUTING.md ├── .travis.yml ├── Makefile ├── setup.py ├── docs └── index.rst └── README.md /maryam/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/web/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/iris/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include maryam/data/* 2 | -------------------------------------------------------------------------------- /maryam/core/util/engines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/footprint/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/osint/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /maryam/core/util/search/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | *.pyc 3 | __pycache__/ 4 | dist/ 5 | -------------------------------------------------------------------------------- /maryam.py: -------------------------------------------------------------------------------- 1 | 2 | from maryam import __main__ 3 | import sys 4 | 5 | if __name__ == '__main__': 6 | __main__.cui(sys.argv[1:]) 7 | -------------------------------------------------------------------------------- /maryam/core/basedir.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../')) 4 | -------------------------------------------------------------------------------- /bin/maryam: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from maryam import __main__ 4 | import sys 5 | 6 | if __name__ == '__main__': 7 | __main__.cui(sys.argv[1:]) 8 | -------------------------------------------------------------------------------- /requirements: -------------------------------------------------------------------------------- 1 | beautifulsoup4 2 | bertopic 3 | certifi 4 | cloudscraper 5 | dask 6 | Flask 7 | huggingface-hub 8 | lxml 9 | matplotlib 10 | numpy 11 | packaging 12 | pandas 13 | requests 14 | scikit-learn 15 | sentence-transformers 16 | top2vec 17 | transformers 18 | typing-extensions 19 | umap 20 | umap-learn 21 | vaderSentiment 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | Name: Feature request 3 | About: Suggest an idea for the project 4 | 5 | --- 6 | 7 | #### Is your feature request related to a problem? Please describe. 8 | A clear and concise description of what the problem is. 9 | 10 | #### Describe the solution you'd like 11 | A clear and concise description of what you want to happen. 12 | 13 | #### Describe alternatives you've considered 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | #### Additional context 17 | Add any other context or screenshots about the feature request here. -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | Fixes # 8 | 9 | 10 | #### Resolves 11 | 12 | 13 | 14 | 15 | #### Changes 16 | 17 | - 18 | - 19 | - 20 | 21 | #### Checklist 22 | 23 | - [ ] I have read the Contribution & Best practices Guideline. 24 | - [ ] My branch is up-to-date with the Upstream `master` branch. 25 | - [ ] The acceptance, integration, unit tests pass locally with my changes 26 | - [ ] I have added tests that prove my fix is effective or that my feature works 27 | - [ ] I have added necessary documentation (if appropriate) -------------------------------------------------------------------------------- /maryam/data/stopwords.csv: -------------------------------------------------------------------------------- 1 | while,should've,weren't,on,your,had,couldn't,isn,re,own,having,its,further,it,didn,have,hasn,our,shan't,between,we,only,few,o,are,too,himself,hers,and,in,both,those,does,to,out,be,under,no,through,yours,won,it's,didn't,yourselves,just,don,she's,hasn't,than,will,you've,during,what,shan,all,the,who,he,ourselves,their,because,you,couldn,into,weren,shouldn't,should,she,do,doing,up,so,whom,wasn't,s,more,over,such,myself,by,that'll,ain,where,being,here,am,doesn,ve,d,same,now,mightn,of,wouldn,hadn't,down,after,don't,you'd,been,isn't,yourself,again,aren't,with,a,each,theirs,or,off,were,when,nor,that,herself,needn,until,haven,can,him,did,wasn,his,y,hadn,any,itself,as,other,was,why,shouldn,has,there,haven't,mightn't,ours,some,is,me,at,you'll,wouldn't,if,but,needn't,before,i,which,t,not,won't,then,my,mustn't,once,ma,aren,they,above,them,ll,against,from,m,an,you're,doesn't,mustn,below,for,how,this,these,most,about,themselves,her,very 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | jobs: 3 | include: 4 | # - name: "Python 3.8.0 on Windows" Needs some change in the project 5 | # os: windows 6 | # language: shell 7 | # before_install: 8 | # - choco install python --version 3.8.0 9 | # - python -m pip install --upgrade pip 10 | # env: PATH=/c/Python38:/c/Python38/Scripts:$PATH 11 | - name: "Python 3.8.0 on Xenial Linux" 12 | python: 3.8 13 | - name: "Python 3.8.0 on macOS" 14 | os: osx 15 | osx_image: xcode11.2 16 | language: shell 17 | before_install: 18 | - python3 -m pip install --upgrade pip 19 | script: 20 | - python3 maryam -h 21 | - python3 maryam -e linkedin -q "OWASP" 22 | - python3 maryam -e show modules 23 | - name: "Python 3.8 on FreeBSD" 24 | python: 3.8 25 | os: freebsd 26 | - name: "Python 3.9 on Xenial Linux" 27 | python: 3.9 28 | - name: "Python 3.9 on FreeBSD" 29 | python: 3.9 30 | os: freebsd 31 | install: 32 | - "pip install -r requirements" 33 | script: 34 | - python maryam -h 35 | - python maryam -e show modules 36 | - python maryam -e linkedin -q "OWASP" -------------------------------------------------------------------------------- /maryam/modules/search/etools.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Etools Search', 20 | 'author': 'Saeed', 21 | 'version': '0.1', 22 | 'description': 'Search your query in the etools.ch and show the results.', 23 | 'sources': ('etools',), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ), 27 | 'examples': ('etools -q ',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | run = self.etools(query) 33 | run.run_crawl() 34 | results = run.results 35 | output = {'results': results} 36 | self.save_gather(output, 'search/etools', query, output=self.options['output']) 37 | return output 38 | 39 | def module_run(self): 40 | self.search_engine_results(module_api(self)) 41 | -------------------------------------------------------------------------------- /maryam/core/util/iris/chart.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | class main: 17 | 18 | def __init__(self): 19 | """ show histogram plot of web term frequency """ 20 | self.framework = main.framework 21 | 22 | def line_chart(vectors, title, xlabel, ylabel, **args): 23 | import matplotlib.pyplot as plt 24 | 25 | for vector in vectors: 26 | x = [i for i in range(len(vector))] 27 | plt.plot(x, vector, **args) 28 | plt.xlabel(xlabel) 29 | plt.ylabel(ylabel) 30 | plt.title(title) 31 | plt.legend() 32 | plt.show() 33 | 34 | def scatter_chart(vectors, title, xlabel, ylabel, **args): 35 | import matplotlib.pyplot as plt 36 | 37 | for vector in vectors: 38 | plt.scatter(vector[0], vector[1], **args) 39 | plt.xlabel(xlabel) 40 | plt.ylabel(ylabel) 41 | plt.title(title) 42 | plt.legend() 43 | plt.show() 44 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | first: help 3 | 4 | 5 | # ------------------------------------------------------------------------------ 6 | # Build 7 | 8 | install: ## Install package 9 | pip install . 10 | 11 | 12 | build: ## Build package by sdist 13 | python3 setup.py sdist 14 | 15 | 16 | build-default: ## Build package by build (recommended) 17 | python3 -m build 18 | 19 | 20 | uninstall: ## Uninstall package 21 | pip uninstall maryam 22 | 23 | upload-pypi: ## Upload package to PyPI 24 | python3 -m twine upload dist/*.tar.gz 25 | 26 | 27 | upload-test: ## Upload package to test PyPI 28 | python3 -m twine upload --repository test dist/*.tar.gz 29 | 30 | 31 | # ------------------------------------------------------------------------------ 32 | # Testing 33 | 34 | check: ## Check linting 35 | flake8 36 | isort . --project maryam --check-only --diff 37 | black . --check 38 | 39 | 40 | fmt: ## Format source 41 | isort . --project maryam 42 | black . 43 | 44 | # ------------------------------------------------------------------------------ 45 | # Other 46 | 47 | clean: ## Clean build files 48 | rm -rf build dist site .pytest_cache .eggs 49 | find . -type f -name '*.py[co]' -delete 50 | find . -type d -name __pycache__ -exec rm -rf {} + 51 | rm -rf *.egg-info 52 | rm -r docs/build 53 | 54 | help: ## Show this help menu 55 | @grep -E '^[0-9a-zA-Z_-]+:.*?##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"; OFS="\t\t"}; {printf "\033[36m%-30s\033[0m %s\n", $$1, ($$2==""?"":$$2)}' 56 | -------------------------------------------------------------------------------- /maryam/core/util/osint/ahmia.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | class main: 21 | 22 | def __init__(self, q): 23 | """ ahmia.fi search engine 24 | 25 | q : query for search 26 | """ 27 | self.framework = main.framework 28 | self.q = self.framework.urlib(q).quote 29 | self._pages = '' 30 | self._links = [] 31 | self.ahmia = 'ahmia.fi' 32 | 33 | def run_crawl(self): 34 | url = f"https://{self.ahmia}/search/?q={self.q}" 35 | try: 36 | req = self.framework.request(url=url) 37 | except: 38 | self.framework.print_exception() 39 | self.framework.error('Ahmia is missed!', 'util/ahmia', 'run_crawl') 40 | self._pages = req.text 41 | 42 | @property 43 | def pages(self): 44 | return self._pages 45 | 46 | @property 47 | def links(self): 48 | return self.framework.page_parse(self._pages).findall(r'redirect_url=(.*?)">') 49 | -------------------------------------------------------------------------------- /maryam/modules/search/arxiv.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | meta = { 17 | 'name': 'ArXiv', 18 | 'author': 'Kaushik', 19 | 'version': '0.2', 20 | 'description': 'ArXiv is a scientific research repository \ 21 | with a public API for searching its articles and papers', 22 | 'sources': ('arxiv',), 23 | 'options': ( 24 | ('query', None, True, 'Query string', '-q', 'store', str), 25 | ('limit', 1, False, 'Max page count (default=1)', '-l', 'store', int), 26 | ), 27 | 'examples': ('arxiv -q -l 2 --output',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | limit = self.options['limit'] 33 | run = self.arxiv(query, limit) 34 | run.run_crawl() 35 | output = {'results': run.results} 36 | self.save_gather(output, 'search/arxiv', query, output=self.options['output']) 37 | return output 38 | 39 | def module_run(self): 40 | self.search_engine_results(module_api(self)) 41 | -------------------------------------------------------------------------------- /maryam/modules/search/duckduckgo.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'DuckDuckGo', 17 | 'author': 'Tarunesh Kumar', 18 | 'version': '0.1', 19 | 'description': 'Search your query in the duckduckgo.com and show the results.', 20 | 'sources': ('duckduckgo',), 21 | 'options': ( 22 | ('query', None, True, 'Query string', '-q', 'store', str), 23 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 24 | ), 25 | 'examples': ('duckduckgo -q -l 15 --output',) 26 | } 27 | 28 | def module_api(self): 29 | query = self.options['query'] 30 | count = self.options['count'] 31 | run = self.duckduckgo(query, count) 32 | run.run_crawl() 33 | results = run.results 34 | output = {'results': results} 35 | self.save_gather(output, 'search/duckduckgo', query, output=self.options['output']) 36 | return output 37 | 38 | def module_run(self): 39 | self.search_engine_results(module_api(self)) 40 | -------------------------------------------------------------------------------- /maryam/modules/search/pubmed.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | meta = { 17 | 'name': 'Pubmed', 18 | 'author': 'Kaushik', 19 | 'version': '0.1', 20 | 'description': 'PubMed comprises more than 32 million citations for biomedical \ 21 | literature from MEDLINE, life science journals, and online books.', 22 | 'sources': ('pubmed',), 23 | 'options': ( 24 | ('query', None, True, 'Query string', '-q', 'store', str), 25 | ('count', 15, False, 'Max result count (default=15)', '-c', 'store', int), 26 | ), 27 | 'examples': ('pubmed -q -l 15 --output',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | count = self.options['count'] 33 | run = self.pubmed(query, count) 34 | run.run_crawl() 35 | output = {'results': run.results} 36 | self.save_gather(output, 'search/pubmed', query, output=self.options['output']) 37 | return output 38 | 39 | def module_run(self): 40 | self.search_engine_results(module_api(self)) 41 | -------------------------------------------------------------------------------- /maryam/modules/search/startpage.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Startpage Search', 20 | 'author': 'Saeed', 21 | 'version': '0.2', 22 | 'description': 'Search your query in the startpage.com and show the results.', 23 | 'options': ( 24 | ('query', None, True, 'Query string', '-q', 'store', str), 25 | ('limit', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-l', 'store', int), 26 | ), 27 | 'examples': ('startpage -q -l 15 --output --api',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | limit = self.options['limit'] 33 | run = self.startpage(query, limit) 34 | run.run_crawl() 35 | results = run.results 36 | output = {'results': results} 37 | self.save_gather(output, 'search/startpage', query, output=self.options['output']) 38 | return output 39 | 40 | def module_run(self): 41 | output = module_api(self) 42 | self.search_engine_results(output) 43 | -------------------------------------------------------------------------------- /maryam/modules/osint/suggest.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Search engine suggestions', 17 | 'author': 'Saeed', 18 | 'version': '0.1', 19 | 'description': 'Keyword autocompleter to find suggestions in search engines', 20 | 'sources': ('google', 'bing', 'yahoo', 'searx'), 21 | 'comments': ( 22 | """example: query='google' out: ['google docs', 'google summer of code', 'google maps', 'google mail', 'google news', ..]""", 23 | ), 24 | 'options': ( 25 | ('query', None, True, 'keyword, domain name, company name, etc', '-q', 'store', str), 26 | ), 27 | 'examples': ('suggest -q amazon --output',) 28 | 29 | } 30 | 31 | def module_api(self): 32 | q = self.options['query'] 33 | run = self.keywords(q) 34 | run.run_crawl() 35 | output = {'suggestions': run.keys} 36 | self.save_gather(output, 'osint/suggest', q, output=self.options['output']) 37 | return output 38 | 39 | def module_run(self): 40 | self.alert_results(module_api(self)) 41 | -------------------------------------------------------------------------------- /maryam/modules/search/scholar.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | meta = { 17 | 'name': 'Google Scholar', 18 | 'author': 'Kaushik', 19 | 'version': '0.1', 20 | 'description': 'Google Scholar is a freely accessible web search engine that \ 21 | indexes scholarly literature across an array of publishing formats and disciplines.', 22 | 'sources': ('scholar',), 23 | 'options': ( 24 | ('query', None, True, 'Query string', '-q', 'store', str), 25 | ('limit', 15, False, 'Max result count (default=15)', '-l', 'store', int), 26 | ), 27 | 'examples': ('scholar -q -l 15 --output',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | limit = self.options['limit'] 33 | run = self.scholar(query, limit) 34 | run.run_crawl() 35 | output = {'results': run.results} 36 | self.save_gather(output, 'search/scholar', query, output=self.options['output']) 37 | return output 38 | 39 | def module_run(self): 40 | self.search_engine_results(module_api(self)) 41 | -------------------------------------------------------------------------------- /maryam/modules/search/piratebay.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | meta = { 17 | 'name': 'Piratebay', 18 | 'author': 'Kaushik', 19 | 'version': '0.1', 20 | 'description': "Piratebay is the world's most persistent torrenting site. \ 21 | Please refrain from abusive usage.", 22 | 'sources': ('piratebay',), 23 | 'options': ( 24 | ('query', None, True, 'Query string', '-q', 'store', str), 25 | ('count', 15, False, 'Max result count (default=15)', '-c', 'store', int), 26 | ), 27 | 'examples': ('piratebay -q -l 15 --output',) 28 | } 29 | 30 | def module_api(self): 31 | query = self.options['query'] 32 | count = self.options['count'] 33 | run = self.piratebay(query, count) 34 | run.run_crawl() 35 | output = {'results': []} 36 | links = run.results 37 | 38 | for item in links: 39 | output['results'].append(item) 40 | 41 | self.save_gather(output, 'search/piratebay', query, output=self.options['output']) 42 | return output 43 | 44 | def module_run(self): 45 | self.search_engine_results(module_api(self)) 46 | -------------------------------------------------------------------------------- /maryam/modules/osint/phone_number_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Phone Number Search', 20 | 'author': 'Kaushik', 21 | 'version': '0.1', 22 | 'description': 'A Search Engine for Phone Number Enumeration.', 23 | 'sources': ('numverify'), 24 | 'options': ( 25 | ('number', None, True, 'Phone number (include area code)', '-n', 'store', str), 26 | ), 27 | 'examples': ('phone_number_search -n +911234567890') 28 | } 29 | 30 | def module_api(self): 31 | num = self.options['number'] 32 | run = self.numverify(num) 33 | run.run_crawl() 34 | output = run.json 35 | self.save_gather(output, 'osint/phone_number_search', num, output=self.options['output']) 36 | return output 37 | 38 | def module_run(self): 39 | output = module_api(self) 40 | if output['valid']: 41 | self.alert_results(output) 42 | else: 43 | self.error('Invalid Number!', 'phone_number_search', 'module_run') 44 | self.error("Number must be of the form '+{area code}{ten digit number}'", 'phone_number_search', 'module_run') 45 | -------------------------------------------------------------------------------- /maryam/modules/search/bing.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Bing Search', 20 | 'author': 'Saeed', 21 | 'version': '0.2', 22 | 'description': 'Search your query in the bing.com and show the results.', 23 | 'sources': ('bing',), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 28 | ), 29 | 'examples': ('bing -q -l 15 --output --api',) 30 | } 31 | 32 | def module_api(self): 33 | query = self.options['query'] 34 | limit = self.options['limit'] 35 | count = self.options['count'] 36 | run = self.bing(query, limit, count) 37 | run.run_crawl() 38 | results = run.results 39 | output = {'results': results} 40 | self.save_gather(output, 'search/bing', query, output=self.options['output']) 41 | return output 42 | 43 | def module_run(self): 44 | self.search_engine_results(module_api(self)) 45 | -------------------------------------------------------------------------------- /maryam/modules/iris/cluster.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | from json import loads 16 | 17 | meta = { 18 | 'name': 'Cluster', 19 | 'author': 'Kaushik', 20 | 'version': '0.1', 21 | 'description': 'Cluster your data using kmeans and fp-growth.', 22 | 'required': ('sklearn', 'kneed', 'mlxtend', 'numpy', 'pandas'), 23 | 'options': ( 24 | ('json', None, True, 'Json file that contains the data', '-j', 'store', str), 25 | ), 26 | 'examples': ('cluster -j test.json') 27 | } 28 | 29 | def module_api(self): 30 | json = self.options['json'] 31 | file = self._is_readable(json) 32 | if not file: 33 | return 34 | data = loads(file.read()) 35 | clusterer = self.cluster(data) 36 | output = {'json': clusterer.perform_clustering()} 37 | # self.save_gather(output, 'iris/cluster', json, output=self.options['output']) 38 | return output 39 | 40 | def module_run(self): 41 | output = module_api(self)['json'] 42 | for index, title in enumerate(output): 43 | print('\n') 44 | print(f"CLUSTER {index+1}") 45 | print(f"TITLE: {title}") 46 | print(' '+'\n '.join(output[title])) 47 | -------------------------------------------------------------------------------- /maryam/modules/search/google.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Google Search', 20 | 'author': 'Saeed', 21 | 'version': '0.6', 22 | 'description': 'Search your query in the google.com and show the results.', 23 | 'sources': ('google',), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 28 | ), 29 | 'examples': ('google -q -l 15 --output --api',) 30 | } 31 | 32 | def module_api(self): 33 | query = self.options['query'] 34 | limit = self.options['limit'] 35 | count = self.options['count'] 36 | run = self.google(query, limit, count) 37 | run.run_crawl() 38 | results = run.results 39 | output = {'results': results} 40 | self.save_gather(output, 'search/google', query, output=self.options['output']) 41 | return output 42 | 43 | def module_run(self): 44 | self.search_engine_results(module_api(self)) 45 | -------------------------------------------------------------------------------- /maryam/modules/search/yahoo.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Yahoo Search', 20 | 'author': 'Saeed', 21 | 'version': '0.1', 22 | 'description': 'Search your query in the search.yahoo.com and show the results.', 23 | 'sources': ('yahoo',), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 28 | ), 29 | 'examples': ('yahoo -q -l 15 --output --api',) 30 | } 31 | 32 | def module_api(self): 33 | query = self.options['query'] 34 | limit = self.options['limit'] 35 | count = self.options['count'] 36 | run = self.yahoo(query, limit, count) 37 | run.run_crawl() 38 | results = run.results 39 | output = {'results': results} 40 | self.save_gather(output, 'search/yahoo', query, output=self.options['output']) 41 | return output 42 | 43 | def module_run(self): 44 | self.search_engine_results(module_api(self)) 45 | -------------------------------------------------------------------------------- /maryam/modules/osint/onion_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Onions Network Search', 20 | 'author': 'Saeed', 21 | 'version': '0.4', 22 | 'description': 'onion_search is used to create the premier \ 23 | search engine for services residing on the Tor anonymity network.', 24 | 'sources': ('ahmia', 'onionland'), 25 | 'options': ( 26 | ('query', None, True, 'Domain Name,\ 27 | Company Name, keyword, etc', '-q', 'store', str), 28 | ), 29 | 'examples': ('onion_search -q ', 'onion_search -q --output') 30 | } 31 | 32 | def module_api(self): 33 | q = self.options['query'] 34 | ahmia = self.ahmia(q) 35 | ahmia.run_crawl() 36 | links = ahmia.links 37 | output = {'links': []} 38 | onionland = self.onionland(q, limit=5) 39 | onionland.run_crawl() 40 | links.extend(onionland.links) 41 | 42 | output['links'] = list(set(links)) 43 | self.save_gather(output, 'osint/onion_search', q, output=self.options['output']) 44 | return output 45 | 46 | def module_run(self): 47 | self.alert_results(module_api(self)) 48 | -------------------------------------------------------------------------------- /maryam/modules/search/dictionary.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Dictionary', 20 | 'author': 'Kaushik', 21 | 'version': '0.1', 22 | 'description': 'This module uses an unofficial google dictionary api to fetch \ 23 | definitions of query.', 24 | 'sources': ('dictionary',), 25 | 'options': ( 26 | ('query', None, True, 'Query to search', '-q', 'store', str), 27 | ), 28 | 'examples': ('dictionary -q ',) 29 | } 30 | 31 | def module_api(self): 32 | query = self.options['query'] 33 | run = self.dictionary(query) 34 | run.run_crawl() 35 | output = {'results': []} 36 | definitions = run.definitions 37 | 38 | for item in definitions: 39 | output['results'].append(item) 40 | 41 | self.save_gather(output, 'search/dictionary', query, output=self.options['output']) 42 | return output 43 | 44 | def module_run(self): 45 | output = module_api(self)['results'] 46 | for item in output: 47 | print() 48 | self.output(f"As {item['partOfSpeech']}") 49 | self.output(item['definition']) 50 | -------------------------------------------------------------------------------- /maryam/modules/search/sepiasearch.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'sepiasearch', 17 | 'author': 'Rishabh Jain', 18 | 'version': '0.1', 19 | 'description': "Sepia Search is a video search engine optimized for PeerTube channels.", 20 | 'sources': ('sepiasearch',), 21 | 'options': ( 22 | ('query', None, True, 'Query string', '-q', 'store', str), 23 | ('limit', 15, False, 'Max result count (default=15)', '-l', 'store', int), 24 | ), 25 | 'examples': ('sepiasearch -q -l 15 --output',) 26 | } 27 | 28 | def module_api(self): 29 | query = self.options['query'] 30 | limit = self.options['limit'] 31 | run = self.sepiasearch(query, limit) 32 | run.run_crawl() 33 | output = {} 34 | count = run.total_result_found 35 | data = run.collected_data 36 | if count: 37 | output['Total Results'] = count 38 | 39 | output['results'] = [] 40 | 41 | if data: 42 | output['results'].extend(data) 43 | 44 | self.save_gather(output, 'search/sepiasearch', query, output=self.options['output']) 45 | return output 46 | 47 | def module_run(self): 48 | self.alert_results(module_api(self)) -------------------------------------------------------------------------------- /maryam/core/util/search/crt.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q): 21 | """ crt.sh search engine 22 | 23 | q : Query for search 24 | """ 25 | self.framework = main.framework 26 | self.q = q 27 | self._pages = '' 28 | self._json_page = '' 29 | self.crt = f"https://crt.sh/?q={q}&output=json" 30 | 31 | def run_crawl(self): 32 | self.framework.verbose('[CRT] Starting Search...') 33 | try: 34 | req = self.framework.request(self.crt) 35 | except: 36 | self.framework.debug('[CRT] ConnectionError') 37 | self.framework.error('CRT is missed!', 'util/crt', 'run_crawl') 38 | return 39 | self._pages = req.text 40 | self._json_page = req.json() 41 | try: 42 | self._json_page = req.json() 43 | if self._json_page == []: 44 | self._json_page = {} 45 | except: 46 | self.framework.error('CRT is missed!', 'util/crt', 'run_crawl') 47 | self._json_page = [] 48 | 49 | @property 50 | def pages(self): 51 | return self._pages 52 | 53 | @property 54 | def json_page(self): 55 | return self._json_page 56 | 57 | @property 58 | def dns(self): 59 | return self.framework.page_parse(self.pages).get_dns(self.q) 60 | -------------------------------------------------------------------------------- /maryam/modules/search/wikileaks.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Wikileaks Search', 20 | 'author': 'Aman Singh', 21 | 'version': '0.1', 22 | 'description': 'WikiLeaks is an international non-profit organisation that publishes news leaks and classified media provided by anonymous sources.', 23 | 'sources': ('wikileaks',), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ), 28 | 'examples': ('wikileaks -q -l 15 --output',) 29 | } 30 | 31 | def module_api(self): 32 | query = self.options['query'] 33 | limit = self.options['limit'] 34 | run = self.wikileaks(query, limit) 35 | run.run_crawl() 36 | 37 | output = {'results': []} 38 | links = run.links_with_data 39 | 40 | for item in links: 41 | output['results'].append(item) 42 | 43 | self.save_gather(output, 'search/wikileaks', query, output=self.options['output']) 44 | return output 45 | 46 | def module_run(self): 47 | output = module_api(self) 48 | for result in output['results']: 49 | self.output(result['title']) 50 | self.output(f"\t{result['link']}",'G') 51 | 52 | -------------------------------------------------------------------------------- /maryam/modules/search/pastebin.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | 19 | import re 20 | 21 | meta = { 22 | 'name': 'Pastebin Search', 23 | 'author': 'Divya Goswami', 24 | 'version': '1.1', 25 | 'description': 'Search your query in the pastebin.com and show the results as paste links.', 26 | 'sources': ('pastebin',), 27 | 'options': ( 28 | ('query', None, True, 'Query string', '-q', 'store', str), 29 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 30 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 31 | ), 32 | 'examples': ('pastebin -q passwords -l 15 --output',) 33 | } 34 | 35 | 36 | def module_api(self): 37 | query = self.options['query'] 38 | limit = self.options['limit'] 39 | count = self.options['count'] 40 | output = {'pastes': []} 41 | pastebin = self.pastebin(query, limit, count) 42 | pastebin.run_crawl() 43 | output['pastes'] = pastebin.links 44 | self.save_gather(output, 'search/pastebin', query, output=self.options.get('output')) 45 | return output 46 | 47 | def module_run(self): 48 | output = module_api(self) 49 | for i in range(len(output['pastes'])): 50 | self.output(f"{output['pastes'][i]}", 'G') 51 | -------------------------------------------------------------------------------- /maryam/core/util/osint/keyserver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | OWASP Maryam! 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | 19 | class main: 20 | 21 | def __init__(self, q, limit=10): 22 | """ keyserver.ubuntu.com search engine 23 | 24 | q : query for search 25 | limit : Number of pages 26 | """ 27 | self.framework = main.framework 28 | self.q = q 29 | self.limit = limit 30 | self._pages = '' 31 | self._json_pages = '' 32 | self.keyserver_api = f"https://keyserver.ubuntu.com/pks/lookup?search=@{self.q}&op=index" 33 | self.acceptable = False 34 | 35 | def run_crawl(self): 36 | self.framework.verbose('[KEYSERVER] Searching in keyserver...') 37 | try: 38 | req = self.framework.request(self.keyserver_api) 39 | except: 40 | self.framework.debug('ConnectionError', 'util/keyserver', 'run_crawl') 41 | self.framework.error('Keyserver is missed!', 'util/keyserver', 'run_crawl') 42 | return 43 | self._pages += req.text 44 | return self.framework.page_parse(self._pages).get_emails(self.q) 45 | 46 | @property 47 | def pages(self): 48 | return self._pages 49 | 50 | @property 51 | def json_pages(self): 52 | return self._json_pages 53 | 54 | @property 55 | def emails(self): 56 | return self.framework.page_parse(self._pages).get_emails(self.q) 57 | 58 | @property 59 | def dns(self): 60 | return self.framework.page_parse(self.pages).get_dns(self.q) 61 | 62 | -------------------------------------------------------------------------------- /maryam/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __VERSION__ = '2.5.3' 19 | 20 | import sys 21 | 22 | # Import framework 23 | from maryam.core import initial 24 | 25 | def cui(args): 26 | core = initial.initialize 27 | help_menu = 'usage: maryam [-h] [-e execute] [-s start] [-v]'\ 28 | '\nmaryam -h\n'\ 29 | '\noptional arguments:'\ 30 | '\n -h, --help show this help message and exit'\ 31 | '\n -e execute execute a command and exit'\ 32 | '\n -s start run a command without exit'\ 33 | '\n -v show version and exit' 34 | if args: 35 | option = args.pop(0) 36 | mode = 'run' 37 | if option in ['-v', '--version']: 38 | print(f"OWASP Maryam V.{__VERSION__}") 39 | exit(0) 40 | if option == '-e': 41 | core = core('execute') 42 | core.onecmd(' '.join(args)) 43 | exit(0) 44 | elif option == '-s': 45 | core = core('run') 46 | core.onecmd(' '.join(args)) 47 | elif option == 'run': 48 | core = core('run') 49 | else: 50 | print(help_menu) 51 | exit(0) 52 | else: 53 | core = core('run') 54 | 55 | while True: 56 | try: 57 | core.cmdloop() 58 | break 59 | except KeyboardInterrupt: 60 | print('\n[!] Use exit command to exit') 61 | except Exception as e: 62 | raise e 63 | 64 | if __name__ == '__main__': 65 | args = [f'"{arg}"' if ' ' in arg else arg for arg in sys.argv[1:]] 66 | cui(args) 67 | -------------------------------------------------------------------------------- /maryam/modules/search/crt.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'CRT Search', 17 | 'author': 'Saeed', 18 | 'version': '0.2', 19 | 'description': 'Search your query in the crt.sh and show the results.', 20 | 'sources': ('crt',), 21 | 'options': ( 22 | ('query', None, True, 'Query string', '-q', 'store', str), 23 | ), 24 | 'examples': ('crt -q ',) 25 | } 26 | 27 | def module_api(self): 28 | query = self.options['query'] 29 | run = self.crt(query) 30 | run.run_crawl() 31 | output = run.json_page 32 | self.save_gather({'certificates': output}, 'search/crt', query, output=self.options['output']) 33 | return output 34 | 35 | def module_run(self): 36 | output = module_api(self) 37 | items = [] 38 | for itm in output: 39 | i_d = f"https://crt.sh/?caid={itm.get('id')}" 40 | entry_timestamp = itm.get('entry_timestamp').split('T')[0] 41 | not_before = itm.get('not_before').split('T')[0] 42 | not_after = itm.get('not_after').split('T')[0] 43 | nvalue = itm.get('name_value') 44 | issuer = f"https://crt.sh/?caid={itm.get('issuer_ca_id')}" 45 | if '\n' in nvalue: 46 | for line in nvalue.split('\n'): 47 | items.append((i_d, entry_timestamp, not_before, not_after, line, issuer)) 48 | continue 49 | items.append((i_d, entry_timestamp, not_before, not_after, nvalue, issuer)) 50 | self.table(items, header=['crt.sh ID', 'Logged at', 'Not Before', 'Not After', 'Maching Identities', 'Issuer Name'], linear=True, sep='_') 51 | -------------------------------------------------------------------------------- /maryam/core/util/osint/bing_mobile_view.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | class main: 21 | 22 | def __init__(self, url): 23 | """ Bing mobile friendly test screenshot 24 | 25 | url : url for screenshot 26 | """ 27 | self.framework = main.framework 28 | self.url = url 29 | self._raw_image_data = [] 30 | 31 | def screenshot(self): 32 | self.framework.verbose('[Bing Mobile View] Fetching mobile view of the URL...') 33 | bing_api_url = 'https://www.bing.com/webmaster/tools/mobile-friendliness-result' 34 | self.framework._global_options['rand_agent'] = True 35 | try: 36 | response = self.framework.request(url=bing_api_url, method='POST', \ 37 | data={'url': self.url, 'retry': '0'}, timeout=40).text # Setting high timeout of as some req take long 38 | except: 39 | self.framework.error('ConnectionError.', 'util/bing_mobile_view', 'screenshot') 40 | return False 41 | else: 42 | 43 | if 'Bing Webmaster services could not be reached' in response : 44 | self.framework.error('Bing Webmaster services could not be reached', 'util/bing_mobile_view', 'screenshot') 45 | return False 46 | 47 | self._raw_image_data = re.findall(r'data\:image.*"', response) # Regex for base64 encoded image 48 | 49 | @property 50 | def raw_image_data(self): 51 | if self._raw_image_data == []: # If no image return empty string else first image 52 | return '' 53 | return self._raw_image_data[0] 54 | 55 | -------------------------------------------------------------------------------- /maryam/modules/iris/iris_cluster.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Iris Cluster', 17 | 'author': 'Shaad', 18 | 'version': '0.1', 19 | 'description': 'Get Iris Search result and clustered results for your query', 20 | 'required': ('kneed', 'mlxtend', 'numpy', 'sklearn'), 21 | 'options': ( 22 | ('query', None, True, 'Query string', '-q', 'store', str), 23 | ), 24 | 'examples': ('iris_cluster -q ') 25 | } 26 | 27 | 28 | def module_api(self): 29 | query = self.options['query'] 30 | output_option_value = self.options['output'] 31 | output = {} 32 | mode = self._mode 33 | # Computing iris search result 34 | self._mode = 'api_mode' 35 | iris_search_result = self.opt_proc('iris', args=f'-q "{query}" --api', output='web_api') 36 | # Computing cluster results 37 | clusterer = self.cluster(iris_search_result) 38 | output = {'results': clusterer.perform_clustering()} 39 | 40 | self._mode = mode 41 | # Resetting options for iris_search_module 42 | self.options = {} 43 | self.options['query'] = query 44 | self.options['output'] = output_option_value 45 | 46 | self.save_gather(output, 'iris/iris_cluster', query, output=self.options['output']) 47 | return output 48 | 49 | def module_run(self): 50 | output = module_api(self)['results'] 51 | 52 | print('\n\nCLUSTER RESULT: ') 53 | for index, title in enumerate(output): 54 | print('\n') 55 | print(f"CLUSTER {index+1}") 56 | print(f"TITLE: {title}") 57 | print(' '+'\n '.join(output[title])) 58 | -------------------------------------------------------------------------------- /maryam/modules/osint/email_pwned.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | import re 16 | from bs4 import BeautifulSoup 17 | 18 | def remove_tags(html_text): 19 | soup = BeautifulSoup(html_text, 'html.parser') 20 | return soup.get_text() 21 | 22 | meta = { 23 | 'name': 'Pwned database search', 24 | 'author': 'Vikas Kundu', 25 | 'version': '1.2', 26 | 'description': 'Search your email for data breaches', 27 | 'comments': ( 28 | 'Using XmlHttp API of haveibeenpwned.com', 29 | 'to get JSON data' 30 | ), 31 | 'sources': ('haveibeenpwned.com',), 32 | 'options': ( 33 | ('email', None, True, 'Email to search for breach', '-e', 'store', str), 34 | ), 35 | 'examples': ('email_pwned -e --output',) 36 | } 37 | 38 | 39 | def scrap(email): 40 | import cloudscraper 41 | url = f"https://eapi.pcloud.com/checkpwned?checkemail={email}" 42 | scraper = cloudscraper.create_scraper() 43 | result = scraper.get(url) 44 | if result.text: 45 | return remove_tags(result.text) 46 | else: 47 | return False 48 | 49 | 50 | def module_api(self): 51 | output = {'content':[]} 52 | email = self.options['email'] 53 | self.verbose('[PAWNED] Searching for pwning...') 54 | pwns = scrap(email) 55 | if pwns: 56 | output['content'] = pwns 57 | else: 58 | output['content'] = 'no breach' 59 | self.save_gather(output, 'osint/email_pwned', email, 60 | output=self.options['output']) 61 | return output 62 | 63 | 64 | def module_run(self): 65 | output = module_api(self) 66 | self.alert_results(output) 67 | -------------------------------------------------------------------------------- /maryam/core/util/search/discord.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q, limit=3): 21 | """ 22 | discord users and servers search 23 | 24 | q : Query for search 25 | limit : Number of pages 26 | """ 27 | 28 | self.framework = main.framework 29 | self.q = q 30 | self._pages = '' 31 | self.limit = limit 32 | self._links = [] 33 | self.disboard = 'https://disboard.org/search?' 34 | self.discordhub = 'https://discordhub.com/user/search?' 35 | 36 | def run_crawl(self): 37 | query = self.q.split('_')[0] 38 | url_s = [f"{self.disboard}keyword={query}&page={i}" for i in range(1, self.limit+1)] 39 | url_u = [f"{self.discordhub}user_search_bar=%23{query}&page={i}" for i in range(1, self.limit+1)] 40 | if self.q.split('_')[1] == 's': 41 | max_attempt = len(url_s) 42 | urls = url_s 43 | if self.q.split('_')[1] == 'u': 44 | max_attempt = len(url_u) 45 | urls = url_u 46 | for url in range(max_attempt): 47 | self.framework.verbose(f"[DISCORD] Searching in {url} page...") 48 | try: 49 | req = self.framework.request(url=urls[url], allow_redirects=True) 50 | except Exception as e: 51 | self.framework.error('ConnectionError', 'util/discord', 'run_crawl') 52 | max_attempt -= 1 53 | if max_attempt == 0: 54 | self.framework.error('Discord is missed!', 'util/discord', 'run_crawl') 55 | break 56 | self._pages += req.text 57 | @property 58 | def pages(self): 59 | return self._pages 60 | -------------------------------------------------------------------------------- /maryam/modules/footprint/entry_points.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import json 19 | 20 | meta = { 21 | 'name': 'Find Web Entry Points', 22 | 'author': 'Saeed', 23 | 'version': '0.1', 24 | 'description': 'Crawl web pages to find entry points(inputs, urls with param).', 25 | 'options': ( 26 | ('domain', None, True, 'Domain string', '-d', 'store', str), 27 | ('debug', False, False, 'debug the scraper', '--debug', 'store_true', bool), 28 | ('limit', 1, False, 'Scraper depth level', '-l', 'store', int), 29 | ('thread', 1, False, 'The number of links that open per round', '-t', 'store', int), 30 | ), 31 | 'examples': ('entry_points -d ', 'entry_points -d --output --debug -l 10 -t 3') 32 | } 33 | 34 | def module_api(self): 35 | domain = self.options['domain'] 36 | run = self.web_scrap(domain, self.options['debug'], self.options['limit'], self.options['thread']) 37 | run.run_crawl() 38 | get = run.query_links 39 | parser = self.page_parse(run.pages) 40 | forms = parser.get_forms 41 | urls = {} 42 | urlib = self.urlib('null') 43 | for link in get: 44 | params = urlib.self_params(link) 45 | urls.update(params) 46 | output = {'params': urls, 'forms': forms} 47 | self.save_gather(output, 'footprint/entry_points', domain, output=self.options['output']) 48 | return output 49 | 50 | def module_run(self): 51 | output = module_api(self) 52 | self.alert('FORMS') 53 | print(json.dumps(output['forms'], indent=4)) 54 | print(json.dumps(output['params'], indent=4)) 55 | -------------------------------------------------------------------------------- /maryam/core/util/search/ask.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q, limit=2): 21 | """ ask.com search engine 22 | 23 | q : Query for search 24 | limit : Number of pages 25 | """ 26 | self.framework = main.framework 27 | self.q = self.framework.urlib(q).quote 28 | self.limit = limit 29 | self._pages = '' 30 | self.ask = 'www.ask.com' 31 | 32 | def run_crawl(self): 33 | urls = [f"https://{self.ask}/web?q={self.q}&page={i}" for i in range(1, self.limit+1)] 34 | max_attempt = len(urls) 35 | for url in range(len(urls)): 36 | self.framework.verbose(f"[ASK] Searching in {url} page...") 37 | try: 38 | req = self.framework.request(url=urls[url]) 39 | except: 40 | self.framework.error('ConnectionError', 'util/ask', 'run_crawl') 41 | max_attempt -= 1 42 | if max_attempt == 0: 43 | self.framework.error('Ask is missed!', 'util/ask', 'run_crawl') 44 | break 45 | else: 46 | page = req.text 47 | if '>Next' not in page: 48 | self._pages += page 49 | break 50 | self._pages += page 51 | 52 | @property 53 | def pages(self): 54 | return self._pages 55 | 56 | @property 57 | def dns(self): 58 | return self.framework.page_parse(self._pages).get_dns(self.q) 59 | 60 | @property 61 | def emails(self): 62 | return self.framework.page_parse(self._pages).get_emails(self.q) 63 | 64 | @property 65 | def docs(self): 66 | return self.framework.page_parse(self._pages).get_docs(self.q) 67 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributions Best Practices 2 | 3 | **Commits** 4 | 5 | * Write clear meaningful git commit messages (Do read https://chris.beams.io/posts/git-commit/) 6 | * Make sure your PR's description contains GitHub's special keyword references that automatically close the related issue when the PR is merged. (More info at https://github.com/blog/1506-closing-issues-via-pull-requests ) 7 | * When you make very very minor changes to a PR of yours (like for example fixing a failing travis build or some small style corrections or minor changes requested by reviewers) make sure you squash your commits afterwards so that you don't have an absurd number of commits for a very small fix. (Learn how to squash at https://davidwalsh.name/squash-commits-git ) 8 | * When you're submitting a PR for a UI-related issue, it would be really awesome if you add a screenshot of your change or a link to a deployment where it can be tested out along with your PR. It makes it very easy for the reviewers and you'll also get reviews quicker. 9 | 10 | **Feature Requests and Bug Reports** 11 | 12 | * When you file a feature request or when you are submitting a bug report to the [issue tracker](https://github.com/saeeddhqan/Maryam/issues), make sure you add steps to reproduce it. Especially if that bug is some weird/rare one. 13 | 14 | **Join the development** 15 | 16 | * Before you join development, please set up the project on your local machine, run it and go through the application completely. Press on any button you can find and see where it leads to. Explore. (Don't worry ... Nothing will happen to the app or to you due to the exploring :wink: Only thing that will happen is, you'll be more familiar with what is where and might even get some cool ideas on how to improve various aspects of the app.) 17 | * If you would like to work on an issue, drop in a comment at the issue. If it is already assigned to someone, but there is no sign of any work being done, please free to drop in a comment so that the issue can be assigned to you if the previous assignee has dropped it entirely. 18 | 19 | Do read the [Open Source Developer Guide](https://github.com/saeeddhqan/maryam/wiki/Development-Guide). -------------------------------------------------------------------------------- /maryam/core/util/engines/baidu.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from re import search 19 | 20 | class main: 21 | 22 | def __init__(self, q, limit=2): 23 | """ baidu.com search engine 24 | 25 | q : Query for search 26 | limit : Number of pages 27 | """ 28 | self.framework = main.framework 29 | self.q = self.framework.urlib(q).quote 30 | self.limit = 25 if limit > 25 else limit 31 | self._pages = '' 32 | self.baidu = 'baidu.com' 33 | 34 | def run_crawl(self): 35 | set_page = lambda x: (x - 1) * 10 + 1 36 | urls = [f"https://{self.baidu}/s?wd={self.q}&oq={self.q}&pn={set_page(i)}&ie=utf-8" for i in range(1, self.limit+1)] 37 | max_attempt = len(urls) 38 | for url in range(len(urls)): 39 | self.framework.verbose(f"[BAIDU] Searching in {url} page...") 40 | try: 41 | req = self.framework.request(url=urls[url]) 42 | except: 43 | self.framework.error('ConnectionError', 'util/baidu', 'run_crawl') 44 | max_attempt -= 1 45 | if max_attempt == 0: 46 | self.framework.error('Baidu is missed!', 'util/baidu', 'run_crawl') 47 | break 48 | else: 49 | self._pages += req.text 50 | 51 | @property 52 | def pages(self): 53 | return self._pages 54 | 55 | @property 56 | def dns(self): 57 | return self.framework.page_parse(self._pages).get_dns(self.q) 58 | 59 | @property 60 | def emails(self): 61 | return self.framework.page_parse(self._pages).get_emails(self.q) 62 | 63 | @property 64 | def docs(self): 65 | return self.framework.page_parse(self._pages).get_docs(self.q) 66 | -------------------------------------------------------------------------------- /maryam/core/util/osint/exalead.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q, limit=1): 21 | """ exalead.com search engine 22 | 23 | q : Query for search 24 | limit : Number of pages 25 | """ 26 | self.framework = main.framework 27 | self.q = q 28 | self._pages = '' 29 | self.exalead = 'www.exalead.com' 30 | self.limit = limit 31 | 32 | def run_crawl(self): 33 | set_page = lambda x: (x - 1) * 10 + 1 34 | urls = [f"https://{self.exalead}/search/web/results/?q={self.q.replace(' ', '%20')}&elements_per_page=50&start_index={set_page(i)}" 35 | for i in range(1, self.limit+1)] 36 | max_attempt = len(urls) 37 | for url in range(max_attempt): 38 | self.framework.debug(f"[EXALEAD] Searching in {url} page...") 39 | try: 40 | req = self.framework.request(url=urls[url]) 41 | except: 42 | self.framework.error('ConnectionError.', 'util/exalead', 'run_crawl') 43 | max_attempt -= 1 44 | if max_attempt == 0: 45 | self.framework.error('Exalead is missed!', 'util/exalead', 'run_crawl') 46 | break 47 | else: 48 | self._pages += req.text 49 | 50 | @property 51 | def pages(self): 52 | return self._pages 53 | 54 | @property 55 | def dns(self): 56 | return self.framework.page_parse(self._pages).get_dns(self.q) 57 | 58 | @property 59 | def emails(self): 60 | return self.framework.page_parse(self._pages).get_emails(self.q) 61 | 62 | @property 63 | def docs(self): 64 | return self.framework.page_parse(self._pages).get_docs(self.q) 65 | -------------------------------------------------------------------------------- /maryam/core/util/search/photon.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | class main: 16 | 17 | def __init__(self, q, limit): 18 | """ 19 | photon uses osm data for locations and reverse geolocation 20 | q : query 21 | limit : max result count 22 | """ 23 | 24 | self.framework = main.framework 25 | self.q = q.split('_')[0] 26 | self.lat = q.split('_')[1] 27 | self.lon = q.split('_')[2] 28 | self._pages = '' 29 | self._json = {} 30 | self.ph = 'https://photon.komoot.io/' 31 | 32 | def run_crawl(self): 33 | urls = [f"{self.ph}api/?",f"{self.ph}reverse?"] 34 | if self.q and self.lat and self.lon: 35 | if self.q == 'q': 36 | url = urls[1] 37 | payloads = {'lat': self.lat, 'lon': self.lon} 38 | else: 39 | url = urls[0] 40 | payloads = {'q': self.q, 'lat': self.lat, 'lon': self.lon} 41 | else: 42 | url = urls[0] 43 | payloads = {'q': self.q} 44 | self.framework.verbose("[PHOTON]Searching in photon.komoot.io...") 45 | try: 46 | req = self.framework.request( 47 | url = url, 48 | params = payloads 49 | ) 50 | result = req.text 51 | result_json = req.json() 52 | except: 53 | self.framework.error('ConnectionError', 'util/photon', 'run_crawl') 54 | else: 55 | if 'message' in req.text: 56 | self.framework.verbose(f"[PHOTON] Search ended with the message: {req.json()['message'].split(',')[0]}") 57 | self._pages = result 58 | self._json = result_json 59 | 60 | @property 61 | def pages(self): 62 | return self._pages 63 | 64 | @property 65 | def json(self): 66 | return self._json -------------------------------------------------------------------------------- /maryam/modules/osint/article_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Article Search', 20 | 'author': 'Kaushik', 21 | 'version': '0.1', 22 | 'description': 'Search for scientific papers and articles from Google Scholar \ 23 | , Arxiv, and Pubmed', 24 | 'sources': ('scholar', 'arxiv', 'pubmed'), 25 | 'options': ( 26 | ('query', None, True, 'Search query for papers or articles', '-q', 'store', str), 27 | ('limit', 15, False, 'Max result count (default=15)', '-l', 'store', int), 28 | ('exclude medical', False, False, 'Exclude medical results', '-e', 'store_true', bool), 29 | ), 30 | 'examples': ('article_search -q ', 'article_search -q --output') 31 | } 32 | 33 | def module_api(self): 34 | q = self.options['query'] 35 | no_medical = self.options['exclude medical'] 36 | limit = self.options['limit']//3 if no_medical else self.options['limit']//4 37 | 38 | scholar = self.scholar(q, limit) 39 | scholar.run_crawl() 40 | results = scholar.results 41 | 42 | # core_ac = self.core_ac(q, limit) 43 | # core_ac.run_crawl() 44 | # results.extend(core_ac.results) 45 | 46 | arxiv = self.arxiv(q, limit) 47 | arxiv.run_crawl() 48 | results.extend(arxiv.results) 49 | 50 | if not no_medical: 51 | pubmed = self.pubmed(q, limit) 52 | pubmed.run_crawl() 53 | results.extend(pubmed.results) 54 | 55 | output = {'results': results} 56 | 57 | self.save_gather(output, 'osint/article_search', q, output=self.options['output']) 58 | return output 59 | 60 | def module_run(self): 61 | self.search_engine_results(module_api(self)) 62 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from setuptools import setup, find_packages 19 | 20 | setup( 21 | name='maryam', 22 | version='2.5.2-2', 23 | url='https://github.com/saeeddhqan/Maryam', 24 | author='Saeed Dehqan', 25 | author_email='saeed.dehghan@owasp.org', 26 | packages=find_packages(), 27 | include_package_data=True, 28 | package_data={"maryam": ['data/*', 'modules/osint/*', 'modules/search/*', 'modules/iris/*', 'modules/footprint/*']}, 29 | license='GPL-V3', 30 | description='OWASP Maryam is a modular/optional open-source framework based on OSINT and data gathering.', 31 | long_description=open('README.md').read(), 32 | long_description_content_type='text/markdown', 33 | keywords=['OWASP', 'OSINT', 'search-engine', 'social-networks', 'Maryam'], 34 | scripts=['bin/maryam'], 35 | install_requires=[ 36 | 'beautifulsoup4', 37 | 'bertopic', 38 | 'certifi', 39 | 'cloudscraper', 40 | 'dask', 41 | 'Flask', 42 | 'huggingface-hub', 43 | 'lxml', 44 | 'matplotlib', 45 | 'numpy', 46 | 'packaging', 47 | 'pandas', 48 | 'requests', 49 | 'scikit-learn', 50 | 'sentence-transformers', 51 | 'top2vec', 52 | 'transformers', 53 | 'typing-extensions', 54 | 'umap', 55 | 'umap-learn', 56 | 'vaderSentiment', 57 | ], 58 | classifiers=[ 59 | 'Programming Language :: Python :: 3.10', 60 | 'Development Status :: 5 - Production/Stable', 61 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 62 | 'Natural Language :: English', 63 | 'Operating System :: POSIX :: Linux', 64 | 'Environment :: Console', 65 | ] 66 | ) 67 | -------------------------------------------------------------------------------- /maryam/core/util/osint/onionland.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | 19 | class main: 20 | 21 | def __init__(self, q, limit): 22 | """ use onionlandsearchengine.com 23 | 24 | q : Query for search 25 | limit : Number of pages 26 | """ 27 | self.framework = main.framework 28 | self.q = self.framework.urlib(q).quote 29 | self.limit = 20 if limit > 20 else limit 30 | self._pages = '' 31 | self._links = [] 32 | self.onionlandsearchengine = 'onionlandsearchengine.com' 33 | 34 | def run_crawl(self): 35 | urls = [f"https://{self.onionlandsearchengine}/search?q={self.q}&page={i}" for i in range(1, self.limit)] 36 | max_attempt = len(urls) 37 | plen = 0 38 | for url in range(max_attempt): 39 | self.framework.debug(f'[ONIONLAND] Searching in {url} page...') 40 | try: 41 | req = self.framework.request(url=urls[url]) 42 | except: 43 | self.framework.print_exception() 44 | max_attempt -= 1 45 | if max_attempt == 0: 46 | self.framework.error('Onionland is missed!', 'util/onionland', 'run_crawl') 47 | break 48 | else: 49 | self._pages += req.text 50 | if plen == 0: 51 | plen = len(self.framework.reglib(self._pages).search('class="page"')) 52 | if plen-1 == url: 53 | return 54 | 55 | @property 56 | def pages(self): 57 | return self._pages 58 | 59 | @property 60 | def links(self): 61 | self._links = [x.replace('\n', '') for x in self.framework.reglib(self._pages).search(r'\n(https?://.*)')] 62 | self._links.extend([x.replace('class="link">\n', '') for x in self.framework.reglib(self._pages).search(r'class="link">\n(https?://.*)')]) 63 | return self._links 64 | -------------------------------------------------------------------------------- /maryam/core/util/search/dictionary.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | import json 18 | 19 | class main: 20 | 21 | def __init__(self, q): 22 | """ Google Dictionary search engine 23 | q : Query to search 24 | """ 25 | self.framework = main.framework 26 | self.q = q 27 | self._json = '' 28 | self.dictionary = 'https://api.dictionaryapi.dev/api/v2/entries/en_US/' 29 | 30 | def run_crawl(self): 31 | self.q = self.framework.urlib(self.q).quote 32 | self.url = f"{self.dictionary}{self.q}" 33 | 34 | self.framework.verbose('Searching Dictionary...') 35 | try: 36 | req = self.framework.request(url=self.url) 37 | self._json = req.json() 38 | except: 39 | self.framework.error('ConnectionError.', 'util/dictionary', 'run_crawl') 40 | self.framework.error('Dictionary is missed!', 'util/dictionary', 'run_crawl') 41 | 42 | @property 43 | def json(self): 44 | return self._json 45 | 46 | @property 47 | def raw(self): 48 | return json.dumps(self._json) 49 | 50 | @property 51 | def definitions(self): 52 | definition_list = [] 53 | 54 | if not (type(self._json) == dict and \ 55 | self._json.get('title') == 'No Definitions Found'): 56 | for context in self._json: 57 | meanings = context['meanings'] 58 | if len(meanings) > 0: 59 | for meaning in meanings: 60 | partOfSpeech = meaning['partOfSpeech'] 61 | definition_sections = meaning['definitions'] 62 | 63 | for section in definition_sections: 64 | definition_list.append({ 65 | 'partOfSpeech': partOfSpeech.title(), 66 | 'definition': section['definition'] 67 | }) 68 | return definition_list 69 | -------------------------------------------------------------------------------- /maryam/core/util/osint/virustotal.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | class main: 21 | 22 | def __init__(self, q, limit=2): 23 | """ netcraft.com for find dns 24 | 25 | q : Query for search 26 | """ 27 | self.framework = main.framework 28 | self.q = q 29 | self.limit = limit 30 | self.base_url = f"https://www.virustotal.com/ui/domains/{q}/subdomains?relationships=resolutions&cursor=STMwCi4=&limit=40" 31 | self._pages = '' 32 | 33 | def request(self, url): 34 | try: 35 | req = self.framework.request(url=url) 36 | except: 37 | self.framework.error('[VIRUSTOTAL] ConnectionError') 38 | self.framework.error('VirusTotal is missed!') 39 | return False 40 | return req 41 | 42 | def run_crawl(self): 43 | count = 0 44 | while True: 45 | self.framework.verbose(f"[VIRUSTOTAL] Searching in {count} page...") 46 | req = self.request(self.base_url) 47 | if not req: 48 | return 49 | self._pages += req.text 50 | json = req.json() 51 | if count+1 == self.limit: 52 | break 53 | if 'links' in json: 54 | if 'next' in json['links']: 55 | self.base_url = json['links']['next'] 56 | else: 57 | return 58 | else: 59 | return 60 | count += 1 61 | 62 | @property 63 | def pages(self): 64 | return self._pages 65 | 66 | @property 67 | def dns(self): 68 | resp = [] 69 | parser = self.framework.page_parse(str(self._pages)).get_dns(self.q) 70 | for host in list(set(parser)): 71 | if host[0].isdigit(): 72 | matches = re.match(r".+([0-9])[^0-9]*$", host) 73 | host = host[matches.start(1) + 1:] 74 | if host.startswith('.'): 75 | host = host[1:] 76 | resp.append(host) 77 | return resp 78 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. OWASP Maryam documentation master file, created by 2 | sphinx-quickstart on Thu May 27 12:13:59 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ************ 7 | OWASP Maryam 8 | ************ 9 | 10 | Open-source intelligence(OSINT) is a method of using open source tools to collect 11 | information and analyze them for a specific purpose. OSINT can be very helpful for 12 | hackers to use to garner data about particular organizations. Today, using 13 | open-sources like bing, google, yahoo, etc.., for data gathering is one of the important 14 | steps for reconnaissance and this is a common task. It should be a tool to automate this 15 | routine. One of the best tools in this field is The OWASP Maryam. 16 | 17 | `github `_ 18 | 19 | Introduction 20 | ############ 21 | 22 | OWASP Maryam is a modular/optional open-source framework based on OSINT and data gathering. Maryam is written in the Python programming language and has been designed to provide a powerful environment to harvest data from open sources and search engines and collect data quickly and thoroughly. 23 | 24 | Getting Started 25 | ############### 26 | 27 | 28 | Install 29 | ####### 30 | 31 | **From pip** 32 | 33 | You can install maryam by pip: 34 | :: 35 | pip install maryam 36 | 37 | **From source** 38 | 39 | The repository can be loaded using the following command: 40 | :: 41 | git clone https://github.com/saeeddhqan/maryam.git 42 | cd maryam 43 | 44 | 45 | The next step is to install the requirements 46 | :: 47 | pip3 install -r requirements 48 | python setup.py install 49 | 50 | The installation is finished and you can run with: 51 | :: 52 | maryam 53 | 54 | 55 | **Update** 56 | 57 | If it already exists, these commands remove the old version completely and replace it 58 | with the new version. From the Maryam install directory: 59 | :: 60 | cd .. 61 | rm -rf maryam 62 | git clone https://github.com/saeeddhqan/maryam.git 63 | cd maryam 64 | python setup.py install 65 | maryam 66 | 67 | And these commands, update the remote URL of the current repository. From the 68 | Maryam install directory: 69 | :: 70 | git remote set-url origin https://github.com/saeeddhqan/maryam.git 71 | git reset --hard HEAD~1 72 | git pull 73 | -------------------------------------------------------------------------------- /maryam/modules/osint/image_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Image Search', 17 | 'author': 'Saeed', 18 | 'version': '0.1', 19 | 'description': 'Search in open-sources to find images.', 20 | 'sources': ('bing', 'google'), 21 | 'options': ( 22 | ('query', None, True, 'Query, host Name, company Name, keyword, , etc', '-q', 'store', str), 23 | ('engines', 'google', True, 'Search engines with comma separator', '-e', 'store', str), 24 | ('thread', 3, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 25 | ), 26 | 'examples': ('image_search -q amazon -e google,bing,qwant',) 27 | } 28 | 29 | IMGS = [] 30 | def remove_dups(self): 31 | urls = [] 32 | new = [] 33 | for i in IMGS: 34 | a = self.urlib(i['i'].lower()).sub_service() 35 | if a not in urls: 36 | urls.append(a) 37 | new.append(i) 38 | return new 39 | 40 | def search(self, name, q): 41 | global IMGS 42 | try: 43 | engine = getattr(self, name + '_images') 44 | q = q 45 | varnames = engine.__init__.__code__.co_varnames 46 | attr = engine(q) 47 | 48 | attr.run_crawl() 49 | IMGS += attr.results 50 | except Exception as e: 51 | print(e) 52 | 53 | def module_api(self): 54 | query = self.options['query'] 55 | engines = self.options['engines'].lower().split(',') 56 | 57 | self.thread(search, self.options['thread'], engines, query, meta['sources']) 58 | INGS = remove_dups(self) 59 | output = {'results' : IMGS} 60 | self.save_gather(output, 'osint/image_search', query, output=self.options['output']) 61 | return output 62 | 63 | def module_run(self): 64 | for i in module_api(self)['results']: 65 | self.output(i['a']) 66 | self.output(i['i']) 67 | self.output(i['t']) 68 | self.output(i['d']) 69 | print() 70 | -------------------------------------------------------------------------------- /maryam/core/util/osint/urlscan.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | def __init__(self, u, limit=100): 20 | """ urlscan.io scanner 21 | 22 | u : The url for scanning 23 | limit : Number of pages 24 | """ 25 | self.framework = main.framework 26 | self.url = u 27 | self.limit = limit 28 | self._ips = [] 29 | self._domains = [] 30 | self.urlscan_api = f"https://urlscan.io/api/v1/search/?q=domain:{self.url}" 31 | 32 | def run_crawl(self): 33 | domains = [] 34 | self.framework.verbose('[URLSCAN] Searching in urlscan.io...') 35 | try: 36 | req = self.framework.request(self.urlscan_api) 37 | result_json = req.json() 38 | except: 39 | self.framework.error('[URLSCAN] Connection Error') 40 | return 41 | else: 42 | if not result_json['total']: 43 | self.framework.verbose('[URLSCAN] Not yet in our database.') 44 | return 45 | elif 'results' not in result_json: 46 | self.framework.error(f"failed with an error: {result_json['description']}", 'util/urlscan', 'run_crawl') 47 | return 48 | else: 49 | for results in result_json['results']: 50 | domains.append(results['task']['domain']) 51 | domains.append(results['page']['domain']) 52 | for i in range(len(domains)): 53 | if self.url in domains[i]: 54 | self._domains.append(domains[i]) 55 | 56 | @property 57 | def dns(self): 58 | return self._domains 59 | 60 | @property 61 | def get_ips(self): 62 | return self._ips -------------------------------------------------------------------------------- /maryam/core/util/search/scholar.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | import lxml 18 | 19 | class main: 20 | 21 | def __init__(self, q, limit=15): 22 | """ scholar.google.com search engine 23 | 24 | q : query for search 25 | limit : maximum result count 26 | """ 27 | self.framework = main.framework 28 | self.q = q 29 | self.max = limit 30 | self._articles = [] 31 | self._results = [] 32 | 33 | def run_crawl(self): 34 | start = 0 35 | self.framework.verbose('Searching the scholar.google.com domain...') 36 | for start in range(0, (self.max//10)+1): 37 | url = f"https://scholar.google.com/scholar" 38 | payload = {'as_vis': '1', 'q': self.q, 'start': start} 39 | try: 40 | req = self.framework.request( 41 | url=url, 42 | params=payload) 43 | except: 44 | self.framework.error('[SCHOLAR] ConnectionError', 'util/search/scholar', 'run_crawl') 45 | self.framework.error('Google Scholar is missed!', 'util/search/scholar', 'run_crawl') 46 | else: 47 | doc = lxml.html.document_fromstring(req.text) 48 | self._articles.extend(doc.find_class('gs_ri')) 49 | start += 10 50 | 51 | @property 52 | def articles(self): 53 | return self._articles 54 | 55 | @property 56 | def results(self): 57 | findlink = lambda x: x.xpath('h3/a')[0].attrib['href'] 58 | findauthors = lambda x: x.find_class('gs_a')[0].text_content() 59 | findtitle = lambda x: x.xpath('h3/a')[0].text_content() 60 | finddesc = lambda x: x.find_class('gs_rs')[0].text_content() 61 | 62 | for count,article in enumerate(self._articles): 63 | if count==self.max: 64 | break 65 | self._results.append({ 66 | 't': findtitle(article), 67 | 'a': findlink(article), 68 | 'c': findauthors(article), 69 | 'd': finddesc(article) 70 | }) 71 | 72 | return self._results 73 | -------------------------------------------------------------------------------- /maryam/modules/search/wikipedia.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | meta = { 17 | 'name': 'Wikipedia Search', 18 | 'author': 'Tarunesh Kumar', 19 | 'version': '0.3', 20 | 'description': 'Search your query in the Wikipedia and show the results.', 21 | 'sources': ('wikipedia',), 22 | 'options': ( 23 | ('query', None, False, 'Query string or wiki page id(e.g, 64959383)', '-q', 'store', str), 24 | ('count', 50, False, 'Number of links per page(min=10, max=100, default=50)', '-c', 'store', int), 25 | ), 26 | 'examples': ('wikipedia -q 64959383',) 27 | } 28 | 29 | def module_api(self): 30 | query = self.options['query'] 31 | count = self.options['count'] 32 | wiki = self.wikipedia(query, count) 33 | target = '' 34 | output = {'titles': [], 'links': [], 'page': {}} 35 | if query.isnumeric() and len(query) > 3: 36 | output['page'] = wiki.page() 37 | else: 38 | wiki.run_crawl() 39 | links_with_title = wiki.links_with_title 40 | for link, title, pid in links_with_title: 41 | output['titles'].append(f"{title}[{pid}]") 42 | output['links'] = wiki.links 43 | 44 | self.save_gather(output, 'search/wikipedia', query, output=self.options.get('output')) 45 | return output 46 | 47 | def module_run(self): 48 | output = module_api(self) 49 | if output['page'] == {}: 50 | self.alert('links') 51 | for i in range(len(output['titles'])): 52 | self.output(output['titles'][i]) 53 | self.output(f"\t{output['links'][i]}", 'G') 54 | return 55 | page = {**output['page']} 56 | target = page.pop('title', self.options['query']) 57 | if 'extract' not in page: 58 | self.output('Not found') 59 | return 60 | description = page.pop('extract') 61 | header = ['Key', 'Value'] 62 | self.table(page.items(), header, title=target) 63 | self.heading('Description') 64 | self.output(self.textwrapping('\t', description), prep='\t') 65 | -------------------------------------------------------------------------------- /maryam/core/util/engines/yandex.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | import webbrowser 20 | 21 | class main: 22 | 23 | def __init__(self, q, limit=2, count=100): 24 | """ yandex.com search engine 25 | 26 | q : query for search 27 | limit : Number of pages 28 | count : Number of links 29 | """ 30 | self.framework = main.framework 31 | self.q = self.framework.urlib(q).quote 32 | self.count = count 33 | self.limit = limit 34 | self._pages = '' 35 | self.yandex = 'yandex.com' 36 | 37 | def run_crawl(self): 38 | urls = [f"https://{self.yandex}/search?text={self.q}&numdoc={self.count}&p={i}" for i in range(1, self.limit+1)] 39 | max_attempt = len(urls) 40 | for url in range(len(urls)): 41 | self.framework.debug(f"[YANDEX] Searching in {url} page...") 42 | try: 43 | req = self.framework.request(url=urls[url], allow_redirects=True) 44 | except: 45 | self.framework.error('ConnectionError', 'util/yandex', 'run_crawl') 46 | max_attempt -= 1 47 | if max_attempt == 0: 48 | self.framework.error('Yandex is missed!', 'util/yandex', 'run_crawl') 49 | break 50 | else: 51 | if 'Oops!' in req.text: 52 | self.framework.error('Yandex CAPTCHA triggered.', 'util/yandex', 'run_crawl') 53 | return 54 | 55 | page = req.text 56 | if ']">next' not in page: 57 | self._pages += page 58 | break 59 | self._pages += page 60 | 61 | @property 62 | def pages(self): 63 | return self._pages 64 | 65 | @property 66 | def dns(self): 67 | return self.framework.page_parse(self._pages).get_dns(self.q) 68 | 69 | @property 70 | def emails(self): 71 | return self.framework.page_parse(self._pages).get_emails(self.q) 72 | 73 | @property 74 | def docs(self): 75 | return self.framework.page_parse(self._pages).get_docs(self.q) 76 | -------------------------------------------------------------------------------- /maryam/core/util/iris/meta_search_util.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | from math import trunc 16 | 17 | class main: 18 | 19 | def __init__(self): 20 | self.framework = main.framework 21 | 22 | 23 | def make_cite(self, url: 'URL string') -> 'cite': 24 | urlib = self.framework.urlib(url) 25 | path = urlib.path 26 | host = f"{urlib.scheme}://{urlib.netloc}" 27 | host_len = len(host) 28 | if path in ('', '/'): 29 | return host 30 | else: 31 | if host_len > 50: 32 | return f"{host} › ..." 33 | if '?' in path: 34 | path = path[:path.rfind('?')] 35 | if '.' in path: 36 | path = path[:path.rfind('.')] 37 | suffix = '' 38 | if path[:22] != urlib.path: 39 | suffix = '...' 40 | path = path[:-1] if path.endswith('/') else path 41 | path = path.replace('/', ' › ')[:22] + suffix 42 | cite = f"{host}{path}" 43 | return cite 44 | 45 | 46 | def remove_dups(self, res): 47 | urls = [] 48 | new = [] 49 | for i in res: 50 | a = self.urlib(i['a'].lower()).sub_service() 51 | if a not in urls: 52 | urls.append(a) 53 | new.append(i) 54 | return new 55 | 56 | 57 | def simple_merge(results) -> 'merging results based on quality of engines': 58 | engines_len = len(results) 59 | merged = [] 60 | 61 | for i in range(len(min(results, key=len))): 62 | for e in range(engines_len): 63 | merged.append(results[e%engines_len].pop(0)) 64 | 65 | 66 | for i in results: 67 | for j in i: 68 | merged.append(j) 69 | 70 | return merged 71 | 72 | 73 | def compute_count_consensus( 74 | e: dict(type=list, help='list of search engines sorted by quality'), 75 | l: dict(type=int, help='number of results'), 76 | ) -> 'a list of numbers': 77 | x = len(e) 78 | o = {} 79 | for i in e: 80 | o[i] = trunc(l / x) 81 | l -= l - (l % x) 82 | if l != 0: 83 | if l < x: 84 | for i in range(l): 85 | o[e[i]] += 1 86 | return o 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://app.travis-ci.com/saeeddhqan/Maryam.svg?branch=master)](https://app.travis-ci.com/github/saeeddhqan/Maryam) 2 | ![Version 2.5.3](https://img.shields.io/badge/Version-2.5.3-green.svg) 3 | ![GPLv3 License](https://img.shields.io/badge/License-GPLv3-green.svg) 4 | ![Python 3.12.x](https://img.shields.io/badge/Python-3.12.x-green.svg) 5 | [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4577/badge)](https://bestpractices.coreinfrastructure.org/projects/4577) 6 | 7 | # OWASP Maryam 8 | 9 | OWASP Maryam is a modular open-source framework based on OSINT and data gathering. It is designed to provide a robust environment to harvest data from open sources and search engines quickly and thoroughly. 10 | 11 | # Installation 12 | 13 | ### Supported OS 14 | - Linux 15 | - FreeBSD 16 | - Darwin 17 | - OSX 18 | 19 | ```bash 20 | $ pip install maryam 21 | ``` 22 | Alternatively, you can install the latest version with the following command (Recommended): 23 | ```bash 24 | pip install git+https://github.com/saeeddhqan/maryam.git 25 | ``` 26 | 27 | # Usage 28 | 29 | ```bash 30 | # Using dns_search. --max means all of resources. --api shows the results as json. 31 | # .. -t means use multi-threading. 32 | maryam -e dns_search -d ibm.com -t 5 --max --api --form 33 | # Using youtube. -q means query 34 | maryam -e youtube -q "" 35 | maryam -e google -q "" 36 | maryam -e dnsbrute -d domain.tld 37 | # Show framework modules 38 | maryam -e show modules 39 | # Set framework options. 40 | maryam -e set proxy .. 41 | maryam -e set agent .. 42 | maryam -e set timeout .. 43 | # Run web API 44 | maryam -e web api 127.0.0.1 1313 45 | ``` 46 | 47 | # Contribution 48 | 49 | Here is a start guide: [Development Guide](https://github.com/saeeddhqan/maryam/wiki/Development-Guide) 50 | You can add a new search engine to the util classes or use the current search engines to write a new module. 51 | The best help to write a new module is checking the current modules. 52 | 53 | # Roadmap 54 | 55 | - Write a language model based search 56 | 57 | # Links 58 | ### [OWASP](https://owasp.org/www-project-maryam/) 59 | ### [Wiki](https://github.com/saeeddhqan/maryam/wiki) 60 | ### [Install](https://github.com/saeeddhqan/maryam/wiki#install) 61 | ### [Modules Guide](https://github.com/saeeddhqan/maryam/wiki/modules) 62 | ### [Development Guide](https://github.com/saeeddhqan/maryam/wiki/Development-Guide) 63 | 64 | To report bugs, requests, or any other issues please [create an issue](https://github.com/saeeddhqan/maryam/issues). 65 | -------------------------------------------------------------------------------- /maryam/modules/search/sanctionsearch.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'SanctionSearch', 20 | 'author': 'Kaushik', 21 | 'version': '0.1', 22 | 'description': 'SanctionSearch is an international list of blacklisted, dangerous\ 23 | personnel and entities.', 24 | 'sources': ('sanctionsearch',), 25 | 'options': ( 26 | ('query', None, False, 'Name to search', '-q', 'store', str), 27 | ('id', None, False, 'Id to search', '-i', 'store', int), 28 | ('limit', 15, False, 'Max result count (default=15)', '-l', 'store', int), 29 | ), 30 | 'examples': ('sanctionsearch -q -l 20', 31 | 'sanctionsearch -i ') 32 | } 33 | 34 | NAMESEARCH = False 35 | 36 | def module_api(self): 37 | global NAMESEARCH 38 | 39 | query = self.options['query'] 40 | limit = self.options['limit'] 41 | _id = self.options['id'] 42 | 43 | if query is None and _id is None: 44 | return {} 45 | elif query is not None and _id is not None: 46 | _id = None 47 | elif query is not None: 48 | NAMESEARCH = True 49 | 50 | run = self.sanctionsearch(query=query, id=_id, limit=limit) 51 | if NAMESEARCH: 52 | output_param = query 53 | run.name_crawl() 54 | output = {'results': []} 55 | data = run.data 56 | for item in data: 57 | output['results'].append(item) 58 | else: 59 | output_param = _id 60 | run.id_crawl() 61 | output = run.data 62 | self.save_gather(output, 'search/sanctionsearch', output_param, output=self.options['output']) 63 | return output 64 | 65 | def module_run(self): 66 | output = module_api(self) 67 | if NAMESEARCH: 68 | output = output['results'] 69 | for name in output: 70 | print() 71 | self.output(name['name']) 72 | if len(name['address'].strip()) > 0: 73 | self.output(name['address']) 74 | self.output(name['link']) 75 | else: 76 | if output is not None: 77 | self.alert_results(output) 78 | -------------------------------------------------------------------------------- /maryam/core/util/iris/tf_histogram.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | import re 16 | import os 17 | from collections import Counter 18 | BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../')) 19 | 20 | 21 | class main: 22 | 23 | def __init__(self, docs: 'documents', form: 'documet form. e.g html', without_punc=True): 24 | """ show histogram plot of web term frequency """ 25 | self.framework = main.framework 26 | self.form = form.lower() 27 | self.docs = docs.lower() 28 | self.without_punc = without_punc 29 | if self.form == 'html': 30 | pp = self.framework.page_parse(self.docs) 31 | pp.remove_html_tags 32 | self.docs = pp.page 33 | self.words = self.docs.split() 34 | if self.without_punc: 35 | self._punc() 36 | 37 | 38 | def remove_stopwords(self, rest): 39 | stops = open(os.path.join(BASEDIR, '../../', 'data', 'stopwords.csv')).read().split(',') 40 | self.words = [x for x in self.words if x not in stops and x not in rest] 41 | 42 | 43 | def _punc(self): 44 | self.words = re.findall(r"[\w\-_#]{2,}", self.docs) 45 | 46 | 47 | def _counter(self, last): 48 | """ last: number of terms to show in plot """ 49 | bow = Counter(self.words) 50 | return bow.most_common(last) 51 | 52 | 53 | def plot_histogram(self, title, last, should_show=False): 54 | import pandas as pd 55 | import matplotlib.pyplot as plt 56 | 57 | most_common = self._counter(last) 58 | clean_tweets_no_urls = pd.DataFrame(most_common, 59 | columns=['words', 'count']) 60 | fig, ax = plt.subplots(figsize=(25, 25)) 61 | clean_tweets_no_urls.sort_values(by='count').plot.barh(x='words', 62 | y='count', 63 | ax=ax, 64 | color="black") 65 | ax.set_title(title) 66 | 67 | filename = os.path.join(self.framework.workspace,title.replace(' ','_')+'.png') 68 | plt.savefig(filename, format="png") 69 | 70 | if should_show: 71 | plt.show() 72 | 73 | return filename 74 | -------------------------------------------------------------------------------- /maryam/core/util/osint/numverify.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import random 19 | 20 | class main: 21 | 22 | def __init__(self, num): 23 | """ numverify search engine 24 | num : number to search 25 | country : country code 26 | 27 | """ 28 | self.framework = main.framework 29 | self.num = num 30 | self._json = '' 31 | self.numverify = 'http://apilayer.net/api/validate' 32 | self.keys = [ 33 | '3489ab1ab092628f8b2a63b3d94b75f3', 34 | 'ea52c77e9877dd4fdb83bdec925d3eeb', 35 | 'e6f3944b3724f836871a4f9dcc546eb6', 36 | '3796398e6c96912c29a56ec0177b8d6b', 37 | '95f0bc6561937986c3386b1ffcd36ae9', 38 | 'd49245f8853efbff4dfd245387531727', 39 | '62552e99ed3cb92159be4153242b7eea', 40 | '1322a4a256acd589e2e3ce73e65fdd3f', 41 | '3fcf94548f2177e781eff270d43790f6', 42 | '7778341260206c06fac4a2727a27955f', 43 | 'a0bfc16589d513fc472832935c9c6528', 44 | '3618d844c1b6802106a6309c92c71ac2', 45 | '376bbc8ef5ee4a9d8f6a81a7ddd9f4cd', 46 | 'd4e8383a4f2efec6e523faad8797c4d1', 47 | '0a45e3f8273a7f929f4ec237e4f24d10', 48 | '64edafd9cc5d839d8d9bac9ac0837851', 49 | '5165b0fec39bb03e42f43d4cc5513dcb', 50 | 'e2a851b9b76a579c0fef4304969546e7', 51 | '9a39c3058615ec330c7223a596f62934', 52 | '2a06b63cb88a13ebeea7e5f83f01f8d7', 53 | '1ed52249226585cdb176926e34d75aa5', 54 | '8a6cb00cac3c904dd1c82b29ab5061d0', 55 | 'fe11bd5cfea457f9fda5ed3fecc8fc54' 56 | ] 57 | 58 | def run_crawl(self): 59 | self.framework.verbose('Searching...') 60 | try: 61 | payload = { 62 | 'access_key': random.choice(self.keys), 63 | 'number': self.num, 64 | 'format': 1 65 | } 66 | req = self.framework.request( 67 | url=self.numverify, 68 | params=payload) 69 | self._json = req.json() 70 | except Exception as e: 71 | self.framework.error(f"ConnectionError {e}.", 'util/numverify', 'run_crawl') 72 | self.framework.error('Numverify is missed!', 'util/numverify', 'run_crawl') 73 | return 74 | 75 | @property 76 | def json(self): 77 | return self._json 78 | -------------------------------------------------------------------------------- /maryam/core/util/search/pubmed.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import lxml 19 | 20 | class main: 21 | 22 | def __init__(self, q, count=15): 23 | """ pubmed.ncbi.nlm.nih.gov search engine 24 | 25 | q : query for search 26 | count : maximum result count 27 | """ 28 | self.framework = main.framework 29 | self.q = q 30 | self.max = count 31 | self._rawhtml = '' 32 | self._results = [] 33 | self.pubmed = 'https://pubmed.ncbi.nlm.nih.gov' 34 | self.xpath_names = { 35 | 'results': './/article', 36 | 'results_content':'.//div[@class="full-view-snippet"]', 37 | 'results_title':'.//a[@class="docsum-title"]', 38 | 'results_a': './/a[@class="docsum-title"]', 39 | 'results_cite': './/span[@class="docsum-authors full-authors"]' 40 | } 41 | self.xpaths = { 42 | self.xpath_names['results']: [ 43 | self.xpath_names['results_content'], 44 | self.xpath_names['results_title'], 45 | self.xpath_names['results_a'], 46 | self.xpath_names['results_cite'] 47 | ] 48 | } 49 | 50 | def run_crawl(self): 51 | self.framework.verbose('Searching the pubmed domain...') 52 | 53 | payload = {'term': self.q, 'size': 200} 54 | try: 55 | req = self.framework.request( 56 | url=self.pubmed, 57 | params=payload) 58 | except Exception as e: 59 | self.framework.error(f"ConnectionError {e}.", 'util/pubmed', 'run_crawl') 60 | self.framework.error('Pubmed is missed!', 'util/pubmed', 'run_crawl') 61 | return 62 | 63 | self._rawhtml += req.text 64 | 65 | @property 66 | def raw(self): 67 | return self._rawhtml 68 | 69 | @property 70 | def results(self): 71 | parser = self.framework.page_parse(self._rawhtml) 72 | results = parser.get_engine_results(self.xpaths, self.xpath_names) 73 | 74 | for count,article in enumerate(results): 75 | if count == self.max: 76 | break 77 | self._results.append({ 78 | 't': article['t'], 79 | 'a': self.pubmed + article['a'], 80 | 'c': article['c'], 81 | 'd': article['d'] if len(article['d']) > 0 else 'No abstract available' 82 | }) 83 | 84 | return self._results 85 | -------------------------------------------------------------------------------- /maryam/core/util/search/sepiasearch.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | 16 | class main : 17 | def __init__(self, q, limit=15): 18 | """ sepiasearch.org search 19 | 20 | q : The query for search 21 | limit : The number of details min 15 if exist 22 | """ 23 | self.q = q 24 | self.limit = limit 25 | self.url = 'https://sepiasearch.org/api/v1/search/videos?' 26 | self.params = {'search': self.q, 'boostLanguages[]': 'en', 'nsfw': 'false', 'start': 0, 'count': self.limit, 'sort': '-match'} 27 | self.headers = {'Content-Type': 'application/x-www-form-urlencoded'} 28 | self._collected_data = [] 29 | self._result_found = 0 30 | 31 | def run_crawl(self): 32 | self.framework.verbose('[SEPIASEARCH] Extracting Data From API') 33 | response = self.framework.request(self.url, params=self.params, headers=self.headers) 34 | if response.status_code != 200: 35 | self.framework.error('Request Fail', 'util/sepiasearch', 'run_crawl') 36 | return 37 | 38 | response = response.json() 39 | self._result_found = response['total'] 40 | for result in response['data']: 41 | self._collected_data.append({ 42 | 'Title': result['name'], 43 | 'Author': result['account']['displayName'], 44 | 'URL': result['url'], 45 | 'Length': self.sec_to_hours(int(result.get('duration'))), 46 | 'Thumbnail': result['thumbnailUrl'], 47 | 'Embeded URL': result.get('embedUrl'), 48 | 'Published Date': result['publishedAt'], 49 | }) 50 | 51 | def sec_to_hours(self, minute): 52 | """Convert minutes in hours""" 53 | if isinstance(minute, int): 54 | minu, sec = divmod(minute, 60) 55 | return "{}:{}:{} hours".format(*divmod(minu, 60), sec) 56 | return None 57 | 58 | @property 59 | def collected_data(self): 60 | return self._collected_data 61 | 62 | @property 63 | def total_result_found(self): 64 | return self._result_found -------------------------------------------------------------------------------- /maryam/modules/iris/iris.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | import concurrent.futures 16 | 17 | meta = { 18 | 'name': 'Iris Meta Search Engine(experiment version)', 19 | 'author': 'Saeed, Kaushik', 20 | 'version': '0.5', 21 | 'description': 'Iris is a built-in meta search engine.', 22 | 'comments': ('It should be note that this is a beta version and has many bugs!',), 23 | 'contributors': 'Aman, Dimitris, Divya, Vikas, Kunal', 24 | 'sources': ('google', 'bing', 'duckduckgo', 'etools', 'startpage', 'yahoo'), 25 | 'options': ( 26 | ('query', None, True, 'Query string', '-q', 'store', str), 27 | ), 28 | 'examples': ('iris -q ',) 29 | } 30 | 31 | RESULTS = {} 32 | MAPPED = {'google': 100, 'bing': 100, 'duckduckgo': 30, 33 | 'yahoo': 100, 'carrot2': 100, 'searx': 100, 'startpage': 30} 34 | 35 | def thread(self, function, query, limit, workers): 36 | threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=len(workers)) 37 | futures = (threadpool.submit(function, self, x, query, limit) for x in workers) 38 | for _ in concurrent.futures.as_completed(futures): 39 | pass 40 | 41 | def search(self, name, q, limit): 42 | global RESULTS 43 | count = MAPPED[name] 44 | try: 45 | engine = getattr(self, name) 46 | q = q 47 | varnames = engine.__init__.__code__.co_varnames 48 | if 'limit' in varnames and 'count' in varnames: 49 | attr = engine(q, limit, count) 50 | elif 'limit' in varnames: 51 | attr = engine(q, limit) 52 | else: 53 | attr = engine(q) 54 | attr.run_crawl() 55 | RESULTS[name] = attr.results 56 | except Exception as e: 57 | print(e) 58 | 59 | def module_api(self): 60 | global RESULTS 61 | query = self.options['query'] 62 | engines = MAPPED.keys() 63 | thread(self, search, query, 3, engines) 64 | simple_merge = self.meta_search_util.simple_merge([RESULTS[x] for x in engines if x in RESULTS]) 65 | # TODO: Removing duplicates need to be done during merging 66 | final_results = self.meta_search_util.remove_dups(self, simple_merge) 67 | output = {'results': final_results} 68 | self.save_gather(output, 'iris/iris', query, output=self.options['output']) 69 | return output 70 | 71 | def module_run(self): 72 | print() 73 | self.search_engine_results(module_api(self)) 74 | -------------------------------------------------------------------------------- /maryam/modules/osint/crawler.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Web Crawler', 20 | 'author': 'Saeed', 21 | 'version': '0.5', 22 | 'description': 'Crawl web pages to find links, JS Files, CSS files, Comments\ 23 | and everything else interesting, supports concurrency.', 24 | 'comments': ("Try to use --output in order to save results thoroughly. Your terminal may not show all the results",), 25 | 'options': ( 26 | ('domain', None, True, 'Domain string', '-d', 'store', str), 27 | ('debug', False, False, 'Debug the scraper', '--debug', 'store_true', bool), 28 | ('wapps', False, False, 'Using wapps module for identifying web apps', '--wapps', 'store_true', bool), 29 | ('limit', 1, False, 'Scraper depth level', '-l', 'store', int), 30 | ('thread', 1, False, 'The number of links that open per round', '-t', 'store', int), 31 | ), 32 | 'examples': ('crawler -d ', 33 | 'crawler -d -l 10 -t 3 --output --debug') 34 | } 35 | 36 | def module_api(self): 37 | domain = self.options['domain'] 38 | run = self.web_scrap(domain, self.options['debug'], self.options['limit'], self.options['thread']) 39 | run.run_crawl() 40 | output = {'js': run.js, 'cdn': run.cdn, 41 | 'getlinks': run.query_links, 'exlinks': run.external_links, 42 | 'links': run.links, 'css': run.css, 'comments': run.comments, 43 | 'emails': run.emails, 'phones': run.phones, 'media': run.media} 44 | output['usernames'] = run.networks 45 | if self.options['wapps']: 46 | output['wapps'] = self._loaded_modules['wapps'].module_api(self) 47 | self.save_gather(output, 'osint/crawler', domain, output=self.options['output']) 48 | return output 49 | 50 | def module_run(self): 51 | output = module_api(self) 52 | if 'wapps' in output: 53 | self.alert_results(output.pop('wapps')) 54 | 55 | for obj in output: 56 | self.alert(f"{obj}({len(output[obj])})") 57 | if output[obj] == []: 58 | self.output('\t..') 59 | else: 60 | for obj_i in output[obj]: 61 | if obj == 'usernames': 62 | if output[obj][obj_i] != []: 63 | self.output(f"\t{obj_i}") 64 | for user in output[obj][obj_i]: 65 | self.output(f"\t\t{user}", 'G') 66 | else: 67 | self.output(f"\t{obj_i}", 'G') 68 | -------------------------------------------------------------------------------- /maryam/core/util/engines/bing_images.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | from lxml import html 16 | from json import loads 17 | 18 | class main: 19 | def __init__(self, q, limit=3, count=28): 20 | """ bing.com search engine 21 | q : Query for search 22 | limit : Number of pages 23 | count : Number of results 24 | """ 25 | self.framework = main.framework 26 | self.q = q 27 | self.agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0' 28 | self.url = 'https://www.bing.com/images/search' 29 | self._pages = '' 30 | self.count = 28 if count > 28 else count 31 | self.limit = 10 if limit > 10 else limit 32 | 33 | def run_crawl(self): 34 | set_page = lambda x: (x*self.count)+1 35 | payload = {'count': self.count, 'first': 1, 'form': 'IBASEP', 'q': self.q} 36 | max_attempt = 0 37 | for i in range(self.limit): 38 | self.framework.verbose(f"[BING Image] Searching in {i} page...", end='\r') 39 | try: 40 | req = self.framework.request( 41 | url=self.url, 42 | params=payload, 43 | headers={'user-agent': self.agent}, 44 | allow_redirects=True) 45 | except Exception as e: 46 | self.framework.error(f"ConnectionError: {e}", 'util/engines/bing_images', 'run_crawl') 47 | max_attempt += 1 48 | if max_attempt == self.limit: 49 | self.framework.error('Bing is missed!', 'util/engines/bing_images', 'run_crawl') 50 | break 51 | else: 52 | self._pages += req.text 53 | payload['first'] = set_page(i+1) 54 | 55 | @property 56 | def results(self): 57 | items = [] 58 | tree = html.fromstring(self._pages) 59 | for item in tree.xpath('//div[@class="imgpt"]'): 60 | try: 61 | info = item.xpath('.//div[@class="img_info hon"]')[0].xpath("./span[@class='nowrap']")[0].text 62 | m = loads(item.xpath('./a/@m')[0]) 63 | title = m.get('t', '').replace(u'\ue000', '').replace(u'\ue001', '') 64 | items.append({'i': m['murl'], 65 | 't': title, 66 | 'a': m['purl'], 67 | 'd': info 68 | }) 69 | except: 70 | continue 71 | return items 72 | 73 | @property 74 | def pages(self): 75 | return self._pages 76 | 77 | @property 78 | def links(self): 79 | return [x['a'] for x in self.results] 80 | 81 | -------------------------------------------------------------------------------- /maryam/modules/search/photon.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Photon Search', 20 | 'author': 'Divya Goswami', 21 | 'version': '1.0', 22 | 'description': 'Search-as-you-type and typo tolerant geolocator. Works as reverse geocoder too.', 23 | 'sources': ('photon.komoot.io',), 24 | 'options': ( 25 | ('query', '', False, 'Enter a location name', '-q', 'store', str), 26 | ('latitude', '', False, 'Enter latitude', '-lat', 'store', str), 27 | ('longitude', '', False, 'Enter longitude', '-lon', 'store', str), 28 | ('limit', 3, False, 'Search limit(number of results, default=not set)', '-l', 'store', int), 29 | ('engine', 'photon', False, 'Engine used is photon', '-e', 'store', str), 30 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 31 | ), 32 | 'examples': ('photon -q berlin --output', 33 | 'photon -q berlin -lat 52 -lon 13', 34 | 'photon -lat 10 -lon 52' 35 | ) 36 | } 37 | 38 | def module_api(self): 39 | global PAGES, JSON 40 | query = self.options['query'] 41 | lat = self.options['latitude'] 42 | lon = self.options['longitude'] 43 | limit = self.options['limit'] 44 | engine = self.options['engine'] 45 | output = {} 46 | q_formats = { 47 | 'detail': f"{query}_{lat}_{lon}", 48 | 'reverse': f"q_{lat}_{lon}" 49 | } 50 | if query == '': 51 | query = q_formats['reverse'] 52 | else: 53 | query = q_formats['detail'] 54 | run = self.photon(query, limit) 55 | run.run_crawl() 56 | PAGES = run.pages 57 | JSON = run.json 58 | output = JSON 59 | self.save_gather(output, 'search/photon', query, output=self.options.get('output')) 60 | return output 61 | 62 | def module_run(self): 63 | output = module_api(self)['features'] 64 | final = [] 65 | for item in range(len(output)): 66 | self.output(output[item]['properties']['type'].title()) 67 | self.output(f"Coordinates: {output[item]['geometry']['coordinates']}") 68 | properties = output[item]['properties'] 69 | prop_k = list(properties.keys()) 70 | for x in prop_k: 71 | if x not in ['osm_id','osm_type','osm_key','osm_value','countrycode','type','extent','postcode']: 72 | final.append(properties[x]) 73 | self.output(', '.join(set(final))) 74 | final = [] 75 | print() 76 | -------------------------------------------------------------------------------- /maryam/core/util/osint/darksearch.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | 21 | class main: 22 | 23 | def __init__(self, q, limit=1): 24 | """ 25 | darksearch.io search engine 26 | 27 | q : Query for search 28 | limit : Number of pages 29 | """ 30 | 31 | self.framework = main.framework 32 | self.q = q 33 | self._pages = '' 34 | self.limit = limit 35 | self._links = [] 36 | self._json = [] 37 | self.darksearch = 'darksearch.io' 38 | self._json_links = [] 39 | 40 | def run_crawl(self): 41 | urls = [f"https://{self.darksearch}/api/search?query={self.q}&page={i}" for i in range(1, self.limit+1)] 42 | max_attempt = len(urls) 43 | for url in range(max_attempt): 44 | self.framework.verbose(f"[DARKSEARCH] Searching in {url} page...") 45 | try: 46 | req = self.framework.request(url=urls[url], allow_redirects=True) 47 | except Exception as e: 48 | self.framework.error('ConnectionError', 'util/darksearch', 'run_crawl') 49 | max_attempt -= 1 50 | if max_attempt == 0: 51 | self.framework.error('Darksearch is missed!', 'util/darksearch', 'run_crawl') 52 | break 53 | 54 | if req.json()['data'] is None: 55 | break 56 | self._pages += req.text 57 | self._json.append(req.json()) 58 | 59 | @property 60 | def pages(self): 61 | return self._pages 62 | 63 | @property 64 | def json(self): 65 | return self._json 66 | 67 | @property 68 | def links(self): 69 | for page in self.json: 70 | results = page.get('data', {}) 71 | if not results: 72 | return {} 73 | self._links.extend(x.get('link') for x in results) 74 | return self._links 75 | 76 | @property 77 | def json_links(self): 78 | for page in self.json: 79 | results = page.get('data', {}) 80 | if not results: 81 | return {} 82 | self._json_links.extend(results) 83 | return self._json_links 84 | -------------------------------------------------------------------------------- /maryam/core/util/osint/hunter.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q, key, limit=10): 21 | """ hunter.io search engine 22 | 23 | q : query for search 24 | key : hunter.io api key 25 | limit : Number of pages 26 | """ 27 | self.framework = main.framework 28 | self.q = q 29 | self.limit = limit 30 | self.key = key 31 | self._pages = '' 32 | self._json_pages = '' 33 | self.hunter_api = f"https://api.hunter.io/v2/domain-search?domain={self.q}&api_key={self.key}" 34 | self.acceptable = False 35 | def run_crawl(self): 36 | self.framework.verbose('[HUNTER] Searching in hunter...') 37 | try: 38 | req = self.framework.request(self.hunter_api) 39 | except: 40 | self.framework.debug('ConnectionError', 'util/hunter', 'run_crawl') 41 | self.framework.error('Hunter is missed!', 'util/hunter', 'run_crawl') 42 | return 43 | self._pages = req.text 44 | self._json_pages = req.json() 45 | 46 | # Key validation 47 | if 'errors' in self._json_pages: 48 | code = self._json_pages.get('errors')[0]['details'] 49 | self.framework.error(f"failed with error: {code}", 'util/hunter', 'run_crawl') 50 | self.acceptable = False 51 | return 52 | 53 | # Request validation 54 | if not self._json_pages.get('data').get('emails'): 55 | if self._json_pages.get('meta').get('results') == 0: 56 | self.framework.verbose('[HUNTER] no results found') 57 | else: 58 | self.framework.verbose('[HUNTER] request was not accepted!') 59 | else: 60 | self.acceptable = True 61 | 62 | @property 63 | def pages(self): 64 | return self._pages 65 | 66 | @property 67 | def json_pages(self): 68 | return self._json_pages 69 | 70 | @property 71 | def emails(self): 72 | return self.framework.page_parse(self._pages).get_emails(self.q) 73 | 74 | @property 75 | def json_emails(self): 76 | emails = [] 77 | if self.acceptable: 78 | for x in range(self.limit): 79 | emails.append(self._json_pages['data']['emails'][x]['value']) 80 | return emails 81 | return [] 82 | 83 | @property 84 | def dns(self): 85 | return self.framework.page_parse(self.pages).get_dns(self.q) 86 | -------------------------------------------------------------------------------- /maryam/core/util/search/openstreetmap.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | class main: 16 | 17 | def __init__(self, q): 18 | """ 19 | openstreetmap search engine for places 20 | q : query 21 | limit : max result count 22 | """ 23 | 24 | self.framework = main.framework 25 | self.q = self.framework.urlib(q).quote 26 | self._pages = '' 27 | self.xpath_name = { 28 | 'results': '//li[@class="list-group-item search_results_entry"]', 29 | 'results_a': './a', 30 | } 31 | self.xpath = { 32 | self.xpath_name['results']: [ 33 | self.xpath_name['results_a'] 34 | ] 35 | } 36 | 37 | def run_crawl(self): 38 | urls = [f"https://www.openstreetmap.org/geocoder/search_geonames?query={self.q}", 39 | f"https://www.openstreetmap.org/geocoder/search_osm_nominatim?query={self.q}"] 40 | self.framework.verbose('Searching openstreetmap...') 41 | 42 | for url in urls: 43 | try: 44 | req = self.framework.request(url=url) 45 | except Exception as e: 46 | self.framework.error(f"ConnectionError {e}", 'util/openstreetmap', 'run_crawl') 47 | self.framework.error('Openstreetmap is missed!', 'util/openstreetmap', 'run_crawl') 48 | return 49 | else: 50 | self._pages += req.text 51 | 52 | @property 53 | def pages(self): 54 | return self._pages 55 | 56 | @property 57 | def results(self): 58 | results = [] 59 | parser = self.framework.page_parse(self._pages) 60 | xpath_results = parser.html_fromstring(self.xpath) 61 | if not xpath_results: 62 | return results 63 | root = xpath_results[self.xpath_name['results']] 64 | for i in range(len(root[self.xpath_name['results_a']])): 65 | if root[self.xpath_name['results_a']][i].get('data-prefix') is not None: 66 | category = root[self.xpath_name['results_a']][i].get('data-prefix') 67 | else: 68 | category = '' 69 | link = f"https://www.openstreetmap.org{root[self.xpath_name['results_a']][i].get('href')}" 70 | result = { 71 | 'category': category, 72 | 'location': root[self.xpath_name['results_a']][i].get('data-name'), 73 | 'latitude': root[self.xpath_name['results_a']][i].get('data-lat'), 74 | 'longitude': root[self.xpath_name['results_a']][i].get('data-lon'), 75 | 'link': link 76 | } 77 | results.append(result) 78 | 79 | return results 80 | -------------------------------------------------------------------------------- /maryam/core/util/osint/netcraft.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from hashlib import sha1 19 | import re 20 | 21 | class main: 22 | 23 | def __init__(self, q, limit=4): 24 | """ netcraft.com for find dns 25 | 26 | q : Query for search 27 | """ 28 | self.framework = main.framework 29 | self.q = q 30 | self.limit = limit 31 | self.base_url = f"https://searchdns.netcraft.com/?restriction=site+ends+with&host={q}" 32 | self._pages = '' 33 | 34 | def request(self, url, cookies=None): 35 | cookies = cookies or {} 36 | try: 37 | req = self.framework.request(url=url, cookies=cookies) 38 | except Exception as e: 39 | self.framework.error(f"ConnectionError {e}.", 'util/netcraft', 'request') 40 | self.framework.error('Netcraft is missed!', 'util/netcraft', 'request') 41 | return False 42 | return req 43 | 44 | def get_next(self, resp): 45 | link_regx = re.compile(r'Next Page . 16 | """ 17 | 18 | import re 19 | 20 | meta = { 21 | 'name': 'Discord Search', 22 | 'author': 'Divya Goswami', 23 | 'version': '1.0', 24 | 'description': 'Search the name of any user or server on discord [from disboard and discordhub]', 25 | 'sources': ('discord',), 26 | 'options': ( 27 | ('user', None, False, 'User flag (search for a user based on the integer hash)', '-u', 'store', str), 28 | ('server', None, False, 'Server flag (search for a server)', '-s', 'store', str), 29 | ('limit', 3, False, 'Search limit(number of pages, default=3)', '-l', 'store', int), 30 | ('engine', 'discord', False, 'Engine used is discord', '-e', 'store', str), 31 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 32 | ), 33 | 'examples': ('discord -u --output', 'discord -s -l 5') 34 | } 35 | 36 | PAGES = '' 37 | 38 | def search(self, name, q, q_formats, limit, count): 39 | global PAGES, LINKS 40 | engine = getattr(self, name) 41 | varnames = engine.__init__.__code__.co_varnames 42 | if 'limit' in varnames: 43 | attr = engine(q, limit) 44 | attr.run_crawl() 45 | PAGES += attr.pages 46 | 47 | def module_api(self): 48 | global PAGES 49 | user = self.options['user'] 50 | server = self.options['server'] 51 | limit = self.options['limit'] 52 | engine = ['discord'] 53 | output = { 'servers': [], 'users': [] } 54 | if user and server: 55 | query = f"{user}_u" 56 | elif user: 57 | query = f"{user}_u" 58 | elif server: 59 | query = f"{server}_s" 60 | else: 61 | return output 62 | u_list = [] 63 | s_list = [] 64 | self.thread(search, self.options['thread'], engine, query, {}, limit, 0, meta['sources']) 65 | user_reg = r'

(.+)<\/a' 66 | server_reg = r'. 16 | """ 17 | 18 | import re 19 | 20 | class main: 21 | 22 | def __init__(self, q, limit=1): 23 | """ wikileaks.org search engine 24 | 25 | q : Query for search 26 | limit : Number of Pages 27 | 28 | """ 29 | 30 | self.framework = main.framework 31 | self.q = q 32 | self.agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0' 33 | self._pages = '' 34 | self.limit = limit 35 | self._links = [] 36 | self._results = [] 37 | self._links_with_data = [] 38 | 39 | def run_crawl(self): 40 | from bs4 import BeautifulSoup as bs 41 | 42 | max_attempts = 0 43 | url = 'https://search.wikileaks.org/advanced' 44 | params = {'query': self.q, 'include_external_sources': True, 'page': 1} 45 | while True: 46 | self.framework.verbose(f"[WIKILEAKS] Searching in {params['page']} page ....", end='\r') 47 | try: 48 | req = self.framework.request( 49 | url=url, 50 | params=params, 51 | headers={'User-Agent': self.agent}, 52 | allow_redirects=True) 53 | except Exception as e: 54 | self.framework.error(f"ConnectionError: {e}", 'util/wikileaks', 'run_crawl') 55 | max_attempts += 1 56 | if max_attempts == self.limit: 57 | self.framework.error('Wikileaks is missed!', 'util/wikileaks', 'run_crawl') 58 | return 59 | else: 60 | self._pages += req.text 61 | params['page'] += 1 62 | if params['page'] >= self.limit: 63 | break 64 | 65 | soup = bs(self._pages, 'html.parser') 66 | self._results = soup.find_all('div', class_='info') 67 | for result in self._results: 68 | self._links.append(result.find_all('a')[0]['href']) 69 | 70 | @property 71 | def pages(self): 72 | return self._pages 73 | 74 | @property 75 | def links(self): 76 | return self._links 77 | 78 | @property 79 | def links_with_data(self): 80 | for result in self._results: 81 | self._links_with_data.append({ 82 | 'title': result.find_all('a')[0].text, 83 | 'link': result.find_all('a')[0]['href'] 84 | }) 85 | 86 | return self._links_with_data 87 | 88 | @property 89 | def dns(self): 90 | return self.framework.page_parse(self._pages).get_dns(self.q) 91 | 92 | @property 93 | def emails(self): 94 | return self.framework.page_parse(self._pages).get_emails(self.q) 95 | 96 | @property 97 | def docs(self): 98 | return self.framework.page_parse(self._pages).get_docs(self.q, self.links) 99 | -------------------------------------------------------------------------------- /maryam/core/util/search/piratebay.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | class main: 19 | 20 | def __init__(self, q, limit=15): 21 | """ piratebay search engine 22 | q : query for search 23 | limit : maximum result count 24 | """ 25 | self.framework = main.framework 26 | self.q = self.framework.urlib(q).quote_plus 27 | self.max = limit 28 | self._json = '' 29 | self._results = [] 30 | 31 | def run_crawl(self): 32 | url = 'https://apibay.org/q.php' 33 | payload = {'q' : self.q, 'cat':''} 34 | self.framework.verbose('[PIRATEBAY] Searching piratebay...') 35 | try: 36 | req = self.framework.request( 37 | url=url, 38 | params=payload, 39 | allow_redirects=True) 40 | except: 41 | self.framework.error('ConnectionError.', 'util/piratebay', 'run_crawl') 42 | self.framework.error('Piratebay is missed!', 'util/piratebay', 'run_crawl') 43 | 44 | else: 45 | self._rawhtml = req.text 46 | self._json = req.json() 47 | 48 | def make_magnet(self, info_hash, name): 49 | return (f"magnet:?xt=urn:btih:{info_hash}&dn={name}" 50 | '&tr=udp://tracker.coppersurfer.tk:6969/announce' 51 | '&tr=udp://open.demonii.com:1337' 52 | '&tr=udp://tracker.openbittorrent.com:6969/announce' 53 | '&tr=udp://tracker.opentrackr.org:1337' 54 | '&tr=udp://tracker.leechers-paradise.org:6969/announce' 55 | '&tr=udp://tracker.dler.org:6969/announce' 56 | '&tr=udp://opentracker.i2p.rocks:6969/announce' 57 | '&tr=udp://47.ip-51-68-199.eu:6969/announce') 58 | 59 | @property 60 | def raw(self): 61 | return self._rawhtml 62 | 63 | @property 64 | def json(self): 65 | return self._json 66 | 67 | def format_size(self, num, suffix='B'): 68 | for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: 69 | if abs(num) < 1024.0: 70 | return f"{num:3.1f} {unit}{suffix}" 71 | num /= 1024.0 72 | return f"{num:.1f} Yi{suffix}" 73 | 74 | @property 75 | def results(self): 76 | for count,torrent in enumerate(self.json): 77 | if count == self.max: 78 | break 79 | 80 | title = torrent['name'] 81 | info = torrent['info_hash'] 82 | seedleech = f"seeders: {torrent['seeders']} | leechers: {torrent['leechers']}" 83 | size = self.format_size(int(torrent['size'])) 84 | 85 | self._results.append({ 86 | 't': title, 87 | 'a': self.make_magnet(info, title), 88 | 'c': f"{torrent['username']} | {size}", 89 | 'd': seedleech 90 | }) 91 | 92 | return self._results 93 | -------------------------------------------------------------------------------- /maryam/core/util/search/arxiv.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from lxml import etree 19 | 20 | class main: 21 | 22 | def __init__(self, q, limit=1): 23 | """ arxiv.org search engine 24 | 25 | q : query for search 26 | limit : maximum result count 27 | """ 28 | self.framework = main.framework 29 | self.q = q 30 | self.limit = limit 31 | self._tree = '' 32 | self._rawxml = [] 33 | self._articles = [] 34 | self._links = [] 35 | self._results = [] 36 | self.url = 'https://export.arxiv.org/api/query' 37 | self.NSMAP = {'w3':'http://www.w3.org/2005/Atom'} 38 | 39 | def run_crawl(self): 40 | page = 1 41 | set_start = lambda x: (x - 1) * 10 + 1 42 | payload = {'search_query': f"all:{self.q}", 'start': set_start(page)} 43 | self.framework.verbose('[ARXIV] Searching the arxiv.org domain...') 44 | 45 | max_attempt = 0 46 | while True: 47 | try: 48 | req = self.framework.request( 49 | url=self.url, 50 | params=payload) 51 | except: 52 | self.framework.error('ConnectionError', 'arxiv', 'run_crawl') 53 | self.framework.error('ArXiv is missed!', 'arxiv', 'run_crawl') 54 | return 55 | else: 56 | self._rawxml.append(req.text) 57 | self._tree = etree.fromstring(req.text.encode('utf-8')) 58 | self._articles.extend(self.find(self._tree, './/w3:entry')) 59 | if page >= self.limit: 60 | break 61 | page += 1 62 | payload['start'] = set_start(page) 63 | 64 | def find(self, x, tofind): 65 | return x.findall(tofind, namespaces=self.NSMAP) 66 | 67 | @property 68 | def raw(self): 69 | return self._rawxml 70 | 71 | @property 72 | def articles(self): 73 | return self._articles 74 | 75 | @property 76 | def links(self): 77 | self._links = list(map(lambda x: x.text, 78 | self.find(self._tree, './/w3:id'))) 79 | return self._links 80 | 81 | @property 82 | def results(self): 83 | findlink = lambda x: self.find(x, './/w3:id')[0].text 84 | findauthors = lambda x: ', '.join(list(map(lambda x: x.text, 85 | self.find(x, './/w3:author/w3:name')))) 86 | findtitle = lambda x: self.find(x, './/w3:title')[0].text 87 | findsummary = lambda x: self.find(x, './/w3:summary')[0].text.strip() 88 | 89 | for article in self._articles: 90 | self._results.append({ 91 | 't': findtitle(article), 92 | 'a': findlink(article), 93 | 'c': findauthors(article), 94 | 'd': findsummary(article) 95 | }) 96 | 97 | return self._results 98 | -------------------------------------------------------------------------------- /maryam/modules/iris/topicmodeling.py: -------------------------------------------------------------------------------- 1 | # modules/iris/topicmodeling.py 2 | # # TESTING The Main Function = 3 | # topicmodeling -i mixed.json -t json -m all-distilroberta-v1 4 | # topicmodeling -i mixed.json -t json -s -v -m all-distilroberta-v1 5 | # topicmodeling -i mixed.json -t json -s -m all-distilroberta-v1 6 | # topicmodeling -i mixed.json -t json -v -m all-distilroberta-v1 7 | # topicmodeling -i testdataset.csv -t csv -m all-mpnet-base-v2 8 | # topicmodeling -i testdataset.csv -t csv -s -v -m all-mpnet-base-v2 9 | # topicmodeling -i testdataset.csv -t csv -s -m all-mpnet-base-v2 10 | # topicmodeling -i testdataset.csv -t csv -v -m all-mpnet-base-v2 11 | # # TESTING The Reporting & API Function = 12 | # topicmodeling -i mixed.json -t json -m all-distilroberta-v1 --output 13 | # report json testreport iris/topicmodeling 14 | # topicmodeling -i mixed.json -t json -m all-distilroberta-v1 --api 15 | # # TESTING The Function: Search Topics Using a Keyword (Top2Vec) = 16 | # topicmodeling -i mixed.json -t json -m all-distilroberta-v1 -k music 17 | # Note: download the dataset for testing from https://github.com/keamanansiber/Maryam/tree/master/notebooks 18 | # Hatma Suryotrisongko 19 | 20 | 21 | meta = { 22 | 'name': 'Topic Modeling', 23 | 'author': 'Hatma Suryotrisongko', 24 | 'version': '0.1.0', 25 | 'description': 'Topic Modeling Algorithms.', 26 | 'required': ('dask', 'scikit-learn', 'umap', 'bertopic', 'gensim', 'top2vec'), 27 | 'options': ( 28 | ('inputfile', None, True, 'Input file that contains the data', '-i', 'store', str), 29 | ('filetype', None, True, 'File type: csv/json', '-t', 'store', str), 30 | ('keyword', None, False, 'Search keyword: ', '-k', 'store', str), 31 | ('showcharts', None, False, 'Show charts?', '-s', 'store_true', bool), 32 | ('verbose', None, False, 'Verbose output?', '-v', 'store_true', bool), 33 | ('pretrained_model', None, True, 'model for embedding', '-m', 'store', str), 34 | ), 35 | 'examples': ('topicmodeling -i mixed.json -t json -k music -s True -v False -m all-mpnet-base-v2') 36 | } 37 | 38 | 39 | def module_api(self): 40 | 41 | run = self.topic(self.options['inputfile'], self.options['filetype'], self.options['keyword'], self.options['showcharts'], self.options['verbose']) 42 | run.run_sklearn_cluster_kmeans(self.options['pretrained_model'], self.options['showcharts'], self.options['verbose']) 43 | 44 | results = run.run_topic_modeling_bertopic(self.options['pretrained_model'], self.options['verbose']) 45 | self.output("\n\nResults = \n") 46 | self.output( results ) 47 | 48 | output = {'results': results.to_json(orient="records") } 49 | 50 | inputfile = self.options['inputfile'] 51 | self.save_gather(output, 'iris/topicmodeling', inputfile, output=self.options['output']) 52 | 53 | if self.options['keyword'] is not None: 54 | self.output(" keyword = " + self.options['keyword']) 55 | run.run_search_topics_top2vec(self.options['keyword'], self.options['showcharts'], self.options['verbose']) 56 | 57 | return output 58 | 59 | 60 | def module_run(self): 61 | output = module_api(self) 62 | self.output("\n\nOutput = \n") 63 | self.output( output ) 64 | -------------------------------------------------------------------------------- /maryam/modules/osint/docs_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Documentations Search', 17 | 'author': 'Saeed', 18 | 'version': '0.8', 19 | 'description': 'Search in open-sources to find relevant documents. filetypes[pdf,doc,docx,ppt,pptx,xlsx,txt,..].', 20 | 'sources': ('bing', 'google', 'yahoo', 'metacrawler', 'etools', 'duckduckgo'), 21 | 'options': ( 22 | ('query', None, True, 'Host Name, Company Name, keyword, query, etc', '-q', 'store', str), 23 | ('file', 'pdf', True, 'Filetype [pdf,doc,docx,ppt,pptx,xlsx,txt,..]', '-f', 'store', str), 24 | ('limit', 2, False, 'Search limit(number of pages, default=2)', '-l', 'store', int), 25 | ('count', 50, False, 'number of results per page(min=10)', '-c', 'store', int), 26 | ('site', False, False, 'If this is set, search just limited to the site', '-s', 'store_false', bool), 27 | ('engines', 'bing,google', True, 'Search engines with comma separator', '-e', 'store', str), 28 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 29 | ), 30 | 'examples': ('docs_search -q amazon -f pdf -e google,bing,metacrawler --thread 3', 31 | 'docs_search -q amazon -f pdf -e google,bing,metacrawler -l 3') 32 | } 33 | 34 | DOCS = [] 35 | 36 | def search(self, name, q, q_formats, limit, count): 37 | global DOCS 38 | try: 39 | engine = getattr(self, name) 40 | q = q 41 | varnames = engine.__init__.__code__.co_varnames 42 | if 'limit' in varnames and 'count' in varnames: 43 | attr = engine(q, limit, count) 44 | elif 'limit' in varnames: 45 | attr = engine(q, limit) 46 | else: 47 | attr = engine(q) 48 | 49 | attr.run_crawl() 50 | DOCS += attr.docs 51 | except Exception as e: 52 | print(e) 53 | 54 | def module_api(self): 55 | query = self.options['query'] 56 | _type = self.options['file'].lower() 57 | limit = self.options['limit'] 58 | count = self.options['count'] 59 | engines = self.options['engines'].lower().split(',') 60 | # Make dork 61 | if self.options['site']: 62 | dork = self.urlib(f"filetype:{_type} site:{self.options['site']} {query}").quote 63 | else: 64 | dork = f"{query} filetype:{_type}" 65 | 66 | self.thread(search, self.options['thread'], engines, dork, {}, limit, count, meta['sources']) 67 | output = {_type : list(set(DOCS))} 68 | self.save_gather(output, 'osint/docs_search', query, \ 69 | [_type], output=self.options['output']) 70 | return output 71 | 72 | def module_run(self): 73 | self.alert_results(module_api(self)) 74 | -------------------------------------------------------------------------------- /maryam/core/util/search/wikipedia.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | class main: 16 | 17 | def __init__(self, q, count=10): 18 | """ wikipedia.org search 19 | q : Query for search 20 | count : Number of results 21 | """ 22 | self.framework = main.framework 23 | self.q = q 24 | self.num = count 25 | self._links = [] 26 | self._titles = [] 27 | self._pids = [] 28 | self.url = 'https://en.wikipedia.org/w/api.php' 29 | self.headers = {'User-Agent' : 'OWASP Maryam(github.com/saeeddhqan/maryam)'} 30 | 31 | 32 | def run_crawl(self): 33 | self.framework.verbose('[WIKIPEDIA] Searching...', end='\r') 34 | payload = { 35 | 'action': 'query', 36 | 'list': 'search', 37 | 'prop': '', 38 | 'srsearch': self.q, 39 | 'srlimit': self.num, 40 | 'utf8': '', 41 | 'format': 'json' 42 | } 43 | 44 | res = self.wiki_request(payload) 45 | if res is None: 46 | return 47 | 48 | res = res.json() 49 | results = res['query']['search'] 50 | 51 | for result in results: 52 | title = result['title'] 53 | pid = result['pageid'] 54 | link = 'https://en.wikipedia.org/wiki/' + '_'.join(title.split(' ')) 55 | self._titles.append(title) 56 | self._links.append(link) 57 | self._pids.append(pid) 58 | 59 | 60 | def page(self): 61 | self.framework.verbose(f"[WIKIPEDIA] Getting page {self.q}...", end='\r') 62 | payload = { 63 | 'action': 'query', 64 | 'pageids': self.q, 65 | 'prop': 'info|extracts', 66 | 'inprop': 'url', 67 | 'explaintext': '', 68 | 'exintro': '', 69 | 'format': 'json' 70 | } 71 | 72 | res = self.wiki_request(payload) 73 | if not res: 74 | return 75 | 76 | res = res.json() 77 | return res['query']['pages'][str(self.q)] 78 | 79 | def wiki_request(self, payload): 80 | try: 81 | req = self.framework.request( 82 | url=self.url, 83 | params=payload, 84 | headers=self.headers, 85 | allow_redirects=False) 86 | return req 87 | except Exception as e: 88 | self.framework.error(f"ConnectionError: {e}", 'util/wikipedia', 'wiki_request') 89 | return None 90 | 91 | @property 92 | def links(self): 93 | return self._links 94 | 95 | @property 96 | def titles(self): 97 | return self._titles 98 | 99 | @property 100 | def pids(self): 101 | return self._pids 102 | 103 | @property 104 | def links_with_title(self): 105 | return zip(self._links, self._titles , self._pids) 106 | -------------------------------------------------------------------------------- /maryam/modules/osint/cloud_storage.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Cloud Storage', 20 | 'author': 'Vikas Kundu', 21 | 'version': '0.1', 22 | 'description': 'Search for the query in online storage services like GoogleDrive, OneDrive, Dropbox,\ 23 | Amazon S3, Box and show the results.', 24 | 'sources': ('google', 'etools', 'bing', 'duckduckgo'), 25 | 'options': ( 26 | ('query', None, True, 'Query string', '-q', 'store', str), 27 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 28 | ('count', 50, False, 'Number of links per page(min=10, max=100, default=50)', '-c', 'store', int), 29 | ('engine', 'bing', False, 'Engine names for search(default=google)', '-e', 'store', str), 30 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 31 | ), 32 | 'examples': ('cloud_storage -q -l 15 --output',) 33 | } 34 | 35 | LINKS = [] 36 | PAGES = '' 37 | 38 | def search(self, name, q, q_formats, limit, count): 39 | global PAGES,LINKS 40 | engine = getattr(self, name) 41 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 42 | varnames = engine.__init__.__code__.co_varnames 43 | if 'limit' in varnames and 'count' in varnames: 44 | attr = engine(q, limit, count) 45 | elif 'limit' in varnames: 46 | attr = engine(q, limit) 47 | else: 48 | attr = engine(q) 49 | 50 | attr.run_crawl() 51 | LINKS += attr.links 52 | PAGES += attr.pages 53 | 54 | def module_api(self): 55 | query = self.options['query'] 56 | limit = self.options['limit'] 57 | count = self.options['count'] 58 | engine = self.options['engine'].split(',') 59 | 60 | sites = ['drive.google.com', 'dl.dropboxusercontent.com', '1drv.ms', 'box.com/s', 's3.amazonaws.com'] 61 | output = { 'links': [] } 62 | 63 | for site_url in sites: 64 | q_formats = { 65 | 'default_q': f'site:{site_url} {query}', 66 | 'millionshort_q': f'site:{site_url} "{query}"', 67 | 'qwant_q': f'site:{site_url} {query}' 68 | } 69 | 70 | self.thread(search, self.options['thread'], engine, query, q_formats, limit, count, meta['sources']) 71 | 72 | output['links'] += list( self.reglib().filter(r"https?://([\w\-\.]+\.)?"\ 73 | + site_url.replace('.',r'\.')+"/", list( set(LINKS) ) ) ) #escaping . for regex search using replace() 74 | 75 | self.save_gather(output, 'osint/cloud_storage', query, output=self.options.get('output')) 76 | return output 77 | 78 | def module_run(self): 79 | self.alert_results(module_api(self)) 80 | -------------------------------------------------------------------------------- /maryam/modules/search/youtube.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | 19 | meta = { 20 | 'name': 'Youtube Search', 21 | 'author': 'Aman Rawat', 22 | 'version': '0.6', 23 | 'description': 'Search your query in the youtube.com and show the results.', 24 | 'sources': ('google', 'etools', 'bing', 'millionshort', 'duckduckgo'), 25 | 'options': ( 26 | ('query', None, True, 'Query string', '-q', 'store', str), 27 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 28 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 29 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 30 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 31 | ), 32 | 'examples': ('youtube -q -l 15 --output',) 33 | } 34 | 35 | LINKS = [] 36 | PAGES = '' 37 | 38 | def search(self, name, q, q_formats, limit, count): 39 | global PAGES,LINKS 40 | engine = getattr(self, name) 41 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 42 | varnames = engine.__init__.__code__.co_varnames 43 | if 'limit' in varnames and 'count' in varnames: 44 | attr = engine(q, limit, count) 45 | elif 'limit' in varnames: 46 | attr = engine(q, limit) 47 | else: 48 | attr = engine(q) 49 | 50 | attr.run_crawl() 51 | LINKS += attr.links 52 | PAGES += attr.pages 53 | 54 | def module_api(self): 55 | query = self.options['query'] 56 | limit = self.options['limit'] 57 | count = self.options['count'] 58 | engines = self.options['engine'].split(',') 59 | output = {'videos': [], 'channels': [], 'usernames': []} 60 | q_formats = { 61 | 'default_q': f"site:youtube.com {query}", 62 | 'qwant_q': f"site:www.youtube.com {query}", 63 | 'millionshort_q': f'site:www.youtube.com "{query}"', 64 | } 65 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 66 | 67 | links = filter(lambda x: '/feed/' not in\ 68 | x and 'www.youtube.com' in x and ('/watch?' in x.lower() or '/playlist?' in x.lower()), list(set(LINKS))) 69 | networks = self.page_parse(PAGES).get_networks 70 | output['usernames'] = list(set(networks['Youtube user'])) 71 | output['channels'] = list(set(networks['Youtube channel'])) 72 | for video in links: 73 | output['videos'].append(video) 74 | self.save_gather(output, 'search/youtube', query, output=self.options.get('output')) 75 | return output 76 | 77 | def module_run(self): 78 | self.alert_results(module_api(self)) 79 | -------------------------------------------------------------------------------- /maryam/core/util/helpers/keywords.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | import json 20 | 21 | class main: 22 | 23 | def __init__(self, q): 24 | """ to find suggestions for keywords 25 | example: 26 | in: 'google' 27 | out: ['google docs', 'google summer of code', 'google maps', 'google mail', 'google news', ..] 28 | 29 | q : Query string 30 | """ 31 | self.framework = main.framework 32 | self.q = self.framework.urlib(q).quote 33 | self._keys_category = [] 34 | self._keys = [] 35 | self._yahoo = \ 36 | 'https://search.yahoo.com/sugg/gossip/gossip-us-ura/?f=1&output=sd1&command=&pq=a&l=3&nresults=30000&b=3&s=1c&callback=' 37 | self._google = 'https://www.google.com/complete/search?q=&cp=10&client=gws-wiz&xssi=t&gs_pcrt=undefined&hl=en&authuser=0' 38 | self._bing = 'https://www.bing.com/AS/Suggestions?pt=&mkt=de-de&qry=&cp=0&css=0&cvid=1' 39 | # self._zapmeta = 'https://www.zapmeta.com/suggest?q=' 40 | self._searx = 'https://searx.be/autocompleter?q=' 41 | 42 | def run_crawl(self): 43 | keys = {} 44 | for source in ('yahoo', 'google', 'bing',\ 45 | 'searx'): 46 | attr = getattr(self, '_' + source) 47 | if isinstance(attr, dict): 48 | url = attr['url'] 49 | method = 'POST' 50 | data = attr.get('payload', {'q':''}) 51 | data['q'] = self.q 52 | else: 53 | url = attr.replace('', self.q) 54 | method = 'GET' 55 | data = {} 56 | try: 57 | req = self.framework.request(url, method=method, data=data) 58 | except Exception as e: 59 | self.framework.error('ConnectionError', 'util/keywords', 'run_crawl') 60 | self.framework.print_exception() 61 | keys[source] = req 62 | 63 | keys['yahoo'] = [x['k'] for x in keys['yahoo'].json().get('r', [])] 64 | try: 65 | google = json.loads(f"{keys['google'].text[5:]}")[0] 66 | keys['google'] = [re.sub(r"|<\\/b>|", '', x[0]) for x in google] 67 | except: 68 | keys['google'] = [] 69 | keys['bing'] = re.findall(r'([^<]+)', re.sub(r'<.?strong>', '', keys['bing'].text)) 70 | # keys['zapmeta'] = [x[0] for x in keys['zapmeta'].json()] if hasattr(keys['zapmeta'], 'json') else [] 71 | try: 72 | keys['searx'] = keys['searx'].json()[self.q] 73 | except: 74 | keys['searx'] = [] 75 | 76 | self._keys_category = keys 77 | for s in keys: 78 | for i in keys[s]: 79 | self._keys.append(i) 80 | 81 | @property 82 | def keys_category(self): 83 | return self._keys_category 84 | 85 | @property 86 | def keys(self): 87 | return list(set(self._keys)) 88 | -------------------------------------------------------------------------------- /maryam/modules/search/linkedin.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'LinkedIn Search', 20 | 'author': 'Saeed', 21 | 'version': '0.5', 22 | 'description': 'Search your query in the linkedin.com and show the results.', 23 | 'sources': ('google', 'etools', 'bing', 'millionshort', 'duckduckgo'), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 28 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 29 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 30 | ), 31 | 'examples': ('linkedin -q -l 15 --output',) 32 | } 33 | 34 | 35 | LINKS = [] 36 | PAGES = '' 37 | 38 | def search(self, name, q, q_formats, limit, count): 39 | global PAGES,LINKS 40 | engine = getattr(self, name) 41 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 42 | varnames = engine.__init__.__code__.co_varnames 43 | if 'limit' in varnames and 'count' in varnames: 44 | attr = engine(q, limit, count) 45 | elif 'limit' in varnames: 46 | attr = engine(q, limit) 47 | else: 48 | attr = engine(q) 49 | 50 | attr.run_crawl() 51 | LINKS += attr.links 52 | PAGES += attr.pages 53 | 54 | def module_api(self): 55 | query = self.options['query'] 56 | limit = self.options['limit'] 57 | count = self.options['count'] 58 | engines = self.options['engine'].split(',') 59 | output = {'links': [], 'usernames': [], 'blogs': []} 60 | q_formats = { 61 | 'default_q': f"site:linkedin.com {query}", 62 | 'millionshort_q': f'site:linkedin.com "{query}"', 63 | 'qwant_q': f'site:linkedin.com {query}' 64 | } 65 | 66 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 67 | 68 | usernames = self.page_parse(PAGES).get_networks 69 | 70 | for link in usernames['Linkedin company'] + usernames['Linkedin individual']: 71 | link = link.replace('\\x22', '') 72 | if link not in output['usernames']: 73 | output['usernames'].append(link) 74 | 75 | output['links'] = list(self.reglib().filter(r"https?://([\w\-\.]+\.)?linkedin\.com/", list(set(LINKS)))) 76 | output['blogs'] = list(self.reglib().filter(r"https?://([\w\-\.]+\.)?linkedin\.com/pulse/", output['links'])) 77 | self.save_gather(output, 'search/linkedin', query, output=self.options.get('output')) 78 | return output 79 | 80 | def module_run(self): 81 | self.alert_results(module_api(self)) 82 | -------------------------------------------------------------------------------- /maryam/core/util/search/reddit_pushshift.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from datetime import datetime 19 | 20 | class main: 21 | 22 | def __init__(self, query, count=50, search='posts', username='', subreddit='', before='', after='', score='' ): 23 | """ Search in reddit using the pushshift api 24 | 25 | query : query to seach for 26 | count : no. of results to show 27 | search : search in posts or comments 28 | username : reddit username 29 | subreddit : reddit subreddit 30 | before : show data only before this timestamp 31 | after : show data only after this timestamp 32 | score : score of the posts or comments 33 | 34 | """ 35 | self.framework = main.framework 36 | self.query = query 37 | self.count = count 38 | self.search = search 39 | self.username = username 40 | self.subreddit = subreddit 41 | self.before = before 42 | self.after = after 43 | self.score = score 44 | self._usernames = set() # To ensure unique usernames 45 | self._links = set() # To ensure unique links 46 | 47 | def convert_to_epoch_time(self, date): 48 | try: 49 | date_time_obj = datetime.strptime(date, '%d/%m/%y %H:%M:%S') 50 | except ValueError: 51 | self.framework.error("Reddit_pushshift timestamp format is wrong. It should be '%d/%m/%y %H:%M:%S' i.e. 7/6/21 12:0:0. This parameter is skipped!", \ 52 | 'util/reddit_pushshift', 'convert_to_epoch_time') 53 | return '' 54 | else: 55 | return int(date_time_obj.timestamp()) 56 | 57 | def run_crawl(self): 58 | self.framework.verbose('[Reddit Pushshift] Searching...') 59 | self.before = self.convert_to_epoch_time(self.before) if self.before else self.before 60 | self.after = self.convert_to_epoch_time(self.after) if self.after else self.after 61 | base_url = 'https://api.pushshift.io/reddit/submission/search?html_decode=true' if self.search == 'posts' \ 62 | else 'https://api.pushshift.io/reddit/comment/search?html_decode=true' 63 | query_url = f"{base_url}&q={self.query}&author={self.username}&subreddit={self.subreddit}&size={self.count}&before={self.before}&after={self.after}&score={self.score}" 64 | 65 | try: 66 | response = self.framework.request(query_url, timeout=20) # Increased timeout for date based queries which take long 67 | except Exception as e: 68 | self.framework.error('Reddit_pushshift is missed!', 'util/reddit_pushshift', 'search') 69 | return False 70 | else: 71 | data = response.json() 72 | for i in data['data']: 73 | self._usernames.add(i['author']) 74 | self._links.add(f"https://www.reddit.com{i['permalink']}") 75 | 76 | @property 77 | def usernames(self): 78 | return list(self._usernames) 79 | 80 | @property 81 | def links(self): 82 | return list(self._links) 83 | -------------------------------------------------------------------------------- /maryam/modules/search/twitter.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | meta = { 21 | 'name': 'Twitter Search', 22 | 'author': 'Saeed', 23 | 'version': '0.5', 24 | 'description': 'Search your query in the twitter.com and show the results.', 25 | 'sources': ('google', 'etools', 'bing', 'millionshort', 'duckduckgo'), 26 | 'options': ( 27 | ('query', None, True, 'Query string', '-q', 'store', str), 28 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 29 | ('count', 50, False, 'The number of results per page(min=10, max=100,\ 30 | default=50)', '-c', 'store', int), 31 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 32 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 33 | ), 34 | 'examples': ('twitter -q -l 15 --output',) 35 | } 36 | 37 | LINKS = [] 38 | PAGES = '' 39 | 40 | def search(self, name, q, q_formats, limit, count): 41 | global PAGES,LINKS 42 | engine = getattr(self, name) 43 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 44 | varnames = engine.__init__.__code__.co_varnames 45 | if 'limit' in varnames and 'count' in varnames: 46 | attr = engine(q, limit, count) 47 | elif 'limit' in varnames: 48 | attr = engine(q, limit) 49 | else: 50 | attr = engine(q) 51 | 52 | attr.run_crawl() 53 | LINKS += attr.links 54 | PAGES += attr.pages 55 | 56 | def module_api(self): 57 | query = self.options['query'] 58 | limit = self.options['limit'] 59 | count = self.options['count'] 60 | engines = self.options['engine'].split(',') 61 | output = {'links': [], 'people': [], 'hashtags': []} 62 | q_formats = { 63 | 'default_q': f"site:twitter.com {query}", 64 | 'millionshort_q': f'site:twitter.com "{query}"' 65 | } 66 | 67 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 68 | 69 | usernames = self.page_parse(PAGES).get_networks 70 | for _id in list(set(usernames.get('Twitter'))): 71 | _id = _id[_id.find('/') + 1:] 72 | if _id not in ('i', 'settings', \ 73 | 'notifications', 'messages', 'explore', 'home', 'search'): 74 | output['people'].append(_id) 75 | 76 | output['links'] = list(self.reglib().filter(r'https?://(www\.)?twitter\.com', list(set(LINKS)))) 77 | for link in filter(lambda x: '/hashtag/' in x, output['links']): 78 | link = re.sub(r'https?://(www\.)?twitter.com/hashtag/', '', link) 79 | if re.search(r"^[\w\d_\-]+$", link): 80 | output['hashtags'].append(link) 81 | 82 | self.save_gather(output, 83 | 'search/twitter', query, output=self.options.get('output')) 84 | return output 85 | 86 | def module_run(self): 87 | self.alert_results(module_api(self)) 88 | -------------------------------------------------------------------------------- /maryam/core/util/helpers/reglib.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | class main: 21 | 22 | def __init__(self, string=None): 23 | self.string = str(string) 24 | 25 | protocol_s = r"^([A-z0-9]+:\/\/)" 26 | protocol_m = r"([A-z0-9]+:\/\/)" 27 | email_s = r"^\w+@[A-z_\-.0-9]{5,255}$" 28 | email_m = r"\w+@[A-z_\-.0-9]{5,255}" 29 | phone_s = r"^([0-9]( |-)?)?(\(?[0-9]{3}\)?|[0-9]{3})( |-)?([0-9]{3}( |-)?[0-9]{4}|[A-z0-9]{7})$" 30 | phone_m = r"([0-9]( |-)?)?(\(?[0-9]{3}\)?|[0-9]{3})( |-)?([0-9]{3}( |-)?[0-9]{4}|[A-z0-9]{7})" 31 | domain_s = r"^([A-z0-9]([A-z0-9\-]{0,61}[A-z0-9])?\.)+[A-z]{2,6}(\:[0-9]{1,5})*$" 32 | domain_m = r"[A-z0-9\-]{0,61}\.+[A-z]{2,6}" 33 | url_s = r"^([A-z0-9]+:\/\/)?(www.|[A-z0-9].)[A-z0-9\-\.]+\.[A-z]{2,6}(\:[0-9]{1,5})*(\/($|[A-z0-9.,;?\'\\+&%$#=~_-]+))*$" 34 | url_m = r"ftp|https?://[A-z0-9\-.]{2,255}[\/A-z\.:\-0-9%~@#?&()+_;,\']+" 35 | id_s = r"^@[A-z_0-9\.\-]{2,255}$" 36 | id_m = r"@[A-z_0-9\.\-]{2,255}" 37 | ip_s = r"^\d+\.[\d]+\.[\d]+\.[\d]+$" 38 | ip_m = r"\d+\.[\d]+\.[\d]+\.[\d]+" 39 | social_network_ulinks = { 40 | 'Instagram': r"instagram\.com/[\w.\-]{1,30}", 41 | 'Facebook': [r"facebook\.com/[\w\-]{2,50}", r"fb\.com/[\w\-]{2,50}"], 42 | 'Twitter': r"twitter\.com/[\w\-.]{2,40}", 43 | 'Github': r"github\.com/[\w\-]{1,39}", 44 | 'Github site': [r"[\w\-]{1,39}\.github\.io", r"[\w\-]{1,39}\.github\.com"], 45 | 'Telegram': r"telegram\.me/[\w]{5,32}", 46 | 'Youtube user': r"youtube\.com/user/[\w\-\.]{2,100}", 47 | 'Youtube channel': [r"youtube\.com/c/[\w\-.]{2,100}", \ 48 | r"youtube\.com/channel/[\w\-.]{2,100}"], 49 | 'Linkedin company': r"linkedin\.com/company/[\w\.\-]{3,50}", 50 | 'Linkedin individual': r"linkedin\.com/in/[\w\.\-]{3,50}", 51 | 'Googleplus': r"\.?plus\.google\.com/[\w\-.+]{3,255}", 52 | 'WordPress': r"[\w\-]+\.wordpress\.com", 53 | 'Reddit': r"reddit\.com/user/[A-z0-9_\-]{3,20}", 54 | 'Tumblr': r"[\w\-]{3,32}\.tumblr\.com", 55 | 'Blogger': r"[\w\-]{3,50}\.blogspot\.com" 56 | } 57 | 58 | def search(self, regex, _type=list): 59 | regex = re.compile(regex) 60 | regex = regex.findall(self.string) 61 | return regex 62 | 63 | def sub(self, regex, sub_string): 64 | data = re.sub(regex, sub_string, self.string) 65 | return data 66 | 67 | def filter(self, regex, _list: list) -> list: 68 | if not isinstance(regex, str): 69 | return filter(regex, _list) 70 | else: 71 | return filter(re.compile(regex).match, _list) 72 | 73 | @property 74 | def emails(self): 75 | emails = self.search(self.email_m) 76 | return emails 77 | 78 | @property 79 | def urls(self): 80 | urls = self.search(self.url_m) 81 | return urls 82 | 83 | @property 84 | def domains(self): 85 | domains = self.search(self.domain_m) 86 | return domains 87 | -------------------------------------------------------------------------------- /maryam/modules/search/trello.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | import re 18 | 19 | meta = { 20 | 'name': 'Trello Search', 21 | 'author': 'Vikas Kundu', 22 | 'version': '0.5', 23 | 'description': 'Search public trello boards for data and show the results.', 24 | 'sources': ('google', 'etools', 'bing', 'millionshort', 'duckduckgo'), 25 | 'options': ( 26 | ('query', None, True, 'Query string', '-q', 'store', str), 27 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 28 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 29 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 30 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 31 | ), 32 | 'examples': ('trello -q -l 15 --output',) 33 | } 34 | 35 | 36 | LINKS = [] 37 | PAGES = '' 38 | 39 | def search(self, name, q, q_formats, limit, count): 40 | global PAGES,LINKS 41 | engine = getattr(self, name) 42 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 43 | varnames = engine.__init__.__code__.co_varnames 44 | if 'limit' in varnames and 'count' in varnames: 45 | attr = engine(q, limit, count) 46 | elif 'limit' in varnames: 47 | attr = engine(q, limit) 48 | else: 49 | attr = engine(q) 50 | 51 | attr.run_crawl() 52 | LINKS += attr.links 53 | PAGES += attr.pages 54 | 55 | def module_api(self): 56 | query = self.options['query'] 57 | limit = self.options['limit'] 58 | count = self.options['count'] 59 | engines = self.options['engine'].split(',') 60 | output = {'links': [], 'usernames': [], 'handles': []} 61 | q_formats = { 62 | 'default_q': f"site:trello.com {query}", 63 | 'millionshort_q': f'site:trello.com "{query}"', 64 | 'qwant_q': f'site:trello.com {query}' 65 | } 66 | 67 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 68 | 69 | output['links'] = list(self.reglib().filter(r"https?://([\w\-\.]+\.)?trello\.com/", list(set(LINKS)))) 70 | output['handles'] = [ i for i in set(re.findall(r"@[\w]+", PAGES)) if i not in ['@keyframes', '@media','@font'] ] 71 | 72 | output['usernames'] = [ i.replace('(','@') for i in set(re.findall(r"\([\w]{3,}\)", PAGES)) if query in i ] 73 | output['usernames'] = [ i.replace(')','') for i in output['usernames'] ] 74 | 75 | #using re directly instead of reglib().filter because when PAGES is passed a list, 76 | #it becomes useless for searching trello usernames. See: https://ibb.co/Rz182Bj 77 | 78 | self.save_gather(output, 'search/trello', query, output=self.options.get('output')) 79 | return output 80 | 81 | def module_run(self): 82 | self.alert_results(module_api(self)) 83 | -------------------------------------------------------------------------------- /maryam/core/util/osint/reddit.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import datetime 19 | 20 | class main: 21 | 22 | def __init__(self, q, count=20, sortby='relevance', verbose=False): 23 | """ Reddit search engine 24 | 25 | q : query for search 26 | count : maximum result count 27 | verbose : print entire json 28 | """ 29 | self.framework = main.framework 30 | self.q = q 31 | self.count = count 32 | self.sortby = sortby 33 | self._json = {} 34 | self._posts = [] 35 | self._verbose = verbose 36 | 37 | def run_crawl(self): 38 | self.framework.verbose('[REDDIT] Searching the reddit domain...') 39 | url = f'https://gateway.reddit.com/desktopapi/v1/search' 40 | payload = { 41 | 'q': self.q, 42 | 'sort': self.sortby, 43 | 't': 'all' 44 | } 45 | 46 | for _ in range(self.count//25+1): 47 | try: 48 | res = self.framework.request( 49 | url, 50 | params=payload, 51 | ) 52 | 53 | except Exception as e: 54 | self.framework.error(f"ConnectionError {e}.", 'util/osint/reddit', 'run_crawl') 55 | self.framework.error('Reddit is missed!', 'util/osint/reddit', 'run_crawl') 56 | 57 | else: 58 | self._json.update(res.json()) 59 | self._posts.extend(list(self._json['posts'].values())) 60 | 61 | @property 62 | def json(self): 63 | return self._json 64 | 65 | @property 66 | def posts(self): 67 | return self._posts 68 | 69 | @property 70 | def results(self): 71 | results = [] 72 | for count, post in enumerate(self._posts, 1): 73 | if count > self.count: 74 | break 75 | 76 | date = datetime.datetime.fromtimestamp(int(post['created'])//1000).strftime("%m/%d/%Y, %H:%M:%S") 77 | 78 | media = post['media'] 79 | if media is not None: 80 | type = media['type'] 81 | 82 | if type == 'embed': 83 | desc = media['content'] 84 | 85 | elif type == 'text': 86 | desc = media.get('markdownContent') 87 | 88 | elif type == 'video': 89 | desc = 'Video description currently not supported' 90 | 91 | elif type == 'gallery': 92 | desc = [] 93 | for item in media['gallery']['items']: 94 | id = item['mediaId'] 95 | desc.append(media['mediaMetadata'][id]['s']['u']) 96 | desc = '\n'.join(desc) 97 | 98 | elif type == 'image': 99 | desc = media['content'] 100 | 101 | elif post['source'] is not None: 102 | desc = post['source']['url'] 103 | 104 | else: 105 | desc = 'Description not available' 106 | 107 | results.append({ 108 | 't': post['title'], 109 | 'a': post['permalink'], 110 | 'c': f"author: {post['author']} | date: {date}", 111 | 'd': desc 112 | }) 113 | 114 | return results 115 | 116 | -------------------------------------------------------------------------------- /maryam/modules/osint/email_search.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Find Emails', 17 | 'author': 'Saeed', 18 | 'version': '1.1', 19 | 'description': 'Search in open-sources to find emails.', 20 | 'sources': ('bing', 'pastebin', 'google', 'metacrawler', 21 | 'baidu', 'startpage', 'duckduckgo', 'hunter', 'github', 'keyserver',), 22 | 'options': ( 23 | ('query', None, True, 'Domain name or company name', '-q', 'store', str), 24 | ('limit', 3, False, 'Search limit(number of pages, default=3)', '-l', 'store', int), 25 | ('count', 50, False, 'number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 26 | ('engines', None, True, 'Search engine names. e.g bing,google,..', '-e', 'store', str), 27 | ('key', None, False, 'Give a valid hunter API key. Limit for free plan is 10 results', '-k', 'store', str), 28 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 29 | ), 30 | 'examples': ('email_search -q microsoft.com -e bing --output', 31 | 'email_search -q owasp.org -e google,bing,yahoo -l 20 -t 3 --output') 32 | } 33 | 34 | EMAILS = [] 35 | 36 | def search(self, name, q, q_formats, limit, count): 37 | global EMAILS 38 | engine = getattr(self, name) 39 | eng = name 40 | varnames = engine.__init__.__code__.co_varnames 41 | q = q_formats[f"{eng}"] if f"{eng}" in q_formats else q_formats['default'] 42 | if 'limit' in varnames and 'count' in varnames: 43 | attr = engine(q, limit, count) 44 | elif 'limit' in varnames and 'key' in varnames: 45 | key = q.split('&api_key=')[1] 46 | k_q = q.split('&api_key=')[0] 47 | if key == 'None': 48 | self.error('-k is required for hunter', 'email_search', 'search') 49 | return 50 | else: 51 | attr = engine(k_q, key, limit) 52 | elif 'limit' in varnames: 53 | attr = engine(q, limit) 54 | else: 55 | attr = engine(q) 56 | 57 | attr.run_crawl() 58 | EMAILS.extend(attr.emails) 59 | 60 | def module_api(self): 61 | query = self.options['query'] 62 | domain = self.options['query'].replace('@', '') 63 | urlib = self.urlib(domain) 64 | domain = self.urlib(domain).netroot if '/' in domain else domain 65 | limit = self.options['limit'] 66 | count = self.options['count'] 67 | engines = self.options['engines'].split(',') 68 | key = self.options['key'] 69 | q_formats = { 70 | 'default': f'"%40{domain}"', 71 | 'ask': f"%40{domain}", 72 | 'hunter': f"{domain}&api_key={key}", 73 | 'github': domain, 74 | 'keyserver': domain 75 | } 76 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 77 | output = {'emails': list(set(EMAILS))} 78 | self.save_gather(output, 'osint/email_search', domain,\ 79 | output=self.options['output']) 80 | return output 81 | 82 | def module_run(self): 83 | self.alert_results(module_api(self)) 84 | -------------------------------------------------------------------------------- /maryam/modules/iris/sentiment.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | import concurrent.futures as futures 16 | import copy 17 | from json import loads 18 | from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer 19 | 20 | meta = { 21 | 'name': 'sentiment analysis', 22 | 'author': 'Saeed', 23 | 'version': '0.1', 24 | 'description': 'Running sentiment analysis on your data.', 25 | 'required': ('vaderSentiment',), 26 | 'options': ( 27 | ('json', None, True, 'Json file that contains the data', '-j', 'store', str), 28 | ('key', None, False, 'Data key. the value should be a list. None means the json file contains a list: ["..", ..]', '-k', 'store', str), 29 | ('thread', 5, False, 'The number of thread per each sell(default=10)', '-t', 'store', int), 30 | ('pipe', False, False, 'Dev only! pipe data from other modules(default=False)', '-p', 'store_true', bool), 31 | ), 32 | 'examples': ('sentiment -j test.json -k data', 33 | 'sentiment -j test.json -t 50') 34 | } 35 | 36 | SHOW_MSG = {'neg': 'negative', 'neu': 'neutral', 'pos': 'positive', 'compound': 'compound'} 37 | 38 | def dothetask(i, j): 39 | global OVERALL, SA, MAXES 40 | for cell in DATA[i:j]: 41 | p = SA.polarity_scores(cell) 42 | for i in p: 43 | OVERALL[i] += p[i] 44 | if MAXES[i] != []: 45 | if p[i] > MAXES[i][0]: 46 | MAXES[i] = [p[i], cell] 47 | else: 48 | MAXES[i] = [p[i], cell] 49 | 50 | def thread(nthread): 51 | with futures.ThreadPoolExecutor(max_workers=nthread) as executor: 52 | tire = round(LEN/nthread) 53 | if tire > 0: 54 | for i in range(tire+1): 55 | mx = i*nthread 56 | executor.submit(dothetask, mx, mx+nthread) 57 | else: 58 | executor.submit(dothetask, 0, LEN) 59 | 60 | def module_api(self): 61 | global DATA, LEN, OVERALL, SA, MAXES 62 | SA = SentimentIntensityAnalyzer() 63 | OVERALL = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compound': 0.0} 64 | MAXES = {'neg': [], 'neu': [], 'pos': [], 'compound': []} 65 | if not self.options['pipe']: 66 | json = self.options['json'] 67 | key = self.options['key'] 68 | file = self._is_readable(json) 69 | if not file: 70 | return 71 | DATA = loads(file.read()) 72 | if key not in DATA and key != None: 73 | self.error('The key doesn\'t exists', 'module_api', 'iris/sentiment') 74 | return 75 | if key != None: 76 | DATA = DATA[key] 77 | LEN = len(DATA) 78 | thread(self.options['thread']) 79 | if self.options['pipe']: 80 | return {'overall': OVERALL, 'maxes': MAXES} 81 | output = {json : {key: [OVERALL, MAXES]}} 82 | self.save_gather(output, 'osint/docs_search', json, \ 83 | [key], output=self.options['output']) 84 | return output 85 | 86 | def module_run(self): 87 | module_api(self) 88 | for i in OVERALL: 89 | self.alert(SHOW_MSG[i]) 90 | self.output(f"\t{OVERALL[i]}") 91 | if MAXES[i]: 92 | self.output(f"most {i} sentence: \t{MAXES[i][1]}") 93 | -------------------------------------------------------------------------------- /maryam/core/util/engines/etools.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | import webbrowser 20 | 21 | class main: 22 | 23 | def __init__(self, q): 24 | """ carrot2.org search engine 25 | 26 | q : Query for search 27 | """ 28 | self.framework = main.framework 29 | self.q = q 30 | self._pages = '' 31 | self._json = {} 32 | self.etools = 'https://www.etools.ch/partnerSearch.do' 33 | 34 | def run_crawl(self): 35 | params = {'country': 'web', 'dataSourceResults': 40, 36 | 'dataSources': 'all', 'language': 'en', 'maxRecords': 200, 'partner': 'Carrot2Json', 37 | 'query': self.q, 'safeSearch': 'false'} 38 | headers = {'Host': 'www.etools.ch', 39 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0', 40 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 41 | 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 42 | 'Connection': 'keep-alive', 'Cookie': 'JSESSIONID=6120E7C52197190DE5126DCBF47D38B0', 43 | 'Upgrade-Insecure-Requests': '1', 'Cache-Control': 'max-age=0'} 44 | self.framework.debug(f"[eTOOLS] Searching in 'etools.ch'...") 45 | try: 46 | req = self.framework.request(url=self.etools, params=params, headers=headers, allow_redirects=True) 47 | except: 48 | self.framework.error('ConnectionError.', 'util/carrot2', 'run_crawl') 49 | self.framework.error('eTOOLS is missed!', 'util/carrot2', 'run_crawl') 50 | else: 51 | self._pages = req.text 52 | try: 53 | self._json = req.json() 54 | except: 55 | self.framework.error('eTOOLS is missed!', 'util/carrot2', 'run_crawl') 56 | 57 | @property 58 | def pages(self): 59 | return self._pages 60 | 61 | @property 62 | def results(self): 63 | resp = self._json.get('response') 64 | results = [] 65 | if resp: 66 | merged_records = resp.get('mergedRecords') 67 | for result in merged_records: 68 | results.append({ 69 | 't': result['title'], 70 | 'a': result['url'], 71 | 'c': self.framework.meta_search_util().make_cite(result['url']), 72 | 'd': result['text']}) 73 | return results 74 | 75 | @property 76 | def links(self): 77 | self._links = [] 78 | resp = self._json.get('response') 79 | if resp: 80 | resp = resp.get('mergedRecords') 81 | for link in resp: 82 | self._links.append(link.get('url')) 83 | return self._links 84 | 85 | @property 86 | def json(self): 87 | return self._json 88 | 89 | @property 90 | def dns(self): 91 | return self.framework.page_parse(self._pages).get_dns(self.q) 92 | 93 | @property 94 | def emails(self): 95 | return self.framework.page_parse(self._pages).get_emails(self.q) 96 | 97 | @property 98 | def docs(self): 99 | return self.framework.page_parse(self._pages).get_docs(self.q) 100 | -------------------------------------------------------------------------------- /maryam/modules/search/spotify.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | meta = { 19 | 'name': 'Spotify Search', 20 | 'author': 'Kunal Khandelwal', 21 | 'version': '0.1', 22 | 'description': 'Search artists, albums, playlist and users on spotify', 23 | 'sources': ('google', 'bing', 'duckduckgo', 'millionshort', 'etools'), 24 | 'options': ( 25 | ('query', None, True, 'Query string', '-q', 'store', str), 26 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 27 | ('count', 50, False, 'Number of links per page(min=10, max=100, default=50)', '-c', 'store', int), 28 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 29 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 30 | ), 31 | 'examples': ('spotify -q -l 15 --output',) 32 | } 33 | 34 | LINKS = [] 35 | 36 | def search(self, name, q, q_formats, limit, count): 37 | global LINKS 38 | engine = getattr(self, name) 39 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 40 | varnames = engine.__init__.__code__.co_varnames 41 | if 'limit' in varnames and 'count' in varnames: 42 | attr = engine(q, limit, count) 43 | elif 'limit' in varnames: 44 | attr = engine(q, limit) 45 | else: 46 | attr = engine(q) 47 | 48 | attr.run_crawl() 49 | LINKS += attr.links 50 | 51 | def module_api(self): 52 | global LINKS 53 | query = self.options['query'] 54 | limit = self.options['limit'] 55 | count = self.options['count'] 56 | engines = self.options['engine'].split(',') 57 | output = {'user': [], 'artist': [], 'playlist': [], 'album': []} 58 | q_formats = { 59 | 'default_q': f"site:open.spotify.com {query}", 60 | 'millionshort_q': f'site:open.spotify.com "{query}"', 61 | 'qwant_q': f'site:open.spotify.com {query}' 62 | } 63 | 64 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 65 | 66 | 67 | LINKS = list(self.reglib().filter(r"https?://(open\.)?spotify\.com/", list(set(LINKS)))) 68 | 69 | for link in self.reglib().filter(r"https?://(open\.)?spotify\.com/user/", LINKS): 70 | if '/playlist/' not in link: 71 | output['user'].append(link) 72 | else: 73 | output['playlist'].append(link) 74 | output['artist'] = list(self.reglib().filter(r"https?://(open\.)?spotify\.com/artist/", LINKS)) 75 | output['album'] = list(self.reglib().filter(r"https?://(open\.)?spotify\.com/album/", LINKS)) 76 | output['playlist'] += list(self.reglib().filter(r"https?://(open\.)?spotify\.com/playlist/", LINKS)) 77 | 78 | self.save_gather(output, 'search/spotify', query, output=self.options.get('output')) 79 | return output 80 | 81 | 82 | def module_run(self): 83 | self.alert_results(module_api(self)) 84 | -------------------------------------------------------------------------------- /maryam/modules/search/quora.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | 21 | meta = { 22 | 'name': 'Quora Search', 23 | 'author': 'Aman Rawat', 24 | 'version': '0.2', 25 | 'description': 'Search your query in the quora.com and show the results.', 26 | 'sources': ('google', 'bing', 'millionshort', 'etools', 'duckduckgo'), 27 | 'options': ( 28 | ('query', None, True, 'Query string', '-q', 'store', str), 29 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 30 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 31 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 32 | ('engine', 'bing', False, 'Engine names for search(default=bing)', '-e', 'store', str), 33 | ), 34 | 'examples': ('quora -q -l 15 --output',) 35 | } 36 | 37 | LINKS = [] 38 | 39 | def search(self, name, q, q_formats, limit, count): 40 | global LINKS 41 | engine = getattr(self, name) 42 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 43 | varnames = engine.__init__.__code__.co_varnames 44 | if 'limit' in varnames and 'count' in varnames: 45 | attr = engine(q, limit, count) 46 | elif 'limit' in varnames: 47 | attr = engine(q, limit) 48 | else: 49 | attr = engine(q) 50 | 51 | attr.run_crawl() 52 | if name == 'google': 53 | LINKS += [(x['a'], x['t']) for x in attr.results_original] 54 | LINKS += attr.links 55 | 56 | def module_api(self): 57 | query = self.options['query'] 58 | limit = self.options['limit'] 59 | count = self.options['count'] 60 | engines = self.options['engine'].split(',') 61 | output = {'links': [], 'usernames': []} 62 | q_formats = { 63 | 'default_q': f"site:www.quora.com {query}", 64 | 'millionshort_q': f'site:www.quora.com "{query}"', 65 | 'qwant_q': f'site:www.quora.com {query}' 66 | } 67 | 68 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 69 | 70 | links = list(set(LINKS)) 71 | for link in self.reglib().filter(r"https?://(www\.)?quora\.com/profile/", links): 72 | if link not in output['usernames']: 73 | output['usernames'].append(link) 74 | 75 | for link in links: 76 | if re.search(r"https?://(www\.)?quora\.com", link) and '/profile' not in link: 77 | title = re.sub(r"https?://(www\.)?quora\.com/", '', link) 78 | title = title.replace('-', ' ') 79 | title = self.urlib(title).unquote.split('?')[0] 80 | output['links'].append([link, title]) 81 | 82 | self.save_gather(output, 'search/quora', query, output=self.options.get('output')) 83 | return output 84 | 85 | def module_run(self): 86 | output = module_api(self) 87 | self.alert('usernames') 88 | for user in output['usernames']: 89 | self.output(f"\t{user}", 'G') 90 | self.alert('links') 91 | for item in output['links']: 92 | title, link = item[1], item[0] 93 | self.output(title) 94 | self.output(f"\t{link}", 'G') 95 | print('') 96 | -------------------------------------------------------------------------------- /maryam/core/util/search/pastebin.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | import re 16 | import concurrent.futures 17 | 18 | class main: 19 | def __init__(self, q, limit=1, count=10): 20 | """ pastebin link 21 | q : The query 22 | limit : Number of pages 23 | """ 24 | self.framework = main.framework 25 | self.q = q.split('_')[0] 26 | self._pages = '' 27 | self.sources = ['google', 'bing', 'etools', 'duckduckgo', 'millionshort', 'qwant'] 28 | self._domains = [] 29 | self.extract_url = 'https://pastebin.com/raw' 30 | self.limit = limit 31 | self.count = count 32 | self.thread = 3 33 | self._links = [] 34 | self._links_and_titles = [] 35 | self.q_formats = { 36 | 'default_q': f'site:pastebin.com "{self.q}"', 37 | 'qwant_q': f'site:pastebin.com {self.q}' 38 | } 39 | 40 | def search(self, self2, name, q, q_formats, limit, count): 41 | engine = getattr(self.framework, name) 42 | q = self.q_formats[f"{name}_q"] if f"{name}_q" in self.q_formats else self.q_formats['default_q'] 43 | varnames = engine.__init__.__code__.co_varnames 44 | if 'limit' in varnames and 'count' in varnames: 45 | attr = engine(q, limit, count) 46 | elif 'limit' in varnames: 47 | attr = engine(q, limit) 48 | else: 49 | attr = engine(q) 50 | 51 | attr.run_crawl() 52 | self._pages += attr.pages 53 | self._pastebin_pages = '' 54 | 55 | # def open_pages(self, link): 56 | # heading = re.search(r"pastebin\.com/([\w\d]+)", link) 57 | # title = 'no title' 58 | # if heading: 59 | # head_raw = f"https://pastebin.com/raw/{heading.group(1)}" 60 | # print(head_raw) 61 | # # try: 62 | # head_req = self.framework.request(url=head_raw).text 63 | # # except Exception as e: 64 | # # self.framework.verbose('Pastebin is missed!') 65 | # # else: 66 | # head_title = f"{self.q} pastes {head_req.splitlines()[0].lstrip()[:30]}...".ljust(10, ' ') 67 | # title = head_title.title() 68 | # self._pastebin_pages += head_req 69 | # self._links_and_titles.append([link, title]) 70 | 71 | def run_crawl(self): 72 | self.framework.thread(self.search, self.thread, self.sources, self.q, self.q_formats, self.limit, self.count, self.sources) 73 | self._links = list(set(self.framework.reglib(self._pages).search(r"https://pastebin\.com/[\w\d]+"))) 74 | # print(links) 75 | # self.framework.verbose('Rearranging paste links [give it a few seconds]...') 76 | # with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: 77 | # [executor.submit(self.open_pages, url) for url in links] 78 | 79 | @property 80 | def pages(self): 81 | return self._pastebin_pages 82 | 83 | @property 84 | def links(self): 85 | return self._links 86 | 87 | @property 88 | def links_and_titles(self): 89 | return self._links_and_titles 90 | 91 | @property 92 | def dns(self): 93 | return self.framework.page_parse(self._pastebin_pages).get_dns(self.q) 94 | 95 | @property 96 | def emails(self): 97 | return self.framework.page_parse(self._pastebin_pages).get_emails(self.q) 98 | -------------------------------------------------------------------------------- /maryam/modules/search/tiktok.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | 21 | meta = { 22 | 'name': 'TikTok Search', 23 | 'author': 'Prakhar Jain', 24 | 'version': '0.1', 25 | 'description': 'Search your query on TikTok and show the results.', 26 | 'sources': ('google', 'bing', 'metacrawler', 'millionshort', 'etools', 'duckduckgo'), 27 | 'options': ( 28 | ('query', None, True, 'Query string', '-q', 'store', str), 29 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 30 | ('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int), 31 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 32 | ('engine', 'bing', False, 'Engine names for search(default=bing)', '-e', 'store', str), 33 | ), 34 | 'examples': ('tiktok -q -l 15 --output',) 35 | } 36 | 37 | LINKS = [] 38 | 39 | def search(self, name, q, q_formats, limit, count): 40 | global LINKS 41 | engine = getattr(self, name) 42 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 43 | varnames = engine.__init__.__code__.co_varnames 44 | if 'limit' in varnames and 'count' in varnames: 45 | attr = engine(q, limit, count) 46 | elif 'limit' in varnames: 47 | attr = engine(q, limit) 48 | else: 49 | attr = engine(q) 50 | 51 | attr.run_crawl() 52 | LINKS += attr.links 53 | 54 | def module_api(self): 55 | query = self.options['query'] 56 | limit = self.options['limit'] 57 | count = self.options['count'] 58 | engines = self.options['engine'].split(',') 59 | output = {'videos': [], 'usernames': [], 'tags': [], 'music': []} 60 | q_formats = { 61 | 'default_q': f"site:www.tiktok.com {query}", 62 | 'millionshort_q': f'site:www.tiktok.com "{query}"', 63 | 'qwant_q': f'site:www.tiktok.com {query}' 64 | } 65 | 66 | self.thread(search, self.options['thread'], engines, query, q_formats, limit, count, meta['sources']) 67 | 68 | links = list(set(LINKS)) 69 | for link in self.reglib().filter(r"https?://(www\.)?tiktok\.com/@", links): 70 | user_url = re.sub(r"https?://(www\.)?tiktok\.com/", '', link) 71 | user_url = user_url.rsplit('/') 72 | user = user_url[0] 73 | if user not in output['usernames']: 74 | output['usernames'].append(user) 75 | 76 | for link in self.reglib().filter(r"https?://(www\.)?tiktok\.com/music/", links): 77 | if link not in output['music']: 78 | output['music'].append(link) 79 | 80 | for link in self.reglib().filter(r"https?://(www\.)?tiktok\.com/amp/tag/", links): 81 | if link not in output['tags']: 82 | output['tags'].append(link) 83 | 84 | for link in self.reglib().filter(r"https?://(www\.)?tiktok\.com/", links): 85 | if '/video/' in link: 86 | if link not in output['videos']: 87 | output['videos'].append(link) 88 | 89 | 90 | self.save_gather(output, 'search/tiktok', query, output=self.options.get('output')) 91 | return output 92 | 93 | def module_run(self): 94 | self.alert_results(module_api(self)) 95 | -------------------------------------------------------------------------------- /maryam/modules/osint/social_nets.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | any later version. 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . 13 | """ 14 | 15 | meta = { 16 | 'name': 'Find Social Networks', 17 | 'author': 'Saeed', 18 | 'version': '1.5', 19 | 'description': 'Search to find Usernames in social networks.', 20 | 'sources': ('bing', 'google', 'startpage', 'urlscan'), 21 | 'options': ( 22 | ('query', None, True, 'Company Name or Query', '-q', 'store', str), 23 | ('engines', 'google,bing', False, 'Search engine names. e.g `bing,google,..`', '-e', 'store', str), 24 | ('url', None, False, 'The second source to crawl for', '-u', 'store', str), 25 | ('depth', 1, False, 'Scraper depth level(default=1)(Only if the url is set)', '-d', 'store', int), 26 | ('cthread', 5, False, 'The number of links that open per round(default=5)(Only if the url is set)', '-T', 'store', int), 27 | ('limit', 5, False, 'Search limit(number of pages, default=5)', '-l', 'store', int), 28 | ('count', 100, False, 'number of results per page(min=10, max=100, default=100)', '-c', 'store', int), 29 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 30 | ), 31 | 'examples': ('social_nets -q microsoft -e google,bing,yahoo -c 50 -t 3 --output', 32 | 'social_nets -q microsoft -e google', 33 | 'social_nets -q microsoft -u microsoft.com -e google,bing,yahoo, -c 50 -t 5 -T 10 -d 2') 34 | } 35 | 36 | PAGES = '' 37 | def search(self, name, q, q_formats, limit, count): 38 | global PAGES 39 | engine = getattr(self, name) 40 | eng = name 41 | varnames = engine.__init__.__code__.co_varnames 42 | if 'limit' in varnames and 'count' in varnames: 43 | attr = engine(q, limit, count) 44 | elif 'limit' in varnames: 45 | reg_dom = self.reglib().domain_m 46 | if eng == 'urlscan': 47 | if not re.search(reg_dom, q): 48 | self.verbose('Invalid domain name. Cannot run urlscan') 49 | return 50 | else: 51 | attr = engine(q, limit) 52 | else: 53 | attr = engine(q) 54 | 55 | attr.run_crawl() 56 | PAGES += attr.pages 57 | 58 | def module_api(self): 59 | global PAGES 60 | query = '@' + self.options['query'] 61 | limit = self.options['limit'] 62 | count = self.options['count'] 63 | url = self.options['url'] 64 | depth = self.options['depth'] 65 | crawl_thread = self.options['cthread'] 66 | engines = self.options['engines'].lower().split(',') 67 | page = '' 68 | try: 69 | if '.' in self.options['query']: 70 | page = self.request(self.options['query']).text 71 | except: 72 | page = '' 73 | PAGES += page 74 | self.thread(search, self.options['thread'], engines, query, {}, limit, count, meta['sources']) 75 | if url: 76 | scrap = self.web_scrap(url, False, depth, crawl_thread) 77 | scrap.run_crawl() 78 | PAGES += scrap.pages 79 | usernames = self.page_parse(PAGES).get_networks 80 | self.save_gather(usernames, 'osint/social_nets', query, output=self.options['output']) 81 | return usernames 82 | 83 | def module_run(self): 84 | usernames = module_api(self) 85 | for net in usernames: 86 | lst = list(set(usernames[net])) 87 | if lst != []: 88 | self.alert(net) 89 | for atom in lst: 90 | self.output(f"\t{atom}", 'G') 91 | -------------------------------------------------------------------------------- /maryam/modules/search/facebook.py: -------------------------------------------------------------------------------- 1 | """ 2 | OWASP Maryam! 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import re 19 | 20 | meta = { 21 | 'name': 'Facebook Search', 22 | 'author': 'Saeed', 23 | 'version': '0.1', 24 | 'description': 'Search your query in the facebook.com and show the results.', 25 | 'sources': ('google', 'carrot2', 'bing', 'yahoo', 'millionshort', 'duckduckgo'), 26 | 'options': ( 27 | ('query', None, True, 'Query string', '-q', 'store', str), 28 | ('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int), 29 | ('count', 50, False, 'Number of links per page(min=10, max=100, default=50)', '-c', 'store', int), 30 | ('thread', 2, False, 'The number of engine that run per round(default=2)', '-t', 'store', int), 31 | ('engine', 'google', False, 'Engine names for search(default=google)', '-e', 'store', str), 32 | ), 33 | 'examples': ('facebook -q -l 15 --output',) 34 | } 35 | 36 | LINKS = [] 37 | PAGES = '' 38 | 39 | def search(self, name, q, q_formats, limit, count): 40 | global PAGES,LINKS 41 | engine = getattr(self, name) 42 | q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q'] 43 | varnames = engine.__init__.__code__.co_varnames 44 | if 'limit' in varnames and 'count' in varnames: 45 | attr = engine(q, limit, count) 46 | elif 'limit' in varnames: 47 | attr = engine(q, limit) 48 | else: 49 | attr = engine(q) 50 | 51 | attr.run_crawl() 52 | LINKS += attr.links 53 | PAGES += attr.pages 54 | 55 | def module_api(self): 56 | query = self.options['query'] 57 | limit = self.options['limit'] 58 | count = self.options['count'] 59 | engine = self.options['engine'].split(',') 60 | output = {'links': [], 'people': [], 'groups': [], 'hashtags': []} 61 | q_formats = { 62 | 'default_q': f"site:facebook.com {query}", 63 | 'millionshort_q': f'site:facebook.com "{query}"', 64 | 'qwant_q': f'site:facebook.com {query}' 65 | } 66 | 67 | self.thread(search, self.options['thread'], engine, query, q_formats, limit, count, meta['sources']) 68 | 69 | usernames = self.page_parse(PAGES).get_networks 70 | for _id in list(set(usernames.get('Facebook'))): 71 | _id = _id[_id.find('/')+1:] 72 | if _id not in output['people']: 73 | output['people'].append(_id) 74 | 75 | links = list(self.reglib().filter(r"https?://([A-z\-\.]+\.)?facebook\.com/", list(set(LINKS)))) 76 | for link in filter(lambda x: '/hashtag/' in x, links): 77 | tag = re.sub(r"https?://([A-z\-\.]+\.)?facebook\.com/hashtag/", '', link).replace('/', '') 78 | if re.search(r'^[\w\d_\-\/]+$', tag): 79 | if tag not in output['hashtags']: 80 | output['hashtags'].append(tag) 81 | 82 | for link in filter(lambda x: '/groups/' in x, links): 83 | link = re.sub(r"https?://([A-z\-\.]+\.)?facebook\.com/groups/", '', link).replace('/', '') 84 | if re.search(r'^[\w\d_\-\/]+$', link): 85 | if link not in output['groups']: 86 | output['groups'].append(link) 87 | 88 | output['links'] = links 89 | self.save_gather(output, 'search/facebook', query, output=self.options.get('output')) 90 | return output 91 | 92 | def module_run(self): 93 | self.alert_results(module_api(self)) 94 | --------------------------------------------------------------------------------