├── .github ├── FUNDING.yml └── workflows │ ├── check-for-dupe-url.yml │ ├── codeql-analysis.yml │ └── validate-json.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── INSTALLATION.md ├── LICENSE.md ├── README.md ├── SECURITY.md ├── __init__.py ├── pyproject.toml ├── sample.json ├── validate_whats_my_name.py ├── whats_my_name.py ├── whatsmyname.png ├── whatsmyname ├── __init__.py ├── app │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ └── args.py │ ├── config.py │ ├── extractors │ │ ├── __init__.py │ │ └── file_extractors.py │ ├── models │ │ ├── __init__.py │ │ ├── enums │ │ │ ├── __init__.py │ │ │ └── common.py │ │ └── schemas │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ ├── sites.py │ │ │ └── user_agent.py │ ├── resources │ │ └── user-agents.json │ ├── tasks │ │ ├── __init__.py │ │ └── process.py │ └── utilities │ │ ├── __init__.py │ │ └── formatters.py ├── main.py ├── requirements.txt └── tests │ ├── __init__.py │ ├── app │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ └── test_args.py │ ├── extractors │ │ ├── __init__.py │ │ └── test_file_extractors.py │ ├── models │ │ ├── __init__.py │ │ └── schemas │ │ │ └── __init__.py │ └── tasks │ │ ├── __init__.py │ │ └── test_process.py │ ├── data │ └── wmn-data.json │ └── utilities │ ├── __init__.py │ └── test_formatters.py ├── wmn-data-schema.json └── wmn-data.json /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | ko_fi: webbreacher 4 | -------------------------------------------------------------------------------- /.github/workflows/check-for-dupe-url.yml: -------------------------------------------------------------------------------- 1 | name: Check for Duplicate check_uri 2 | on: [pull_request] 3 | 4 | jobs: 5 | duplicate-url-checker: 6 | runs-on: ubuntu-latest 7 | defaults: 8 | run: 9 | shell: bash 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Check for duplicate in new JSON 13 | run: if [ $(grep "uri_check" wmn-data.json | cut -f4 -d'"' | uniq -D | head -1) ]; then echo "Found duplicate in wmn-data.json!"; exit 1; else echo "No duplicates found"; fi 14 | shell: bash 15 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '17 19 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /.github/workflows/validate-json.yml: -------------------------------------------------------------------------------- 1 | name: Validate JSON 2 | on: [pull_request] 3 | jobs: 4 | verify-json-validation: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v3 8 | 9 | - name: Validate JSON 10 | uses: docker://orrosenblatt/validate-json-action:latest 11 | env: 12 | INPUT_SCHEMA: /wmn-data-schema.json 13 | INPUT_JSONS: /wmn-data.json 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/usage.statistics.xml 10 | .idea/**/dictionaries 11 | .idea/**/shelf 12 | 13 | # Generated files 14 | .idea/**/contentModel.xml 15 | 16 | # Sensitive or high-churn files 17 | .idea/**/dataSources/ 18 | .idea/**/dataSources.ids 19 | .idea/**/dataSources.local.xml 20 | .idea/**/sqlDataSources.xml 21 | .idea/**/dynamic.xml 22 | .idea/**/uiDesigner.xml 23 | .idea/**/dbnavigator.xml 24 | 25 | # Gradle 26 | .idea/**/gradle.xml 27 | .idea/**/libraries 28 | 29 | # Gradle and Maven with auto-import 30 | # When using Gradle or Maven with auto-import, you should exclude module files, 31 | # since they will be recreated, and may cause churn. Uncomment if using 32 | # auto-import. 33 | # .idea/artifacts 34 | # .idea/compiler.xml 35 | # .idea/jarRepositories.xml 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | # *.iml 40 | # *.ipr 41 | 42 | # CMake 43 | cmake-build-*/ 44 | 45 | # Mongo Explorer plugin 46 | .idea/**/mongoSettings.xml 47 | 48 | # File-based project format 49 | *.iws 50 | 51 | # IntelliJ 52 | out/ 53 | 54 | # mpeltonen/sbt-idea plugin 55 | .idea_modules/ 56 | 57 | # JIRA plugin 58 | atlassian-ide-plugin.xml 59 | 60 | # Cursive Clojure plugin 61 | .idea/replstate.xml 62 | 63 | # Crashlytics plugin (for Android Studio and IntelliJ) 64 | com_crashlytics_export_strings.xml 65 | crashlytics.properties 66 | crashlytics-build.properties 67 | fabric.properties 68 | 69 | # Editor-based Rest Client 70 | .idea/httpRequests 71 | 72 | # Android studio 3.1+ serialized cache file 73 | .idea/caches/build_file_checksums.ser 74 | /.venv/ 75 | /Pipfile 76 | /Pipfile.lock 77 | /.idea/ 78 | 79 | # VS Code 80 | .vscode/ 81 | 82 | # Virtualenv 83 | venv/ 84 | .venv/ 85 | 86 | # Byte-compiled / optimized / DLL files 87 | __pycache__/ 88 | *.py[cod] 89 | 90 | # C extensions 91 | *.so 92 | 93 | # Distribution / packaging 94 | bin/ 95 | build/ 96 | develop-eggs/ 97 | dist/ 98 | eggs/ 99 | lib/ 100 | lib64/ 101 | parts/ 102 | sdist/ 103 | var/ 104 | *.egg-info/ 105 | .installed.cfg 106 | *.egg 107 | 108 | # Installer logs 109 | pip-log.txt 110 | pip-delete-this-directory.txt 111 | 112 | # Unit test / coverage reports 113 | .tox/ 114 | .coverage 115 | .cache 116 | nosetests.xml 117 | coverage.xml 118 | 119 | # Translations 120 | *.mo 121 | 122 | # Mr Developer 123 | .mr.developer.cfg 124 | .project 125 | .pydevproject 126 | 127 | # Rope 128 | .ropeproject 129 | 130 | # Django stuff: 131 | *.log 132 | *.pot 133 | 134 | # Sphinx documentation 135 | docs/_build/ 136 | 137 | /.env 138 | 139 | # Apple 140 | .DS_Store 141 | 142 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | `micah` `@` `spotlight-infosec.com`. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | You can contribute to the project in at least three different ways. 2 | 3 | ## Method 1. Non-technical 4 | 5 | Suggest a new site to be covered by the tool. 6 | 7 | How to do that: 8 | 9 | - Find the new site which has public profiles of people (with no authentication required) 10 | - Create a Github Issue and submit the link to an example profile. You can 11 | do that by navigating to [Issues](https://github.com/WebBreacher/WhatsMyName/issues) 12 | and clicking "New issue" 13 | 14 | 15 | ## Method 2. Technical, no programming skills required 16 | 17 | Requires a basic understanding of how Web/HTTP works (HTTP status codes, how 18 | what you see in a website translates to the source code). 19 | And some experience in Github contributions (fork, pull-request). 20 | 21 | Implement support for a new site or fix an existing implementation for a site. 22 | 23 | How to do that: 24 | 25 | - Among existing [Issues](https://github.com/WebBreacher/WhatsMyName/issues) 26 | or from somewhere else, establish which site you want to add 27 | - Using a web client of your choice (preferred `curl` or `wget`) perform 28 | simple requests for two different scenarios: existing profile, 29 | non-existing profile, e.g. 30 | ``` 31 | # existing 32 | curl https://twitter.com/WebBreacher 33 | 34 | # non-existing 35 | curl https://twitter.com/ThisDoesNotExistForSure504 36 | ``` 37 | - Observe the outcome for non-existing profile. Some sites use 404 (error), some use 302 38 | (redirection), some confusingly use 200 (OK) for profiles which don't exist, 39 | e.g. 40 | ``` 41 | $ curl https://github.com/ThisDoesNotExistForSure504 42 | [...] 43 | HTTP request sent, awaiting response... 404 Not Found 44 | ``` 45 | - Observe the outcome for existing profile. The response code should be 200. 46 | And among the downloaded source code find a text expected to be observed in 47 | all profiles. Avoid picking a text which might be dynamic (e.g. include the 48 | profile name). 49 | This seems right: 50 | ``` 51 |

You are browsing the profile of 52 | ``` 53 | This is too specific: 54 | ``` 55 |

You are browsing the profile of WebBreacher

56 | ``` 57 | - Add a section to `web_accounts_list.json` 58 | - Test your configuration by running the tool for a given site, e.g. 59 | ``` 60 | python3 ./web_accounts_list_checker.py -s my.new.site.Ive.added 61 | ``` 62 | - Submit a pull request with that change 63 | - There is also the `sample.json` file that you can use for testing. Simply replace the existing content with new data and test. 64 | 65 | ## Format of the JSON File 66 | 67 | ### Format of the New/Current JSON file 68 | 69 | The format of the `wmn-data.json` JSON was altered due to Issue #414. There are still 3 main elements: 70 | 71 | 1. License - The license for this project and its data 72 | 2. Authors - The people that have contributed to this project 73 | 3. Sites - This is the main data 74 | 75 | Within the `sites` elements, the format is as follows (with several parameters being optional): 76 | 77 | ```json 78 | ... 79 | { 80 | "name" : "name of the site", 81 | "uri_check" : "URI to check the site with the {account} string replaced by a username", 82 | "uri_pretty" : "if the check_uri is for an API, this OPTIONAL element can show a human-readable page", 83 | "post_body" : "[OPTIONAL] if non-empty, then this entry is an HTTP POST and the content of this field are the data", 84 | "invalid_chars" : "[OPTIONAL] if non-empty then checking apps should ignore or strip these characters from usernames", 85 | "e_code" : "the HTTP response code for a good 'account is there' response as an integer", 86 | "e_string" : "the string in the response that we look for for a good response", 87 | "m_string" : "this OPTIONAL string will only be in the response if there is no account found ", 88 | "m_code" : "the HTTP response code for a bad 'account is not there' response as an integer", 89 | "known" : ["a list of user accounts that can be used to test", "for user enumeration"], 90 | "cat" : "a category for what the site is mainly used for. These are found at the top of the JSON", 91 | "valid" : "this true or false boolean field is used to enable or disable this site element" 92 | }, 93 | ... 94 | ``` 95 | 96 | Here are examples of the site elements for both HTTP GET and HTTP POST entries: 97 | 98 | **HTTP GET entry:** 99 | 100 | ```json 101 | { 102 | "name" : "Example GET", 103 | "uri_check" : "https://www.example.com/load_profile_info.php?name={account}", 104 | "uri_pretty" : "https://www.test.com/profile/{account}", 105 | "e_code" : 200, 106 | "e_string" : "regist_at", 107 | "m_code" : 404, 108 | "m_string" : "Account not found", 109 | "known" : ["whoami", "johndoe"], 110 | "cat" : "images", 111 | "valid" : true 112 | }, 113 | ``` 114 | 115 | **HTTP POST entry:** 116 | 117 | ```json 118 | { 119 | "name" : "Example POST", 120 | "uri_check" : "https://www.example.com/interact_api/load_profile_info.php", 121 | "post_body" : "Name=Gareth+Wylie&Age=24&Formula=a%2Bb+%3D%3D+21", 122 | "e_code" : 200, 123 | "e_string" : "regist_at", 124 | "m_code" : 404, 125 | "m_string" : "Account not found", 126 | "known" : ["whoami", "johndoe"], 127 | "cat" : "images", 128 | "valid" : true 129 | }, 130 | ``` 131 | 132 | ## Method 3. Programming, enhancing the tool itself 133 | 134 | Basic python programming skills required. 135 | 136 | -------------------------------------------------------------------------------- /INSTALLATION.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Without Virtual Environment 4 | 5 | ``` 6 | git clone https://github.com/WebBreacher/WhatsMyName 7 | cd whatsmyname 8 | pip3 install -r requirements.txt 9 | ``` 10 | 11 | You are now ready to go for your first command, example below, and see above in the **Command Line Arguments section**: 12 | 13 | ``` 14 | python whats_my_name.py -u yooper 15 | ``` 16 | 17 | or 18 | 19 | ``` 20 | python3 whats_my_name.py -u yooper 21 | ``` 22 | 23 | 24 | ## With Virtual Environment 25 | 26 | If you prefer to use a virtual environment to isolate the script: 27 | 28 | ``` 29 | mkdir myvirtualenv 30 | cd myvirtualenv 31 | ``` 32 | 33 | example if you want to use Python 3.9: 34 | 35 | ``` 36 | python3.9 -m venv myvirtualenv 37 | ``` 38 | 39 | This command will activate your virtual environment: 40 | 41 | ``` 42 | source myvirtualenv/bin/activate 43 | ``` 44 | 45 | Once your virtual environment is activated: 46 | 47 | ``` 48 | git clone https://github.com/WebBreacher/WhatsMyName 49 | cd whatsmyname 50 | pip3 install -r requirements.txt 51 | ``` 52 | 53 | **Without installation of the the requirements.txt file (the Py libs), the script won't work.** 54 | 55 | You are now ready to go for your first command, example below, and see above in the **Command Line Arguments section**: 56 | 57 | ``` 58 | python whats_my_name.py -u yooper 59 | ``` 60 | 61 | or 62 | 63 | ``` 64 | python3 whats_my_name.py -u yooper 65 | ``` 66 | 67 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (C) 2022 Micah Hoffman 2 | 3 | This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License. 4 | To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative Commons, 5 | PO Box 1866, Mountain View, CA 94042, USA. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WhatsMyName 2 | 3 | This repository has the unified data required to perform user and username enumeration on various websites. Content is in a JSON file and can easily be used in other projects such as the ones below: 4 | 5 | ![whatsmyname](whatsmyname.png) 6 | 7 | ## Tools/Web Sites Using WhatsMyName 8 | 9 | * https://whatsmyname.app/ - [Chris Poulter](https://twitter.com/osintcombine) created this site which draws the project's JSON file into an easy to use web interface. 10 | * Filters for category and in search results. 11 | * Exports to CSV and other formats. 12 | * Pulls the latest version of the project's JSON file when run. 13 | * Submit a username in the URL using `https://whatsmyname.app/?q=USERNAME` like https://whatsmyname.app/?q=john 14 | * [Spiderfoot](https://github.com/smicallef/spiderfoot) uses this in the **sfp_account** module. There is also [this video](https://asciinema.org/a/295923) showing how to use this project using the Spiderfoot Command Line Interface (CLI). 15 | * [sn0int](https://github.com/kpcyrd/sn0int) downloads and uses the JSON file in the [kpcyrd/whatsmyname](https://sn0int.com/r/kpcyrd/whatsmyname) module, see https://twitter.com/sn0int/status/1228046880459907073 for details and instructions. 16 | * [WMN_screenshooter](https://github.com/swedishmike/WMN_screenshooter) a helper script that is based on `web_accounts_list_checker.py` and uses Selenium to try and grab screenshots of identified profile pages. 17 | * [LinkScope](https://github.com/AccentuSoft/LinkScope_Client) uses this in the **Whats My Name** resolution under the **Online Identity** category. 18 | * [Blackbird](https://github.com/p1ngul1n0/blackbird) uses the **Whats My Name** list in its search. 19 | * [WhatsMyName-Python](https://github.com/C3n7ral051nt4g3ncy/WhatsMyName-Python) **Whats My Name** simple Python script made by [@C3n7ral051nt4g3ncy](https://github.com/C3n7ral051nt4g3ncy) 20 | * [K2OSINT Bookmarklet](https://github.com/K2SOsint/Bookmarklets/blob/main/WhatsMyName.js) - Bookmarklet that lets you enter a username in a popup and then opens a new tab with the WMN results. 21 | 22 | 23 | ## Content 24 | 25 | * The https://github.com/WebBreacher/WhatsMyName/wiki/Problem-Removed-Sites page has websites that we have had in the project and are currently not working for some reason. We will retest those sites (in the future) and try to find detections. 26 | * If you would like to help with detections, we are happy to accept them via GitHub Pull Request or you can [create an issue](https://github.com/WebBreacher/WhatsMyName/issues) with the details of the site. 27 | * Want to suggest a site to be added? Use [this form](https://spotinfo.co/535y). 28 | 29 | ## Format 30 | 31 | See [CONTRIBUTING](CONTRIBUTING.md) 32 | 33 | ## Command Line Arguments 34 | If you just want to run this script to check user names on sites and don't wish to use it in combination with another tool (like https://whatsmyname.app or one noted above), then you can use the Python script [@yooper](https://github.com/yooper/) made for us, `whats_my_name.py` as shown below. 35 | 36 | There are quite a few command line options available: 37 | 38 | - Check for the user yooper, print out in a table format into console 39 | 40 | `python whats_my_name.py -u yooper -c social` 41 | 42 | - Check for the users yooper and maxim, defaults to outputing json to stdout, only returns the found results. 43 | 44 | `python whats_my_name.py -u yooper maxim` 45 | 46 | - Check for the users yooper and maxim, defaults to outputing json to stdout, returns the not found and found results. 47 | 48 | `python whats_my_name.py -u yooper maxim -a` 49 | 50 | - Check for the users yooper and maxim, defaults to outputing json to stdout, returns the sites where no matches were found. 51 | 52 | `python whats_my_name.py -u yooper maxim -n` 53 | 54 | - Check for the user yooper, on social sites 55 | 56 | `python whats_my_name.py -u yooper -c social` 57 | 58 | - Check for the user yooper, on social sites, using a different web browser agent 59 | 60 | `python whats_my_name.py -u yooper -c social --ua 1 ` 61 | 62 | - Check for the user yooper, print out in a csv format into console 63 | 64 | `python whats_my_name.py -u yooper -c social --format csv` 65 | 66 | - Check for the user yooper, print out in a json (default) format into console 67 | 68 | `python whats_my_name.py -u yooper -c social --format json` 69 | 70 | - Check for the user yooper, dump out response content and response headers, used for debugging purposes 71 | 72 | `python whats_my_name.py -u yooper -s zhihu --verbose --format csv` 73 | 74 | - Check for the user whether they exist or not and get the response from the server, used for debugging 75 | 76 | `python whats_my_name.py -u yooper -a -s zillow --verbose --format csv` 77 | 78 | # Social Media 79 | Come follow us for updates. We are on: 80 | * Mastodon at https://infosec.exchange/@whatsmyname 81 | * Twitter at https://twitter.com/whatsmynameproj 82 | 83 | # Installation 84 | Check the [INSTALLATION.md file](https://github.com/WebBreacher/WhatsMyName/blob/main/INSTALLATION.md) 85 | 86 | # License 87 | Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License. 88 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | To report a vulnerability, please create an [Issue](https://github.com/WebBreacher/WhatsMyName/issues) in this project or send an email to `micah` `@` `spotlight-infosec.com` 6 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append(os.path.dirname(__file__)) -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "what-is-my-name" 3 | version = "0.2.0" 4 | authors = [ 5 | { name="Micah Hoffman", email="micah@spotlight-infosec.com" }, 6 | { name="Dan Cardin", email="dcardin2007@gmail.com" }, 7 | ] 8 | description = "Enumerate over a list of websites and check for registered users by username." 9 | readme = "README.md" 10 | license = { file="LICENSE.md" } 11 | requires-python = ">=3.8" 12 | classifiers = [ 13 | "Programming Language :: Python :: 3", 14 | ] 15 | 16 | [project.urls] 17 | "Homepage" = "https://github.com/WebBreacher/WhatsMyName" 18 | "Bug Tracker" = "https://github.com/WebBreacher/WhatsMyName/issues" 19 | 20 | [build-system] 21 | requires = ["hatchling"] 22 | build-backend = "hatchling.build" 23 | 24 | -------------------------------------------------------------------------------- /sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "_comment1": "Do not submit your changes in this sample.json file as it is only for testing!", 3 | 4 | "categories" : ["archived","art","blog","business","coding","dating","finance","gaming","health", 5 | "hobby","images","misc","music","news","political","search","shopping","social", 6 | "tech","video","XXXPORNXXX"], 7 | 8 | "sites" : [ 9 | { 10 | "name" : "Example GET", 11 | "uri_check" : "https://www.example.com/load_profile_info.php?name={account}", 12 | "uri_pretty" : "https://www.test.com/profile/{account}", 13 | "e_code" : 200, 14 | "e_string" : "regist_at", 15 | "m_code" : 404, 16 | "m_string" : "Account not found", 17 | "known" : ["whoami", "johndoe"], 18 | "cat" : "images", 19 | "valid" : true 20 | }, 21 | { 22 | "name" : "Example POST", 23 | "uri_check" : "https://www.example.com/interact_api/load_profile_info.php", 24 | "post_body" : "Name=Gareth+Wylie&Age=24&Formula=a%2Bb+%3D%3D+21", 25 | "invalid_chars" : ".", 26 | "e_code" : 200, 27 | "e_string" : "regist_at", 28 | "m_code" : 404, 29 | "m_string" : "Account not found", 30 | "known" : ["whoami", "johndoe"], 31 | "cat" : "images", 32 | "valid" : true 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /validate_whats_my_name.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from whatsmyname.main import validate_whats_my_name 3 | 4 | if __name__ == "__main__": 5 | # execute only if run as a script 6 | validate_whats_my_name() 7 | -------------------------------------------------------------------------------- /whats_my_name.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from whatsmyname.main import check_for_presence 3 | 4 | if __name__ == "__main__": 5 | # execute only if run as a script 6 | check_for_presence() 7 | -------------------------------------------------------------------------------- /whatsmyname.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname.png -------------------------------------------------------------------------------- /whatsmyname/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/cli/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/cli/args.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os.path 3 | import string 4 | import tempfile 5 | from argparse import ArgumentParser 6 | from random import choice 7 | from typing import List 8 | 9 | from whatsmyname.app.config import project_path, resource_dir, main_dir 10 | from whatsmyname.app.extractors.file_extractors import user_agent_extractor 11 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 12 | from whatsmyname.app.models.schemas.user_agent import UserAgentSchema 13 | 14 | logger = logging.getLogger() 15 | 16 | 17 | def get_default_args() -> ArgumentParser: 18 | """ 19 | Returns an argument parser with a set of defaults 20 | :return: 21 | """ 22 | parser = ArgumentParser( 23 | description="This standalone script will look up username using the JSON file" 24 | " or will run a check of the JSON file for bad detection strings.") 25 | parser.add_argument('-u', '--usernames', nargs='*', help='[OPTIONAL] If this param is passed then this script will perform the ' 26 | 'lookups against the given user name instead of running checks against ' 27 | 'the JSON file.') 28 | parser.add_argument('-in', '--input_file', nargs='?', 29 | help="[OPTIONAL] Uses a specified file for checking the websites") 30 | parser.add_argument('-s', '--sites', nargs='*', 31 | help='[OPTIONAL] If this parameter is passed the script will check only the named site or list of sites.') 32 | parser.add_argument('-a', '--all', help="Display all results.", action="store_true", default=False) 33 | parser.add_argument('-n', '--not_found', help="Display not found results", action="store_true", default=False) 34 | parser.add_argument('-d', '--debug', help="Enable debug output", action="store_true", default=False) 35 | parser.add_argument('-o', '--output_file', nargs='?', help="[OPTIONAL] Uses a specified output file ") 36 | parser.add_argument('-t', '--timeout', nargs='?', help='[OPTIONAL] Timeout per connection, default is 60 seconds.', default=60) 37 | parser.add_argument('-prt', '--per_request_time', nargs='?', help='[Optional] Timeout per request, default is 15 seconds', default=15) 38 | parser.add_argument('-fmt', '--format', nargs='?', help='[Optional] Format options are json, csv, or table', default='table') 39 | parser.add_argument('-v', '--verbose', help="Enable verbose output", action="store_true", default=False) 40 | parser.add_argument('-fr', '--follow_redirects', help="Follow redirects", action="store_true", default=False) 41 | parser.add_argument('-mr', '--max_redirects', nargs='?', help='[OPTIONAL] Max Redirects, default is 10 ', default=10) 42 | parser.add_argument('-c', '--category', nargs='?', help='[OPTIONAL] Filter by site category ', default=None) 43 | parser.add_argument('-rv', '--random_validate', action="store_true", help='[OPTIONAL] Upon success, validate using random username', default=False) 44 | parser.add_argument('-ua', '--user_agent_id', nargs='?', help='[OPTIONAL] Select the user agent platform to use. ', default=1) 45 | return parser 46 | 47 | 48 | def arg_parser(arguments: ArgumentParser) -> CliOptionsSchema: 49 | """ 50 | Parse the passed in arguments into something the schema will understand 51 | :param arguments: ArgumentParser 52 | :return: 53 | """ 54 | parsed = vars(arguments) 55 | schema: CliOptionsSchema = CliOptionsSchema(**parsed) 56 | 57 | # set global logger levels 58 | if schema.debug: 59 | logger.setLevel(logging.DEBUG) 60 | else: 61 | logger.setLevel(logging.INFO) 62 | 63 | if schema.input_file and not os.path.isfile(schema.input_file): 64 | logger.error('Input file does not exist %s', schema.input_file) 65 | raise Exception(f'Input file does not exist ${schema.input_file}.') 66 | 67 | if not schema.input_file: 68 | input_file: str = os.path.join(main_dir, 'wmn-data.json') 69 | schema.input_file = input_file 70 | logger.debug('Loading default input file %s', input_file) 71 | 72 | if schema.output_file: 73 | schema.output_stdout = False 74 | 75 | if not schema.output_file: 76 | letters = string.ascii_lowercase 77 | schema.output_file = os.path.join(tempfile.gettempdir(), ''.join(choice(letters) for _ in range(10))) 78 | schema.output_stdout = True 79 | 80 | if schema.random_validate: 81 | logger.debug('Randomly generated username is %s', schema.random_username) 82 | 83 | if not schema.usernames: 84 | schema.validate_knowns = True 85 | 86 | user_agents: List[UserAgentSchema] = user_agent_extractor(os.path.join(resource_dir, 'user-agents.json')) 87 | schema.user_agent = next(obj.user_agent for obj in user_agents if obj.id == schema.user_agent_id) 88 | 89 | if not schema.user_agent: 90 | raise Exception('User agent platform %s does not exist', schema.user_agent) 91 | 92 | return schema 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /whatsmyname/app/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | from pathlib import Path 4 | import random 5 | 6 | project_path = os.path.dirname(os.path.realpath(__file__)) 7 | resource_dir = os.path.join(project_path, 'resources') 8 | main_dir = Path(__file__).parent.parent.parent 9 | 10 | 11 | def random_username(length: int = 8) -> str: 12 | return ''.join( 13 | random.choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for x in range(length)) 14 | -------------------------------------------------------------------------------- /whatsmyname/app/extractors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/extractors/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/extractors/file_extractors.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List 3 | 4 | from pydantic import parse_obj_as 5 | 6 | from whatsmyname.app.models.schemas.sites import SiteSchema, SitesConfigurationSchema 7 | from whatsmyname.app.models.schemas.user_agent import UserAgentSchema 8 | 9 | 10 | def site_file_extractor(file_path: str) -> List[SiteSchema]: 11 | """ 12 | Opens a json file, validates it and returns the Site instance 13 | :param file_path: str 14 | :return: List[Site] 15 | """ 16 | with open(file_path) as json_file: 17 | json_data = json.load(json_file) 18 | sites_configuration = SitesConfigurationSchema(**json_data) 19 | return sites_configuration.sites 20 | 21 | 22 | def user_agent_extractor(file_path: str) -> List[UserAgentSchema]: 23 | """ 24 | Opens a json file, validates it and returns a set of user agent instances 25 | :param file_path: str 26 | :return: List[UserAgent] 27 | """ 28 | with open(file_path) as json_file: 29 | json_data = json.load(json_file) 30 | return parse_obj_as(List[UserAgentSchema], json_data) 31 | -------------------------------------------------------------------------------- /whatsmyname/app/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/models/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/models/enums/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/models/enums/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/models/enums/common.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class OutputFormat(str, Enum): 5 | CSV = 'csv' 6 | JSON = 'json' 7 | TABLE = 'table' 8 | 9 | -------------------------------------------------------------------------------- /whatsmyname/app/models/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/models/schemas/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/models/schemas/cli.py: -------------------------------------------------------------------------------- 1 | """ Holds the schema for all the command line options """ 2 | 3 | from typing import Optional, List 4 | from pydantic import BaseModel, Field 5 | from whatsmyname.app.models.enums.common import OutputFormat 6 | 7 | 8 | class CliOptionsSchema(BaseModel): 9 | all: bool = False 10 | input_file: Optional[str] = None 11 | debug: bool = False 12 | output: bool = False 13 | sites: Optional[List[str]] = [] 14 | category: Optional[str] = None 15 | string_error: bool = False 16 | usernames: Optional[List[str]] = [] 17 | follow_redirects: bool = False 18 | timeout: int = 30 19 | format: OutputFormat = OutputFormat.JSON 20 | output_file: Optional[str] = None 21 | output_stdout: bool = False 22 | per_request_timeout: int = 5 23 | not_found: bool = False 24 | verbose: bool = False 25 | max_redirects: int = 10 26 | random_validate: bool = False 27 | random_username: Optional[str] = None, 28 | user_agent_id: int = 1 29 | user_agent: Optional[str] = None 30 | capture_errors: bool = False 31 | capture_error_directory: Optional[str] = None 32 | validate_knowns: bool = False 33 | add_error_hints: bool = False 34 | word_wrap_length: int = 50 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /whatsmyname/app/models/schemas/sites.py: -------------------------------------------------------------------------------- 1 | """Site Schemas""" 2 | 3 | from datetime import datetime 4 | from typing import List, Optional 5 | 6 | from pydantic import BaseModel, model_validator, Field 7 | 8 | 9 | class SiteOutputSchema(BaseModel): 10 | """For exporting to csv, json""" 11 | name: str 12 | category: str 13 | raw_response_data: Optional[str] = None 14 | http_status_code: int 15 | generated_uri: str 16 | username: str 17 | user_agent: str 18 | uri_pretty: Optional[str] = None 19 | error_hint: Optional[str] = None 20 | response_headers: Optional[str] = None 21 | 22 | 23 | class SiteSchema(BaseModel): 24 | name: str 25 | e_code: int 26 | e_string: str 27 | m_string: str 28 | m_code: int 29 | known: List[str] 30 | category: str = Field(alias='cat') 31 | valid: bool 32 | post_body: Optional[str] = None 33 | uri_check: str 34 | request_method: str = None 35 | username: str = None 36 | generated_uri: str = None 37 | http_status_code: int = -1 38 | last_checked_on: Optional[datetime] = None 39 | comment: Optional[str] = None 40 | raw_response_data: Optional[str] = None 41 | cloudflare_enabled: bool = False 42 | user_agent: Optional[str] = None 43 | uri_pretty: Optional[str] = None 44 | error_hint: Optional[str] = None 45 | response_headers: Optional[str] = None 46 | invalid_chars: Optional[str] = None 47 | 48 | @model_validator(mode='after') 49 | def set_request_method(self) -> "SiteSchema": 50 | if self.post_body: 51 | self.request_method = 'POST' 52 | else: 53 | self.request_method = 'GET' 54 | return self 55 | 56 | 57 | class SitesConfigurationSchema(BaseModel): 58 | license: List[str] = [] 59 | authors: List[str] = [] 60 | categories: List[str] = [] 61 | sites: List[SiteSchema] = [] 62 | -------------------------------------------------------------------------------- /whatsmyname/app/models/schemas/user_agent.py: -------------------------------------------------------------------------------- 1 | """ Schema class for the user agent """ 2 | from pydantic import BaseModel 3 | 4 | 5 | class UserAgentSchema(BaseModel): 6 | user_agent: str 7 | platform: str 8 | id: int 9 | -------------------------------------------------------------------------------- /whatsmyname/app/resources/user-agents.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 1, 4 | "platform": "Chrome on Windows", 5 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36" 6 | }, 7 | { 8 | "id": 2, 9 | "platform": "Firefox on Windows", 10 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0" 11 | }, 12 | { 13 | "id": 3, 14 | "platform": "Safari on macOS", 15 | "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6 Safari/605.1.15" 16 | }, 17 | { 18 | "id": 4, 19 | "platform": "Chrome on macOS", 20 | "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0 Safari/537.36" 21 | }, 22 | { 23 | "id": 5, 24 | "platform": "Firefox on macOS", 25 | "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12.5; rv:103.0) Gecko/20100101 Firefox/103.0" 26 | }, 27 | { 28 | "id": 6, 29 | "platform": "Edge on Windows", 30 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/104.0.1293.54" 31 | }, 32 | { 33 | "id": 7, 34 | "platform": "Internet-Explorer on Windows", 35 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko" 36 | } 37 | ] -------------------------------------------------------------------------------- /whatsmyname/app/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/tasks/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/tasks/process.py: -------------------------------------------------------------------------------- 1 | """ Provides url fetching and data splitting functionality """ 2 | import logging 3 | import os 4 | import tempfile 5 | from asyncio import gather, ensure_future 6 | from typing import List, Dict 7 | import json 8 | 9 | import aiohttp 10 | from aiohttp import ClientSession, ClientConnectionError, TCPConnector, ClientTimeout 11 | from multidict import CIMultiDictProxy 12 | 13 | from whatsmyname.app.extractors.file_extractors import site_file_extractor 14 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 15 | from whatsmyname.app.models.schemas.sites import SiteSchema 16 | from whatsmyname.app.utilities.formatters import to_json 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | def get_sites_list(cli_options: CliOptionsSchema) -> List[SiteSchema]: 22 | """ 23 | Returns all the sites, or some of the sites 24 | :param cli_options: 25 | :return: List[Schema] 26 | """ 27 | sites: List[SiteSchema] = site_file_extractor(cli_options.input_file) 28 | 29 | # filter invalid sites 30 | sites = list(filter(lambda site: site.valid, sites)) 31 | 32 | # assign the user agent 33 | for site in sites: 34 | site.user_agent = cli_options.user_agent 35 | 36 | if cli_options.category: 37 | sites = list(filter(lambda site: site.category.lower() == cli_options.category.lower(), sites)) 38 | 39 | if cli_options.sites: 40 | filtered_sites: List[SiteSchema] = [] 41 | site_name: str 42 | for site_name in cli_options.sites: 43 | site: SiteSchema 44 | for site in sites: 45 | if site.name.lower() == site_name.lower(): 46 | filtered_sites.append(site) 47 | if not filtered_sites: 48 | raise Exception('No sites with id(s) ' + ' '.join(cli_options.sites) + ' used input file ' + cli_options.input_file) 49 | 50 | return filtered_sites 51 | else: 52 | return sites 53 | 54 | 55 | def get_validated_site_list(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> List[SiteSchema]: 56 | """ 57 | Return the list of sites using the known usernames 58 | :param sites: 59 | :param cli_options: 60 | :return: 61 | """ 62 | valid_username_site_list: List[SiteSchema] = [] 63 | for site in sites: 64 | valid_username_site_list.extend(generate_username_sites(site.known, [site])) 65 | return valid_username_site_list 66 | 67 | 68 | def generate_username_sites(usernames: List[str], sites: List[SiteSchema]) -> List[SiteSchema]: 69 | """ 70 | Generate sites schemas from the usernames list 71 | :param usernames: 72 | :param sites: 73 | :return: 74 | """ 75 | username: str 76 | user_site_map: Dict[str, List[SiteSchema]] = {} 77 | for username in usernames: 78 | username_has_dot: bool = '.' in username 79 | site: SiteSchema 80 | for site in sites: 81 | # addresses github issue #55 82 | if (site.uri_check.startswith('http://{account}') or site.uri_check.startswith('https://{account}')) and username_has_dot: 83 | logger.debug('Skipping site %s, with username %s', site.uri_check, username) 84 | continue 85 | 86 | site_clone: SiteSchema = site.copy(deep=True) 87 | site_clone.username = username 88 | 89 | # Fixes issue #673 90 | if site_clone.invalid_chars: 91 | for remove_char in site_clone.invalid_chars: 92 | username = username.replace(remove_char, '') 93 | site_clone.username = username.replace(remove_char, '') 94 | 95 | site_clone.generated_uri = site_clone.uri_check.replace('{account}', username) 96 | 97 | if not user_site_map.get(username): 98 | user_site_map[username] = [] 99 | user_site_map[username].append(site_clone) 100 | # flatten dictionary into single list 101 | big_list_of_sites: List[SiteSchema] = [] 102 | for username, list_of_sites in user_site_map.items(): 103 | big_list_of_sites += list_of_sites 104 | 105 | return big_list_of_sites 106 | 107 | 108 | async def process_cli(cli_options: CliOptionsSchema) -> List[SiteSchema]: 109 | """ 110 | Main function for fetching and processing the website requests 111 | :param cli_options: 112 | :return: 113 | """ 114 | sites: List[SiteSchema] 115 | if cli_options.validate_knowns: 116 | sites = get_validated_site_list(cli_options, get_sites_list(cli_options)) 117 | else: 118 | # check the number of usernames we must validate 119 | sites = generate_username_sites(cli_options.usernames, get_sites_list(cli_options)) 120 | return await request_controller(cli_options, sites) 121 | 122 | 123 | def filter_list_by(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> List[SiteSchema]: 124 | """ 125 | By default, only return sites that had a successful hit. 126 | :param cli_options: 127 | :param sites: 128 | :return: 129 | """ 130 | 131 | if cli_options.all: 132 | return sites 133 | 134 | filtered_sites = [] 135 | for site in sites: 136 | if site.raw_response_data: 137 | filtered_sites.append(site) 138 | 139 | site: SiteSchema 140 | if cli_options.not_found: 141 | return list(filter(lambda site: site.http_status_code == site.m_code and site.m_string in site.raw_response_data, filtered_sites)) 142 | 143 | return list(filter(lambda site: site.http_status_code == site.e_code and site.e_string in site.raw_response_data, filtered_sites)) 144 | 145 | 146 | async def request_controller(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> List[SiteSchema]: 147 | """Initiates all the web requests""" 148 | connector: TCPConnector = aiohttp.TCPConnector(ssl=False) 149 | client_timeout = ClientTimeout(total=None, sock_connect=cli_options.timeout, sock_read=cli_options.timeout) 150 | async with aiohttp.ClientSession(connector=connector, timeout=client_timeout) as session: 151 | site: SiteSchema 152 | tasks = [ensure_future(request_worker(session, cli_options, site)) for site in sites] 153 | results = await gather(*tasks) 154 | 155 | return results 156 | 157 | 158 | async def request_worker(session: ClientSession, cli_options: CliOptionsSchema, site: SiteSchema) -> SiteSchema: 159 | """ 160 | Makes the individual requests to web servers 161 | :param session: 162 | :param cli_options: 163 | :param site: 164 | :return: 165 | """ 166 | 167 | headers = { 168 | 'User-Agent': site.user_agent 169 | } 170 | if site.post_body: 171 | try: 172 | post_body_altered: str = site.post_body.replace('{account}', site.username) 173 | form_data = {x[0] : x[1] for x in [x.split("=") for x in post_body_altered[1:].split("&") ]} 174 | async with session.post(site.generated_uri, 175 | data=form_data, 176 | timeout=cli_options.per_request_timeout, 177 | allow_redirects=cli_options.follow_redirects, 178 | headers=headers 179 | ) as response: 180 | site.http_status_code = response.status 181 | site.raw_response_data = await response.text() 182 | site.response_headers = json.dumps({str(key): value for key, value in response.headers.items()}) 183 | return site 184 | 185 | except ClientConnectionError as cce: 186 | logger.error('Site Connection Error %s', site.name, exc_info=False) 187 | site.http_status_code = -1 188 | finally: 189 | return site 190 | 191 | else: 192 | try: 193 | async with session.get(site.generated_uri, 194 | timeout=cli_options.per_request_timeout, 195 | allow_redirects=cli_options.follow_redirects, 196 | headers=headers 197 | ) as response: 198 | site.http_status_code = response.status 199 | site.raw_response_data = await response.text() 200 | site.response_headers = json.dumps({str(key): value for key, value in response.headers.items()}) 201 | return site 202 | 203 | except ClientConnectionError as cce: 204 | logger.error('Site Connection Error %s', site.name, exc_info=False) 205 | site.http_status_code = -1 206 | finally: 207 | return site 208 | -------------------------------------------------------------------------------- /whatsmyname/app/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/app/utilities/__init__.py -------------------------------------------------------------------------------- /whatsmyname/app/utilities/formatters.py: -------------------------------------------------------------------------------- 1 | """Format the data and write it to a temporary file.""" 2 | from typing import List 3 | import csv 4 | 5 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 6 | from whatsmyname.app.models.schemas.sites import SiteSchema, SiteOutputSchema 7 | from terminaltables import AsciiTable 8 | 9 | 10 | def to_csv(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> None: 11 | 12 | field_names: List[str] = ['name', 'category', 'http_status_code', 'generated_uri', 'username'] 13 | if cli_options.verbose: 14 | field_names.append('raw_response_data') 15 | field_names.append('user_agent') 16 | field_names.append('response_headers') 17 | 18 | if cli_options.add_error_hints: 19 | field_names.append('error_hint') 20 | 21 | with open(cli_options.output_file, "w") as fp: 22 | writer = csv.DictWriter(fp, fieldnames=field_names) 23 | writer.writeheader() 24 | site: SiteSchema 25 | for site in sites: 26 | site_output: SiteOutputSchema = SiteOutputSchema(**site.dict()) 27 | writer.writerow(site_output.dict(include=dict.fromkeys(field_names, True))) 28 | 29 | 30 | def to_json(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> None: 31 | field_names = {'name': True, 'category': True, 'http_status_code': True, 'generated_uri': True, 'username': True} 32 | if cli_options.verbose: 33 | field_names['raw_response_data'] = True 34 | field_names['user_agent'] = True 35 | field_names['response_headers'] = True 36 | 37 | if cli_options.add_error_hints: 38 | field_names['error_hint'] = True 39 | 40 | with open(fix_file_name(cli_options.output_file), "w") as fp: 41 | fp.write("[") 42 | site: SiteSchema 43 | for idx, site in enumerate(sites): 44 | site_output: SiteOutputSchema = SiteOutputSchema(**site.dict()) 45 | fp.write(site_output.json(include=field_names)) 46 | if idx < len(sites) - 1: 47 | fp.write(",") 48 | fp.write("]") 49 | 50 | 51 | def fix_file_name(file_name: str) -> str: 52 | return file_name.encode('ascii', 'ignore').decode('ascii') 53 | 54 | 55 | def to_table(cli_options: CliOptionsSchema, sites: List[SiteSchema]) -> None: 56 | """Output a table to the terminal""" 57 | table_data: List[List[str]] = [ 58 | ['Site Name', 'Url', 'Category', 'Result'] 59 | ] 60 | 61 | if cli_options.add_error_hints: 62 | table_data[0].append('Error Hint') 63 | 64 | for site in sites: 65 | pretty_url = site.uri_pretty.replace("{account}", site.username) if site.uri_pretty else site.generated_uri 66 | pretty_url = pretty_url[0:cli_options.word_wrap_length] 67 | row = [site.name, pretty_url, site.category, site.http_status_code] 68 | if cli_options.add_error_hints: 69 | row.append(site.error_hint if site.error_hint else '') 70 | table_data.append(row) 71 | 72 | table = AsciiTable(table_data) 73 | with open(fix_file_name(cli_options.output_file), "w") as fp: 74 | fp.write(table.table) 75 | fp.write("\n") 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /whatsmyname/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import List 3 | 4 | from whatsmyname.app.cli.args import get_default_args, arg_parser 5 | from whatsmyname.app.config import random_username 6 | from whatsmyname.app.models.enums.common import OutputFormat 7 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 8 | from whatsmyname.app.models.schemas.sites import SiteSchema 9 | from whatsmyname.app.tasks.process import process_cli, filter_list_by, generate_username_sites, \ 10 | request_controller, get_sites_list 11 | from whatsmyname.app.utilities.formatters import to_json, to_csv, to_table 12 | 13 | 14 | async def start_check_for_presence(provided_args=None): 15 | 16 | argparse = get_default_args() 17 | if provided_args is None: 18 | cli_options: CliOptionsSchema = arg_parser(argparse.parse_args()) 19 | else: 20 | cli_options: CliOptionsSchema = arg_parser(argparse.parse_args(provided_args)) 21 | sites: List[SiteSchema] = await process_cli(cli_options) 22 | 23 | # filter the sites 24 | sites = filter_list_by(cli_options, sites) 25 | 26 | if cli_options.random_validate: 27 | cli_options.usernames = [cli_options.random_username] 28 | sites = await process_cli(cli_options) 29 | 30 | if cli_options.format == OutputFormat.JSON: 31 | to_json(cli_options, sites) 32 | elif cli_options.format == OutputFormat.CSV: 33 | to_csv(cli_options, sites) 34 | elif cli_options.format == OutputFormat.TABLE: 35 | to_table(cli_options, sites) 36 | 37 | if cli_options.output_stdout: 38 | with open(cli_options.output_file, 'r') as fp: 39 | lines: List[str] = fp.readlines() 40 | for line in lines: 41 | print('{}'.format(line.strip())) 42 | 43 | 44 | def check_for_presence() -> None: 45 | asyncio.run(start_check_for_presence()) 46 | 47 | 48 | def _validate_site(site: SiteSchema) -> None: 49 | code_match: bool = site.http_status_code == site.e_code 50 | # a site will return no response in some cases 51 | if not site.raw_response_data: 52 | site.error_hint = f'No response data from site. {site.e_string}' 53 | return 54 | 55 | string_match: bool = site.raw_response_data.find(site.e_string) >= 0 56 | if code_match and string_match: 57 | site.error_hint = None 58 | elif code_match and not string_match: 59 | site.error_hint = f'Bad Detection String {site.e_string}' 60 | elif not code_match and string_match: 61 | site.error_hint = f'Bad Detection Response Code. Received {site.http_status_code} Expected {site.e_code}.' 62 | else: 63 | site.error_hint = f'Bad Code and String' 64 | 65 | 66 | async def start_validate_whats_my_name() -> None: 67 | """Validates if a site is returning a correct set of values for an unknown and a known user.""" 68 | argparse = get_default_args() 69 | cli_options: CliOptionsSchema = arg_parser(argparse.parse_args()) 70 | cli_options.add_error_hints = True 71 | cli_options.debug = True 72 | cli_options.verbose = True 73 | 74 | # for the first pass, we use a random user name 75 | cli_options.usernames = [random_username()] 76 | 77 | sites: List[SiteSchema] = \ 78 | await request_controller(cli_options, generate_username_sites(cli_options.usernames, get_sites_list(cli_options))) 79 | 80 | # this is the set of sites that did match the expected criteria 81 | invalid_sites = list(filter(lambda site: site.http_status_code != site.m_code or site.m_string not in (site.raw_response_data or ''), sites)) 82 | 83 | # retest these sites with a known user 84 | cli_options.sites = [site.name for site in invalid_sites] 85 | 86 | sites_with_known: List[SiteSchema] = [] 87 | for site in invalid_sites: 88 | sites_with_known.extend(generate_username_sites([site.known[0]], [site])) 89 | 90 | # filter the sites 91 | sites = await request_controller(cli_options, sites_with_known) 92 | sites = list(filter(lambda site: site.http_status_code != site.e_code or site.e_string not in (site.raw_response_data or ''), sites)) 93 | 94 | for site in sites: 95 | _validate_site(site) 96 | 97 | if cli_options.format == OutputFormat.JSON: 98 | to_json(cli_options, sites) 99 | elif cli_options.format == OutputFormat.CSV: 100 | to_csv(cli_options, sites) 101 | elif cli_options.format == OutputFormat.TABLE: 102 | to_table(cli_options, sites) 103 | 104 | if cli_options.output_stdout: 105 | with open(cli_options.output_file, 'r') as fp: 106 | lines: List[str] = fp.readlines() 107 | for line in lines: 108 | print('{}'.format(line.strip())) 109 | 110 | 111 | def validate_whats_my_name() -> None: 112 | asyncio.run(start_validate_whats_my_name()) 113 | -------------------------------------------------------------------------------- /whatsmyname/requirements.txt: -------------------------------------------------------------------------------- 1 | aiodns 2 | aiohttp 3 | aiohttp-socks 4 | aiosignal 5 | async-timeout 6 | attrs 7 | beautifulsoup4 8 | bleach 9 | build 10 | CacheControl 11 | cachy 12 | cchardet 13 | certifi 14 | cffi 15 | charset-normalizer 16 | cleo 17 | clikit 18 | commonmark 19 | coverage 20 | crashtest 21 | cryptography 22 | dataclasses 23 | distlib 24 | docutils 25 | dulwich 26 | elastic-transport 27 | elasticsearch 28 | exceptiongroup 29 | fake-useragent 30 | filelock 31 | frozenlist 32 | geographiclib 33 | geopy 34 | googletransx 35 | html5lib 36 | idna 37 | importlib-metadata 38 | importlib-resources 39 | iniconfig 40 | jaraco.classes 41 | jeepney 42 | jsonschema 43 | keyring 44 | lockfile 45 | more-itertools 46 | msgpack 47 | multidict 48 | numpy 49 | packaging 50 | pandas 51 | pastel 52 | pep517 53 | pexpect 54 | pkginfo 55 | pkgutil_resolve_name 56 | platformdirs 57 | pluggy 58 | poetry 59 | poetry-core 60 | poetry-plugin-export 61 | ptyprocess 62 | py 63 | pycares 64 | pycparser 65 | pydantic 66 | Pygments 67 | pylev 68 | pyparsing 69 | pyrsistent 70 | PySocks 71 | pytest 72 | pytest-asyncio 73 | pytest-cov 74 | python-dateutil 75 | python-socks 76 | pytz 77 | rapidfuzz 78 | readme-renderer 79 | requests 80 | requests-toolbelt 81 | rfc3986 82 | rich 83 | schedule 84 | SecretStorage 85 | shellingham 86 | six 87 | soupsieve 88 | terminaltables 89 | tomli 90 | tomlkit 91 | trove-classifiers 92 | twine 93 | typing_extensions 94 | urllib3 95 | virtualenv 96 | webencodings 97 | yarl 98 | zipp 99 | -------------------------------------------------------------------------------- /whatsmyname/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/cli/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/cli/test_args.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from typing import Dict 3 | 4 | import pytest 5 | 6 | from whatsmyname.app.cli.args import arg_parser, get_default_args 7 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 8 | from whatsmyname.app.tasks.process import process_cli 9 | 10 | 11 | def test_arg_parse_basic() -> None: 12 | 13 | args: ArgumentParser = get_default_args() 14 | schema: CliOptionsSchema = arg_parser(args.parse_args(['-u', 'yooper', '-d'])) 15 | assert schema.debug 16 | 17 | 18 | def test_arg_parse_sites() -> None: 19 | 20 | args: ArgumentParser = get_default_args() 21 | schema: CliOptionsSchema = arg_parser(args.parse_args(['-d', '--sites', '3DNews', 'CNN', '-u', 'yooper'])) 22 | assert schema.debug 23 | assert schema.sites == ['3DNews', 'CNN'] 24 | assert schema.usernames == ['yooper'] 25 | 26 | 27 | def test_bad_input_file() -> None: 28 | args: ArgumentParser = get_default_args() 29 | with pytest.raises(Exception): 30 | schema: CliOptionsSchema = arg_parser(args.parse_args(['-u', 'yooper', '--sites', 'allesovercrypto', '--input_file','nope.txt'])) 31 | -------------------------------------------------------------------------------- /whatsmyname/tests/app/extractors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/extractors/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/extractors/test_file_extractors.py: -------------------------------------------------------------------------------- 1 | """Test loading json files into objects 2 | """ 3 | import os 4 | from argparse import ArgumentParser 5 | 6 | from commonmark.blocks import List 7 | 8 | from whatsmyname.app.cli.args import get_default_args, arg_parser 9 | from whatsmyname.app.config import resource_dir 10 | from whatsmyname.app.extractors.file_extractors import site_file_extractor, user_agent_extractor 11 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 12 | from whatsmyname.app.models.schemas.sites import SiteSchema 13 | from whatsmyname.app.models.schemas.user_agent import UserAgentSchema 14 | 15 | 16 | def test_site_file_extractor(): 17 | args: ArgumentParser = get_default_args() 18 | schema: CliOptionsSchema = arg_parser(args.parse_args(['-u', 'yooper'])) 19 | sites: List[SiteSchema] = site_file_extractor(schema.input_file) 20 | assert len(sites) > 0 21 | 22 | 23 | def test_user_agent_extractor(): 24 | user_agents: List[UserAgentSchema] = user_agent_extractor(os.path.join(resource_dir, 'user-agents.json')) 25 | assert len(user_agents) > 0 -------------------------------------------------------------------------------- /whatsmyname/tests/app/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/models/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/models/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/models/schemas/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/app/tasks/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/app/tasks/test_process.py: -------------------------------------------------------------------------------- 1 | """ Test the functionality within the process module """ 2 | from argparse import ArgumentParser 3 | from typing import List 4 | 5 | from whatsmyname.app.cli.args import arg_parser, get_default_args 6 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 7 | from whatsmyname.app.models.schemas.sites import SiteSchema 8 | from whatsmyname.app.tasks.process import get_sites_list, generate_username_sites, filter_list_by 9 | 10 | 11 | def get_default_cli_options() -> CliOptionsSchema: 12 | args: ArgumentParser = get_default_args() 13 | return arg_parser(args.parse_args(['-u', 'yooper', '-d', '--verbose'])) 14 | 15 | 16 | def test_get_default_sites_list() -> None: 17 | cli_options: CliOptionsSchema = get_default_cli_options() 18 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 19 | assert len(sites_list) > 100 20 | 21 | 22 | def test_filtered_sites_list() -> None: 23 | cli_options: CliOptionsSchema = get_default_cli_options() 24 | cli_options.sites = ['Zillow'] 25 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 26 | assert len(sites_list) == 1 27 | 28 | 29 | def test_generate_username_sites() -> None: 30 | cli_options: CliOptionsSchema = get_default_cli_options() 31 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 32 | count = len(sites_list) 33 | new_sites: List[SiteSchema] = generate_username_sites(['yooper', 'webbreacher'], sites_list) 34 | assert (count*2) == len(new_sites) 35 | 36 | 37 | def test_generate_username_has_dot() -> None: 38 | cli_options: CliOptionsSchema = get_default_cli_options() 39 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 40 | new_sites: List[SiteSchema] = generate_username_sites(['yoo.per'], sites_list) 41 | assert len(new_sites) > 400 42 | 43 | 44 | def test_generate_username_has_invalid_chars() -> None: 45 | cli_options: CliOptionsSchema = get_default_cli_options() 46 | sites_list: List[SiteSchema] = [s for s in get_sites_list(cli_options) if s.name == 'Wanelo'] 47 | new_sites: List[SiteSchema] = generate_username_sites(['john.doe'], sites_list) 48 | assert new_sites[0].username == 'johndoe' 49 | 50 | def test_filter_list_all() -> None: 51 | cli_options: CliOptionsSchema = get_default_cli_options() 52 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 53 | cli_options.all = True 54 | results: List[SiteSchema] = filter_list_by(cli_options, sites_list) 55 | assert len(results) == len(sites_list) 56 | 57 | 58 | def test_filter_list_by_category() -> None: 59 | cli_options: CliOptionsSchema = get_default_cli_options() 60 | cli_options.category = 'social' 61 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 62 | cli_options.all = True 63 | results: List[SiteSchema] = filter_list_by(cli_options, sites_list) 64 | assert len(results) == len(sites_list) 65 | 66 | 67 | def test_filter_found() -> None: 68 | cli_options: CliOptionsSchema = get_default_cli_options() 69 | cli_options.all = False 70 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 71 | site: SiteSchema 72 | for site in sites_list: 73 | site.http_status_code = site.e_code 74 | site.raw_response_data = site.e_string 75 | 76 | results: List[SiteSchema] = filter_list_by(cli_options, sites_list) 77 | assert len(results) < len(sites_list) 78 | 79 | 80 | def test_filter_not_found() -> None: 81 | cli_options: CliOptionsSchema = get_default_cli_options() 82 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 83 | cli_options.all = False 84 | cli_options.not_found = True 85 | site: SiteSchema 86 | for site in sites_list: 87 | site.http_status_code = site.m_code 88 | site.raw_response_data = site.m_string 89 | 90 | results: List[SiteSchema] = filter_list_by(cli_options, sites_list) 91 | assert len(results) < len(sites_list) 92 | 93 | -------------------------------------------------------------------------------- /whatsmyname/tests/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/osint-liar/reveal-my-name/bce8aa718b1d28038bf7587c881deff01e327ef3/whatsmyname/tests/utilities/__init__.py -------------------------------------------------------------------------------- /whatsmyname/tests/utilities/test_formatters.py: -------------------------------------------------------------------------------- 1 | """Test the formatting functionality """ 2 | import csv 3 | import json 4 | from argparse import ArgumentParser 5 | from typing import List 6 | 7 | from whatsmyname.app.cli.args import get_default_args, arg_parser 8 | from whatsmyname.app.models.schemas.cli import CliOptionsSchema 9 | from whatsmyname.app.models.schemas.sites import SiteSchema, SiteOutputSchema 10 | from whatsmyname.app.tasks.process import get_sites_list 11 | from whatsmyname.app.utilities.formatters import to_json, to_csv 12 | from pydantic import parse_obj_as 13 | 14 | 15 | def get_default_cli_options() -> CliOptionsSchema: 16 | args: ArgumentParser = get_default_args() 17 | return arg_parser(args.parse_args(['-u', 'yooper', '-d', '--verbose'])) 18 | 19 | 20 | def test_to_json() -> None: 21 | cli_options: CliOptionsSchema = get_default_cli_options() 22 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 23 | site: SiteSchema = sites_list[0] 24 | site.http_status_code = 200 25 | site.generated_uri = 'test_url' 26 | site.user_agent = 'test' 27 | site.username = 'test username' 28 | to_json(cli_options, [site]) 29 | 30 | with open(cli_options.output_file) as fp: 31 | sites: List[SiteSchema] = parse_obj_as(List[SiteOutputSchema], json.load(fp)) 32 | loaded_site: SiteSchema = sites[0] 33 | assert loaded_site.http_status_code == 200 34 | assert loaded_site.generated_uri == 'test_url' 35 | 36 | 37 | def test_to_csv() -> None: 38 | cli_options: CliOptionsSchema = get_default_cli_options() 39 | sites_list: List[SiteSchema] = get_sites_list(cli_options) 40 | site: SiteSchema = sites_list[0] 41 | site.http_status_code = 200 42 | site.generated_uri = 'test_url' 43 | site.raw_response_data = '' 44 | site.user_agent = 'test' 45 | site.username = 'test username' 46 | to_csv(cli_options, [site]) 47 | 48 | with open(cli_options.output_file) as csvfile: 49 | reader = csv.DictReader(csvfile) 50 | row = next(reader) 51 | assert row['http_status_code'] == '200' 52 | assert row['generated_uri'] == 'test_url' 53 | 54 | -------------------------------------------------------------------------------- /wmn-data-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "definitions": {}, 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "$id": "https://example.com/object1658520065.json", 5 | "title": "Root", 6 | "type": "object", 7 | "required": [ 8 | "license", 9 | "authors", 10 | "categories", 11 | "sites" 12 | ], 13 | "properties": { 14 | "license": { 15 | "$id": "#root/license", 16 | "title": "License", 17 | "type": "array", 18 | "default": [], 19 | "items":{ 20 | "$id": "#root/license/items", 21 | "title": "Items", 22 | "type": "string", 23 | "default": "", 24 | "examples": [ 25 | "Copyright (C) 2022 Micah Hoffman" 26 | ], 27 | "pattern": "^.*$" 28 | } 29 | }, 30 | "authors": { 31 | "$id": "#root/authors", 32 | "title": "Authors", 33 | "type": "array", 34 | "default": [], 35 | "items":{ 36 | "$id": "#root/authors/items", 37 | "title": "Items", 38 | "type": "string", 39 | "default": "", 40 | "examples": [ 41 | "WebBreacher" 42 | ], 43 | "pattern": "^.*$" 44 | } 45 | }, 46 | "categories": { 47 | "$id": "#root/categories", 48 | "title": "Categories", 49 | "type": "array", 50 | "default": [], 51 | "items":{ 52 | "$id": "#root/categories/items", 53 | "title": "Items", 54 | "type": "string", 55 | "default": "", 56 | "examples": [ 57 | "archived" 58 | ], 59 | "pattern": "^.*$" 60 | } 61 | }, 62 | "sites": { 63 | "$id": "#root/sites", 64 | "title": "Sites", 65 | "type": "array", 66 | "default": [], 67 | "items":{ 68 | "$id": "#root/sites/items", 69 | "title": "Items", 70 | "type": "object", 71 | "required": [ 72 | "name", 73 | "uri_check", 74 | "e_code", 75 | "e_string", 76 | "m_string", 77 | "m_code", 78 | "known", 79 | "cat", 80 | "valid" 81 | ], 82 | "properties": { 83 | "name": { 84 | "$id": "#root/sites/items/name", 85 | "title": "Name", 86 | "type": "string", 87 | "default": "", 88 | "examples": [ 89 | "101010.pl" 90 | ], 91 | "pattern": "^.*$" 92 | }, 93 | "uri_check": { 94 | "$id": "#root/sites/items/uri_check", 95 | "title": "Uri_check", 96 | "type": "string", 97 | "default": "", 98 | "examples": [ 99 | "https://101010.pl/@{account}" 100 | ], 101 | "pattern": "^.*$" 102 | }, 103 | "post_body": { 104 | "$id": "#root/sites/items/post_body", 105 | "title": "Post_body", 106 | "type": "string", 107 | "default": "", 108 | "examples": [ 109 | "" 110 | ], 111 | "pattern": "^.*$" 112 | }, 113 | "invalid_chars": { 114 | "$id": "#root/sites/items/invalid_chars", 115 | "title": "Invalid_chars", 116 | "type": "string", 117 | "default": "", 118 | "examples": [ 119 | "." 120 | ], 121 | "pattern": "^.*$" 122 | }, 123 | "e_code": { 124 | "$id": "#root/sites/items/e_code", 125 | "title": "E_code", 126 | "type": "integer", 127 | "default": "", 128 | "examples": [ 129 | 200 130 | ], 131 | "pattern": "^.*$" 132 | }, 133 | "e_string": { 134 | "$id": "#root/sites/items/e_string", 135 | "title": "E_string", 136 | "type": "string", 137 | "default": "", 138 | "examples": [ 139 | "@101010.pl" 140 | ], 141 | "pattern": "^.*$" 142 | }, 143 | "m_string": { 144 | "$id": "#root/sites/items/m_string", 145 | "title": "M_string", 146 | "type": "string", 147 | "default": "", 148 | "examples": [ 149 | "The page you are looking for isn't here." 150 | ], 151 | "pattern": "^.*$" 152 | }, 153 | "m_code": { 154 | "$id": "#root/sites/items/m_code", 155 | "title": "M_code", 156 | "type": "integer", 157 | "default": "", 158 | "examples": [ 159 | 404 160 | ], 161 | "pattern": "^.*$" 162 | }, 163 | "known": { 164 | "$id": "#root/sites/items/known", 165 | "title": "Known", 166 | "type": "array", 167 | "default": [], 168 | "items":{ 169 | "$id": "#root/sites/items/known/items", 170 | "title": "Items", 171 | "type": "string", 172 | "default": "", 173 | "examples": [ 174 | "szekspir" 175 | ], 176 | "pattern": "^.*$" 177 | } 178 | }, 179 | "cat": { 180 | "$id": "#root/sites/items/cat", 181 | "title": "Cat", 182 | "type": "string", 183 | "default": "", 184 | "examples": [ 185 | "social" 186 | ], 187 | "pattern": "^.*$" 188 | }, 189 | "valid": { 190 | "$id": "#root/sites/items/valid", 191 | "title": "Valid", 192 | "type": "boolean", 193 | "examples": [ 194 | true 195 | ], 196 | "default": true 197 | } 198 | } 199 | } 200 | 201 | } 202 | } 203 | } 204 | --------------------------------------------------------------------------------