├── hhhash ├── __init__.py └── create.py ├── AUTHORS ├── pyproject.toml ├── LICENSE ├── .github └── workflows │ └── jekyll-gh-pages.yml ├── README.md └── .gitignore /hhhash/__init__.py: -------------------------------------------------------------------------------- 1 | from hhhash.create import buildhash, hash_from_banner 2 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Alexandre Dulaunoy (original author) 2 | Nils Kuhnert (contributing author) 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | #[build-system] 2 | #requires = ["setuptools>=61.0"] 3 | #build-backend = "setuptools.build_meta" 4 | 5 | [tool.poetry] 6 | name = "HHHash" 7 | version = "0.4" 8 | authors = [ 9 | "Alexandre Dulaunoy " 10 | ] 11 | description = "HHHash library is calculate HHHash from HTTP servers." 12 | readme = "README.md" 13 | classifiers = [ 14 | "Programming Language :: Python :: 3", 15 | "License :: OSI Approved :: MIT License", 16 | "Operating System :: OS Independent", 17 | ] 18 | 19 | homepage = "https://github.com/adulau/HHHash" 20 | repository = "https://github.com/adulau/HHHash/issues" 21 | 22 | [tool.poetry.dependencies] 23 | python = "^3.6" 24 | requests = "^2.20.0" 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 (C) Alexandre Dulaunoy 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /.github/workflows/jekyll-gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["master"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Build job 26 | build: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v3 31 | - name: Setup Pages 32 | uses: actions/configure-pages@v3 33 | - name: Build with Jekyll 34 | uses: actions/jekyll-build-pages@v1 35 | with: 36 | source: ./ 37 | destination: ./_site 38 | - name: Upload artifact 39 | uses: actions/upload-pages-artifact@v1 40 | 41 | # Deployment job 42 | deploy: 43 | environment: 44 | name: github-pages 45 | url: ${{ steps.deployment.outputs.page_url }} 46 | runs-on: ubuntu-latest 47 | needs: build 48 | steps: 49 | - name: Deploy to GitHub Pages 50 | id: deployment 51 | uses: actions/deploy-pages@v2 52 | -------------------------------------------------------------------------------- /hhhash/create.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import hashlib 3 | 4 | 5 | def buildhash(url=None, debug=False, method='GET', timeout=5): 6 | """Build a HHHash from an HTTP request to specific url. 7 | 8 | Keyword arguments: 9 | - `url` -- the url to build the HHHash from the response headers (default None) 10 | - `debug` -- output the headers returned before hashing (default False) 11 | - `method` -- HTTP method to use (GET or HEAD) (default GET) 12 | - `timeout` -- default timeout for the connect/read timeout of request (default 2) 13 | 14 | For more details about the [HHHash algorithm](https://www.foo.be/2023/07/HTTP-Headers-Hashing_HHHash). 15 | """ 16 | if url is None: 17 | return False 18 | if method == 'GET': 19 | r = requests.get(url, timeout=timeout, allow_redirects=False) 20 | elif method == 'HEAD': 21 | r = requests.head(url, timeout=timeout, allow_redirects=False) 22 | else: 23 | return False 24 | hhhash = "" 25 | for header in r.headers.keys(): 26 | hhhash = f"{hhhash}:{header}" 27 | m = hashlib.sha256() 28 | if debug: 29 | print(hhhash[1:]) 30 | m.update(hhhash[1:].encode()) 31 | digest = m.hexdigest() 32 | return f"hhh:1:{digest}" 33 | 34 | def hash_from_banner(banner, debug=False): 35 | """Create a HHHash from an already fetched banner. Lines without colons will be skipped. 36 | 37 | Keyword arguments: 38 | - `banner` -- HTTP banner as a string 39 | - `debug` -- output the headers returned before hashing 40 | 41 | Example usage: 42 | 43 | ``` 44 | >>> hash_from_banner('''HTTP/1.1 200 OK 45 | ... Content-Type: text/html; charset=ISO-8859-1 46 | ... Content-Security-Policy-Report-Only: object-src 'none';base-uri 'self';script-src 'nonce-iV-j91UJEG2jNx4j6EeTug' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/gws/other-hp 47 | ... P3P: CP="This is not a P3P policy! See g.co/p3phelp for more info." 48 | ... Date: Wed, 12 Jul 2023 20:23:42 GMT 49 | ... Server: gws 50 | ... X-XSS-Protection: 0 51 | ... X-Frame-Options: SAMEORIGIN 52 | ... Transfer-Encoding: chunked 53 | ... Expires: Wed, 12 Jul 2023 20:23:42 GMT 54 | ... Cache-Control: private 55 | ... Set-Cookie: 56 | ... Set-Cookie: 57 | ... Set-Cookie: 58 | ... Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000''') 59 | hhh:1:d9576f3e7a381562f7d18a514ab095fa8699e96891d346d0042f83e942373215 60 | 61 | ``` 62 | """ 63 | hhhash = "" 64 | for line in banner.splitlines(): 65 | if ":" not in line: 66 | continue 67 | 68 | header, _ = line.split(":", maxsplit=1) 69 | hhhash = f"{hhhash}:{header.strip()}" 70 | if debug: 71 | print(hhhash[1:]) 72 | m = hashlib.sha256() 73 | m.update(hhhash[1:].encode()) 74 | digest = m.hexdigest() 75 | return f"hhh:1:{digest}" 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTP Headers Hashing (HHHash) 2 | 3 | HTTP Headers Hashing (HHHash) is a technique used to create a fingerprint of an HTTP server based on the headers it returns. HHHash employs one-way hashing to generate a hash value for the set of header keys returned by the server. 4 | 5 | For more details about HHHash background, [HTTP Headers Hashing (HHHash) or improving correlation of crawled content](https://www.foo.be/2023/07/HTTP-Headers-Hashing_HHHash). 6 | 7 | ## Calculation of the HHHash 8 | 9 | To calculate the HHHash, we concatenate the list of headers returned by the HTTP server. This list is ordered according to the sequence in which the headers appear in the server's response. Each header value is separated with `:`. 10 | 11 | The HHHash value is the SHA256 of the list. 12 | 13 | ## HHHash format 14 | 15 | `hhh`:`1`:`20247663b5c63bf1291fe5350010dafb6d5e845e4c0daaf7dc9c0f646e947c29` 16 | 17 | `prefix`:`version`:`SHA 256 value` 18 | 19 | ## Example 20 | 21 | ### Calculating HHHash from a curl command 22 | 23 | Curl will attempt to run the request using HTTP2 by default. In order to get the same hash as the python requests module (which doesn't supports HTTP2), you need to specify the version with the `--http1.1` switch. 24 | 25 | ~~~bash 26 | curl --http1.1 -s -D - https://www.circl.lu/ -o /dev/null | awk 'NR != 1' | cut -f1 -d: | sed '/^[[:space:]]*$/d' | sed -z 's/\n/:/g' | sed 's/.$//' | sha256sum | cut -f1 -d " " | awk {'print "hhh:1:"$1'} 27 | ~~~ 28 | 29 | Output value 30 | ~~~ 31 | hhh:1:78f7ef0651bac1a5ea42ed9d22242ed8725f07815091032a34ab4e30d3c3cefc 32 | ~~~ 33 | 34 | ## Limitations 35 | 36 | HHHash is an effective technique; however, its performance is heavily reliant on the characteristics of the HTTP client requests. Therefore, it is important to note that correlations between a set of hashes are typically established when using the same crawler or HTTP client parameters. 37 | 38 | HTTP2 requires the [headers to be lowercase](https://www.rfc-editor.org/rfc/rfc7540#section-8.1.2). It will then changes the hash so you need to be aware of the HTTP version you're using. 39 | 40 | ### hhhash - Python Library 41 | 42 | The [hhhash package](https://pypi.org/project/hhhash/) can be installed via a `pip install hhhash` or build with Poetry from this repository `poetry build` and `poetry install`. 43 | 44 | #### Usage 45 | 46 | ~~~ipython 47 | In [1]: import hhhash 48 | 49 | In [2]: hhhash.buildhash(url="https://www.misp-lea.org", debug=False) 50 | Out[2]: 'hhh:1:adca8a87f2a537dbbf07ba6d8cba6db53fde257ae2da4dad6f3ee6b47080c53f' 51 | 52 | In [3]: hhhash.buildhash(url="https://www.misp-project.org", debug=False) 53 | Out[3]: 'hhh:1:adca8a87f2a537dbbf07ba6d8cba6db53fde257ae2da4dad6f3ee6b47080c53f' 54 | 55 | In [4]: hhhash.buildhash(url="https://www.circl.lu", debug=False) 56 | Out[4]: 'hhh:1:334d8ab68f9e935f3af7c4a91220612f980f2d9168324530c03d28c9429e1299' 57 | 58 | In [5]: 59 | ~~~ 60 | 61 | ## Other libraries 62 | 63 | - [c-hhhash](https://github.com/hrbrmstr/c-hhhash) - C++ HTTP Headers Hashing CLI 64 | - [go-hhhash](https://github.com/hrbrmstr/go-hhhash) - golang HTTP Headers Hashing CLI 65 | - [R hhhash](https://github.com/hrbrmstr/hhhash) - R library HHHash 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | --------------------------------------------------------------------------------