├── .deepsource.toml ├── .github ├── FUNDING.yml └── workflows │ ├── codeql-analysis.yml │ ├── pip-schedule.yml │ ├── python-package.yml │ └── python-publish.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── cgiproxy └── __init__.py ├── setup.py └── tests └── test_cgiproxy.py /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | [[analyzers]] 4 | name = "python" 5 | enabled = true 6 | 7 | [analyzers.meta] 8 | runtime_version = "3.x.x" 9 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [Datamart] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | #patreon: # Replace with a single Patreon username 5 | #open_collective: # Replace with a single Open Collective username 6 | #ko_fi: # Replace with a single Ko-fi username 7 | #tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | #community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | #liberapay: # Replace with a single Liberapay username 10 | #issuehunt: # Replace with a single IssueHunt username 11 | #otechie: # Replace with a single Otechie username 12 | # custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | custom: ['https://www.paypal.me/vpodk', 'https://amzn.to/3mpgAJh'] 14 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '20 14 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /.github/workflows/pip-schedule.yml: -------------------------------------------------------------------------------- 1 | name: schedule 2 | 3 | on: 4 | schedule: 5 | - cron: '*/15 * * * *' 6 | 7 | jobs: 8 | init: 9 | runs-on: ubuntu-latest 10 | timeout-minutes: 2 11 | steps: 12 | - uses: actions/setup-python@v2 13 | with: 14 | python-version: '3.x' 15 | - run: | 16 | python -m pip install --upgrade pip 17 | pip install cgiproxy 18 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [master] 9 | pull_request: 10 | branches: [master] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: [3.7, 3.8, 3.9] 18 | # python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install flake8 pytest pylint 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | - name: Lint with flake8 32 | run: | 33 | # stop the build if there are Python syntax errors or undefined names 34 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 35 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 36 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 37 | - name: Lint with pylint 38 | run: | 39 | pylint cgiproxy 40 | - name: Test with pytest 41 | run: | 42 | pytest 43 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Publish package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.so 3 | .DS_Store 4 | __pycache__ 5 | /dist 6 | /*.egg-info 7 | /build 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | - "3.7" 8 | - "3.8" 9 | install: pip install cgiproxy 10 | script: python ./tests/test_cgiproxy.py 11 | notifications: 12 | email: 13 | recipients: dev@dtm.io 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CGI Proxy 2 | [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) 3 | [![PyPI Version](https://img.shields.io/pypi/v/cgiproxy.svg)](https://pypi.org/project/cgiproxy/) 4 | [![Python Versions](https://img.shields.io/pypi/pyversions/cgiproxy.svg)](https://pypi.org/project/cgiproxy/) 5 | [![PyPI Downloads](https://img.shields.io/pypi/dm/cgiproxy.svg?style=flat)](https://pypi.org/project/cgiproxy/) 6 | [![PyPI Format](https://img.shields.io/pypi/format/cgiproxy.svg?style=flat)](https://pypi.org/project/cgiproxy/) 7 | [![PyPI Status](https://img.shields.io/pypi/status/cgiproxy.svg?style=flat)](https://pypi.org/project/cgiproxy/) 8 | 9 | Simple CGI HTTP Proxy. 10 | * Sets the `X-Forwarded-For` header with the client IP address; 11 | * Sets the `User-Agent` header with the client' `User-Agent` string; 12 | * Decodes `gzip`-ed content; 13 | * Prints all errors to `stderr`; 14 | * Uses `urllib2` for Python 2 and `urllib.request` for Python 3. 15 | 16 | ## Installation 17 | 18 | Install from PyPI using `pip`: 19 | 20 | ```bash 21 | $ pip install cgiproxy 22 | ``` 23 | 24 | 25 | ## Methods 26 | 27 | ### do_get(url, headers=None) 28 | Performs `GET` request. 29 | 30 | **Arguments:** 31 | * `url` - The request URL as `str`. 32 | * `headers` - Optional HTTP request headers as `dict`. 33 | 34 | **Returns:** 35 | * A tuple of `(content, status_code, response_headers)` 36 | 37 | 38 | ### do_head(url, headers=None) 39 | Performs `HEAD` request. 40 | 41 | **Arguments:** 42 | * `url` - The request URL as `str`. 43 | * `headers` - Optional HTTP request headers as `dict`. 44 | 45 | **Returns:** 46 | * A tuple of `(content='', status_code, response_headers)` 47 | 48 | 49 | ### do_post(url, data=None, headers=None) 50 | Performs `POST` request. Converts query to `POST` params if `data` is `None`. 51 | 52 | **Arguments:** 53 | * `url` - The request URL as `str`. 54 | * `data` - Optional HTTP POST data as URL-encoded `str`. 55 | * `headers` - Optional HTTP request headers as `dict`. 56 | 57 | **Returns:** 58 | * A tuple of `(content, status_code, response_headers)` 59 | 60 | 61 | ### get_http_status(url, headers=None) 62 | Gets HTTP status code. 63 | 64 | **Arguments:** 65 | * `url` - The request URL as `str`. 66 | * `headers` - Optional HTTP request headers as `dict`. 67 | 68 | **Returns:** 69 | * An HTTP status code. 70 | 71 | 72 | ### get_response_headers(url, headers=None) 73 | Gets HTTP response headers. 74 | 75 | **Arguments:** 76 | * `url` - The request URL as `str`. 77 | * `headers` - Optional HTTP request headers as `dict`. 78 | 79 | **Returns:** 80 | * An HTTP response headers as `dict`. 81 | 82 | 83 | ## Examples 84 | 85 | ```python 86 | import cgiproxy 87 | 88 | status = cgiproxy.get_http_status('https://www.pageportrait.com/') 89 | print(200 == status) 90 | 91 | headers = cgiproxy.get_response_headers('https://komito.net/') 92 | print(headers.get('content-type')) 93 | 94 | content, status, headers = cgiproxy.do_head('https://www.dtm.io/') 95 | print('' == content) 96 | print(200 == status) 97 | print(headers.get('content-type')) 98 | 99 | content, status, headers = cgiproxy.do_get('https://www.dtm.io/', headers={ 100 | 'User-Agent': 'Mozilla/5.0 (compatible; Darwin/18.2.0) cgiproxy/18.12', 101 | 'X-Custom-Header': 'value' 102 | }) 103 | print('' != content) 104 | print(200 == status) 105 | print(headers.get('content-type')) 106 | 107 | content, status, headers = cgiproxy.do_post('https://example.com/', data='aaa%3Dbbb%26ccc%3Dddd') 108 | print('' != content) 109 | print(200 == status) 110 | print(headers.get('content-type')) 111 | ``` 112 | -------------------------------------------------------------------------------- /cgiproxy/__init__.py: -------------------------------------------------------------------------------- 1 | """Simple HTTP Proxy. 2 | 3 | See: 4 | https://google.github.io/styleguide/pyguide.html 5 | https://en.wikipedia.org/wiki/X-Forwarded-For 6 | """ 7 | 8 | import os 9 | import platform 10 | import socket 11 | import sys 12 | 13 | from datetime import datetime 14 | from gzip import GzipFile 15 | from io import BytesIO as StringIO 16 | 17 | try: # Python 2.x 18 | import urllib2 19 | from urlparse import parse_qs 20 | from urlparse import urlparse 21 | from urllib import urlencode 22 | except ImportError: # Python 3.x 23 | import urllib.request as urllib2 24 | from urllib.parse import parse_qs 25 | from urllib.parse import urlparse 26 | from urllib.parse import urlencode 27 | 28 | 29 | __version__ = '20.11.14' 30 | 31 | 32 | def do_get(url, headers=None): 33 | """Performs GET request. 34 | 35 | Args: 36 | url: The request URL as string. 37 | headers: Optional HTTP request headers as dict. 38 | 39 | Returns: 40 | A tuple of (content, status_code, response_headers) 41 | """ 42 | headers = _get_request_headers(headers) 43 | return _get_content('GET', url, headers) 44 | 45 | 46 | def do_post(url, data=None, headers=None): 47 | """Performs POST request. Converts query to POST params if data is None. 48 | 49 | Args: 50 | url: The request URL as string. 51 | data: Optional HTTP POST data as string. 52 | headers: Optional HTTP request headers as dict. 53 | 54 | Returns: 55 | A tuple of (content, status_code, response_headers) 56 | """ 57 | headers = _get_request_headers(headers) 58 | if 'Content-Type' not in headers: 59 | headers['Content-Type'] = 'application/x-www-form-urlencoded' 60 | 61 | if data is None: 62 | urlobj = urlparse(url) 63 | data = urlencode(parse_qs(urlobj.query)).encode('utf-8') 64 | 65 | return _get_content('POST', url, headers, data) 66 | 67 | 68 | def do_head(url, headers=None): 69 | """Performs HEAD request. 70 | 71 | Args: 72 | url: The request URL as string. 73 | headers: Optional HTTP request headers as dict. 74 | 75 | Returns: 76 | A tuple of (content, status_code, response_headers) 77 | """ 78 | headers = _get_request_headers(headers) 79 | return _get_content('HEAD', url, headers) 80 | 81 | 82 | def get_http_status(url, headers=None): 83 | """Gets HTTP status code. 84 | 85 | Args: 86 | url: The request URL as string. 87 | headers: Optional HTTP request headers as dict. 88 | 89 | Returns: 90 | An HTTP status code. 91 | """ 92 | _, status, response_headers = do_head(url, headers) 93 | redirect = response_headers.get('location') 94 | 95 | if redirect: 96 | return get_http_status(redirect, headers) 97 | return status 98 | 99 | 100 | def get_response_headers(url, headers=None): 101 | """Gets HTTP response headers. 102 | 103 | Args: 104 | url: The request URL as string. 105 | headers: Optional HTTP request headers as dict. 106 | 107 | Returns: 108 | An HTTP response headers as dict. 109 | """ 110 | response_headers = do_head(url, headers)[-1] 111 | return response_headers 112 | 113 | 114 | def _error(message, error=None): 115 | """Prints error message to stderr. 116 | 117 | Args: 118 | message: The error message as string. 119 | error: Optional error object. 120 | """ 121 | # pylint:disable=consider-using-f-string 122 | # pylint:disable=protected-access 123 | frame = sys._getframe(1) 124 | name = frame.f_globals['__name__'] # __name__ 125 | func = frame.f_back.f_code.co_name 126 | today = datetime.now() 127 | sys.stderr.write('[%s] [%s] [%s.%s] %s\n' % ( 128 | today, 'ERROR', name, func, message)) 129 | # pylint:enable=protected-access 130 | 131 | if error: 132 | sys.stderr.write('[%s] [%s] [%s] %s\n' % ( 133 | today, 'ERROR', error.__class__.__name__, error)) 134 | # pylint:enable=consider-using-f-string 135 | 136 | 137 | def _get_content(method, url, headers, data=None): 138 | """Gets a content, status code and response headers. 139 | 140 | Args: 141 | method: The HTTP method as string. 142 | url: The request URL as string. 143 | headers: The HTTP request headers as dict. 144 | data: Optional HTTP POST data as string. 145 | 146 | Returns: 147 | A tuple of (content, status_code, response_headers) 148 | """ 149 | # pylint:disable=consider-using-f-string 150 | try: 151 | response = _do_request(method, url, headers, data) 152 | except urllib2.HTTPError as error: 153 | _error('Could not load URL: %s' % url, error) 154 | response = error 155 | except urllib2.URLError as error: 156 | _error('Could not load URL: %s' % url, error) 157 | response = None 158 | # pylint:enable=consider-using-f-string 159 | 160 | if response is not None: 161 | headers = response.info() 162 | # headers = dict((key.lower(), value) for key, value in headers.items()) 163 | headers = {key.lower(): value for key, value in headers.items()} 164 | status = response.getcode() 165 | content = _decode_content(response.read(), headers) 166 | response.close() 167 | return (content, status, headers) 168 | return ('', 500, None) 169 | 170 | 171 | def _do_request(method, url, headers=None, data=None): 172 | """Gets a file-like object containing the data. 173 | 174 | Args: 175 | method: The HTTP method as string. 176 | url: The request URL as string. 177 | headers: Optional HTTP request headers as dict. 178 | data: Optional HTTP POST data as string. 179 | 180 | Returns: 181 | A file-like object containing the data. 182 | """ 183 | if url[:5] == 'http:': 184 | handler = urllib2.HTTPHandler 185 | else: 186 | handler = urllib2.HTTPSHandler 187 | 188 | opener = urllib2.build_opener(handler) 189 | request = urllib2.Request(url, data=data) 190 | 191 | for header in headers: 192 | request.add_header(header, headers[header]) 193 | 194 | # pylint:disable=consider-using-f-string 195 | request.get_method = lambda: '%s' % method 196 | # pylint:enable=consider-using-f-string 197 | return opener.open(request) 198 | 199 | 200 | def _get_request_headers(headers=None): 201 | """Gets default HTTP request headers. 202 | 203 | Args: 204 | headers: Optional initial HTTP request headers as dict. 205 | 206 | Returns: 207 | A default HTTP request headers as dict. 208 | """ 209 | if headers is None: 210 | headers = {} 211 | 212 | if 'Accept-Encoding' not in headers: 213 | headers['Accept-Encoding'] = 'gzip, deflate' 214 | 215 | if 'User-Agent' not in headers: 216 | headers['User-Agent'] = _get_user_agent() 217 | 218 | if 'X-Forwarded-For' not in headers: 219 | user_ip = _get_user_ip_address() 220 | host_ip = _get_host_ip_address() 221 | if user_ip and host_ip: 222 | headers['X-Forwarded-For'] = user_ip + ', ' + host_ip 223 | 224 | return headers 225 | 226 | 227 | def _get_user_agent(): 228 | """Gets HTTP user agent.""" 229 | module = os.path.basename(__file__).split('.')[0] 230 | if module == '__init__': 231 | module = os.path.dirname(os.path.abspath(__file__)).split(os.sep)[-1] 232 | 233 | # pylint:disable=consider-using-f-string 234 | user_agent = 'Mozilla/5.0 (compatible; %s/%s) %s/%s' % ( 235 | platform.system(), platform.release(), module, __version__) 236 | # pylint:enable=consider-using-f-string 237 | 238 | return os.environ.get('HTTP_USER_AGENT') or user_agent 239 | 240 | 241 | def _get_user_ip_address(): 242 | """Gets user's IP address.""" 243 | user_ip = os.environ.get('REMOTE_ADDR') 244 | x_proxy = os.environ.get('HTTP_X_FORWARDED_FOR') 245 | 246 | if x_proxy: 247 | user_ip = x_proxy.split(',')[0] 248 | 249 | return user_ip 250 | 251 | 252 | def _get_host_ip_address(): 253 | """Gets server' IP address.""" 254 | # pylint:disable=broad-except 255 | host_ip = None 256 | 257 | try: 258 | host_ip = socket.gethostbyname(socket.gethostname()) 259 | except Exception as error: 260 | _error('Could not get server IP address.', error) 261 | 262 | # pylint:enable=broad-except 263 | return host_ip 264 | 265 | 266 | def _decode_content(content, headers): 267 | """Decodes content.""" 268 | # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html 269 | encodings = headers.get('content-encoding', '').split(',') 270 | 271 | # If multiple encodings have been applied to an entity, the content 272 | # codings MUST be listed in the order in which they were applied. 273 | if 'gzip' in encodings: 274 | try: 275 | content = GzipFile(fileobj=StringIO(content)).read() 276 | except IOError: 277 | # pylint:disable=protected-access 278 | # Decompressing partial content by skipping checksum comparison. 279 | GzipFile._read_eof = lambda *args, **kwargs: None 280 | # pylint:enable=protected-access 281 | content = GzipFile(fileobj=StringIO(content)).read() 282 | # elif 'compress' in encodings: pass 283 | # elif 'deflate' in encodings: pass 284 | 285 | charset = _get_charset(headers) 286 | if charset is not None: 287 | try: 288 | content = content.decode(charset).encode('utf-8') 289 | except UnicodeDecodeError as ex: # pylint: disable=unused-variable 290 | pass 291 | 292 | return content.decode('utf-8') 293 | 294 | 295 | def _get_charset(headers): 296 | """Gets response charset. 297 | 298 | Args: 299 | headers: The HTTP response headers as dict. 300 | 301 | Returns: 302 | A response charset or None. 303 | """ 304 | content_type = headers.get('content-type') 305 | charset = None 306 | 307 | if content_type is not None: 308 | parts = content_type.split('charset=') 309 | if len(parts) == 2: 310 | charset = parts[-1] 311 | 312 | return charset 313 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | import setuptools 4 | 5 | 6 | PACKAGE_NAME = 'cgiproxy' 7 | FILE_NAME = PACKAGE_NAME + '/__init__.py' 8 | VERSION = re.search( 9 | "__version__ = ['\"]([^'\"]+)['\"]", open(FILE_NAME, 'r').read()).group(1) 10 | 11 | with open('README.md', 'r') as fh: 12 | LONG_DESCRIPTION = fh.read() 13 | 14 | setuptools.setup( 15 | name=PACKAGE_NAME, 16 | version=VERSION, 17 | description='The simple HTTP proxy.', 18 | long_description=LONG_DESCRIPTION, 19 | long_description_content_type='text/markdown', 20 | url='https://github.com/Datamart/CGI-Proxy', 21 | author='Valentin Podkamennyi, Alex Krailo', 22 | # author_email='Valentin Podkamennyi , Alex Krailo ', 23 | license='Apache 2.0', 24 | packages=setuptools.find_packages(), 25 | classifiers=[ 26 | 'Development Status :: 5 - Production/Stable', 27 | 'Intended Audience :: Developers', 28 | 'Programming Language :: Python', 29 | 'Programming Language :: Python :: 2', 30 | 'Programming Language :: Python :: 3', 31 | 'License :: OSI Approved :: Apache Software License', 32 | 'Topic :: Software Development :: Libraries :: Python Modules', 33 | ] 34 | ) 35 | -------------------------------------------------------------------------------- /tests/test_cgiproxy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | 5 | cwd = os.path.dirname(os.path.abspath(__file__)) 6 | sys.path.insert(0, os.path.abspath(os.path.join(cwd, '..'))) 7 | 8 | from cgiproxy import (_get_host_ip_address, _get_user_agent, # noqa: E402 9 | _get_user_ip_address, do_get, do_head, get_http_status, 10 | get_response_headers) 11 | 12 | 13 | class TestProxy(unittest.TestCase): 14 | """Unittest test case.""" 15 | TEST_URL = 'https://komito.net/' 16 | PRINT_OUTPUT = False 17 | 18 | def test_get_http_status(self): 19 | """Tests 'get_http_status' method.""" 20 | status = get_http_status(self.TEST_URL) 21 | self.assertEqual(200, status) 22 | if self.PRINT_OUTPUT: 23 | print('get_http_status: %s' % status) 24 | 25 | def test_get_response_headers(self): 26 | """Tests 'get_response_headers' method.""" 27 | headers = get_response_headers(self.TEST_URL) 28 | self.assertEqual(dict, type(headers)) 29 | if self.PRINT_OUTPUT: 30 | print('get_response_headers: %s' % str(headers)) 31 | 32 | def test_do_head(self): 33 | """Tests 'do_head' method.""" 34 | content, status, headers = do_head(self.TEST_URL) 35 | self.assertEqual('', content) 36 | self.assertEqual(int, type(status)) 37 | self.assertEqual(dict, type(headers)) 38 | if self.PRINT_OUTPUT: 39 | print('do_head: %s' % str((content, status, headers))) 40 | 41 | def test_do_get(self): 42 | """Tests 'do_get' method.""" 43 | content, status, headers = do_get(self.TEST_URL) 44 | self.assertNotEqual('', content) 45 | self.assertEqual(int, type(status)) 46 | self.assertGreaterEqual(status, 200) 47 | self.assertEqual(dict, type(headers)) 48 | 49 | def test_get_user_agent(self): 50 | """Tests '_get_user_agent' method.""" 51 | user_agent = _get_user_agent() 52 | self.assertNotEqual('', user_agent) 53 | if self.PRINT_OUTPUT: 54 | print('_get_user_agent: %s' % user_agent) 55 | 56 | def test_get_user_ip_address(self): 57 | """Tests '_get_user_ip_address' method.""" 58 | ip_address = _get_user_ip_address() 59 | self.assertNotEqual('', ip_address) 60 | if self.PRINT_OUTPUT: 61 | print('_get_user_ip_address: %s' % ip_address) 62 | 63 | def test_get_host_ip_address(self): 64 | """Tests '_get_host_ip_address' method.""" 65 | ip_address = _get_host_ip_address() 66 | self.assertNotEqual('', ip_address) 67 | if self.PRINT_OUTPUT: 68 | print('_get_host_ip_address: %s' % ip_address) 69 | 70 | 71 | if __name__ == '__main__': 72 | unittest.main() 73 | --------------------------------------------------------------------------------