├── .coveralls.yml ├── .env-default ├── .gitignore ├── .htaccess ├── .travis.yml ├── Dockerfile.arm7hf ├── Dockerfile.x86_64 ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── app.json ├── bin ├── rebuild-proxy-list └── verify-proxy-address ├── composer.json ├── composer.lock ├── config.php ├── crontab.conf ├── custom-fixtures ├── .gitkeep └── ExampleFixture.php ├── docs ├── Fixtures.md └── images │ └── fixture-example.png ├── phpunit.xml.dist ├── src ├── Controllers │ ├── BaseController.php │ ├── PassThroughController.php │ ├── ProxySelectorController.php │ └── RenderController.php ├── DependencyInjection │ ├── LibraryServices.php │ └── Services.php ├── Entity │ ├── ForwardableRequest.php │ ├── ProxyServerAddress.php │ └── TorProxyServerAddress.php ├── Exception │ ├── AccessDeniedException.php │ ├── Codes.php │ ├── HttpException.php │ └── InvalidConfigurationException.php ├── Factory │ ├── ProxyClientFactory.php │ ├── ProxyProviderFactory.php │ └── RequestFactory.php ├── Fixtures │ ├── FacebookCaptchaTo500.php │ ├── FixtureInterface.php │ └── NotFoundTo500.php ├── InputParams.php ├── Middleware │ ├── ApplicationMiddleware.php │ ├── AuthenticationMiddleware.php │ ├── OneTimeTokenParametersConversionMiddleware.php │ └── ProxyStaticContentMiddleware.php ├── Providers │ └── Proxy │ │ ├── BaseProvider.php │ │ ├── CachedProvider.php │ │ ├── ChainProvider.php │ │ ├── DummyProvider.php │ │ ├── FreeProxyCzProvider.php │ │ ├── FreeProxyListProvider.php │ │ ├── GatherProxyProvider.php │ │ ├── HideMyNameProvider.php │ │ ├── ProxyListOrgProvider.php │ │ ├── ProxyProviderInterface.php │ │ ├── TorProxyProvider.php │ │ └── UsProxyOrgProvider.php ├── Service │ ├── Config.php │ ├── ContentProcessor │ │ ├── ContentProcessor.php │ │ ├── CssProcessor.php │ │ ├── HtmlProcessor.php │ │ └── ProcessorInterface.php │ ├── FixturesManager.php │ ├── Prerenderer.php │ ├── Proxy.php │ ├── Proxy │ │ └── ProxySelector.php │ ├── ProxyCacheBuilder.php │ └── Security │ │ ├── AuthCheckerInterface.php │ │ ├── OneTimeBrowseTokenChecker.php │ │ ├── OneTimeTokenUrlGenerator.php │ │ └── TokenAuthChecker.php └── bootstrap.php ├── tests ├── Factory │ ├── ProxyClientFactoryTest.php │ ├── ProxyProviderFactoryTest.php │ └── RequestFactoryTest.php ├── Fixtures │ ├── FacebookCaptchaTo500Test.php │ └── NotFoundTo500Test.php ├── Middleware │ └── OneTimeTokenParametersConversionMiddlewareTest.php ├── Providers │ └── Proxy │ │ ├── CachedProviderTest.php │ │ ├── ChainProviderTest.php │ │ ├── FreeProxyCzProviderTest.php │ │ ├── FreeProxyListProviderTest.php │ │ ├── GatherProxyProviderTest.php │ │ ├── ProxyListOrgProviderTest.php │ │ ├── TestProxyProviderInterfaceImplementation.php │ │ └── UsProxyOrgProviderTest.php ├── Service │ ├── Controllers │ │ └── PassThroughControllerTest.php │ ├── FixturesManagerTest.php │ ├── Proxy │ │ └── ProxySelectorTest.php │ └── Security │ │ └── TokenAuthCheckerTest.php └── TestCase.php └── web ├── .htaccess └── index.php /.coveralls.yml: -------------------------------------------------------------------------------- 1 | coverage_clover: ./clover.xml 2 | json_path: ./coveralls-upload.json 3 | service_name: travis-ci -------------------------------------------------------------------------------- /.env-default: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riotkit-org/web-proxy/2e586aeff18b6706b61b31f2dddbef6e27004540/.env-default -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | config.custom.php 2 | vendor 3 | .idea 4 | 5 | /custom-fixtures/* 6 | !/custom-fixtures/.gitkeep 7 | !/custom-fixtures/ 8 | !/custom-fixtures/ExampleFixture.php 9 | 10 | /var/*.log 11 | /var/cache/* 12 | 13 | /.bash_history 14 | /.well-known 15 | /maintenance-page 16 | /supervisord.log 17 | /supervisord.pid 18 | -------------------------------------------------------------------------------- /.htaccess: -------------------------------------------------------------------------------- 1 | deny from all -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | php: 3 | - '7.1' 4 | - '7.2' 5 | - hhvm 6 | - hhvm-nightly 7 | - nightly 8 | 9 | matrix: 10 | allow_failures: 11 | - php: hhvm 12 | - php: nightly 13 | - php: hhvm-nightly 14 | 15 | before_script: 16 | - composer install && composer dump-autoload -o 17 | 18 | script: 19 | - ./vendor/bin/phpunit --coverage-text 20 | -------------------------------------------------------------------------------- /Dockerfile.arm7hf: -------------------------------------------------------------------------------- 1 | FROM wolnosciowiec/docker-php-app:arm7hf 2 | 3 | ENV WW_TOKEN="your-api-key-here" \ 4 | WW_EXTERNAL_PROXIES="" \ 5 | WW_CACHE_TTL=360 \ 6 | WW_TIMEOUT=10 \ 7 | WW_FIXTURES="" \ 8 | WW_FIXTURES_MAPPING="" \ 9 | WW_ENCRYPTION_KEY="your-encryption-key-here" \ 10 | WW_ONE_TIME_TOKEN_LIFE_TIME="+2 minutes" \ 11 | WW_PROCESS_CONTENT=1 \ 12 | WW_PRERENDER_URL="http://prerender" \ 13 | WW_PRERENDER_ENABLED=1 \ 14 | WW_TOR_PROXIES="" \ 15 | WW_TOR_PROXIES_VIRTUAL_COUNT=5 16 | 17 | ADD . /var/www/html 18 | ADD crontab.conf /etc/cron.d/www-data 19 | 20 | RUN [ "cross-build-start" ] 21 | 22 | RUN cd /var/www/html \ 23 | && chown www-data:www-data /var/www/html -R \ 24 | && su www-data -s /bin/bash -c "make deploy" 25 | 26 | RUN [ "cross-build-end" ] 27 | -------------------------------------------------------------------------------- /Dockerfile.x86_64: -------------------------------------------------------------------------------- 1 | FROM wolnosciowiec/docker-php-app 2 | 3 | ENV WW_TOKEN="your-api-key-here" \ 4 | WW_EXTERNAL_PROXIES="" \ 5 | WW_CACHE_TTL=360 \ 6 | WW_TIMEOUT=10 \ 7 | WW_FIXTURES="" \ 8 | WW_FIXTURES_MAPPING="" \ 9 | WW_ENCRYPTION_KEY="your-encryption-key-here" \ 10 | WW_ONE_TIME_TOKEN_LIFE_TIME="+2 minutes" \ 11 | WW_PROCESS_CONTENT=1 \ 12 | WW_PRERENDER_URL="http://prerender" \ 13 | WW_PRERENDER_ENABLED=1 \ 14 | WW_TOR_PROXIES="" \ 15 | WW_TOR_PROXIES_VIRTUAL_COUNT=5 16 | 17 | ADD . /var/www/html 18 | ADD crontab.conf /etc/cron.d/www-data 19 | 20 | RUN cd /var/www/html \ 21 | && chown www-data:www-data /var/www/html -R \ 22 | && su www-data -s /bin/bash -c "make deploy" 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #!make 2 | 3 | #include .env 4 | #export $(shell sed 's/=.*//' .env) 5 | 6 | .SILENT: 7 | 8 | SHELL := /bin/bash 9 | 10 | ## Colors 11 | COLOR_RESET = \033[0m 12 | COLOR_INFO = \033[32m 13 | COLOR_COMMENT = \033[33m 14 | 15 | ENV="prod" 16 | 17 | ## This help dialog 18 | help: 19 | printf "${COLOR_COMMENT}Usage:${COLOR_RESET}\n" 20 | printf " make [target]\n\n" 21 | printf "${COLOR_COMMENT}Available targets:${COLOR_RESET}\n" 22 | awk '/^[a-zA-Z\-\_0-9\.@]+:/ { \ 23 | helpMessage = match(lastLine, /^## (.*)/); \ 24 | if (helpMessage) { \ 25 | helpCommand = substr($$1, 0, index($$1, ":")); \ 26 | helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \ 27 | printf " ${COLOR_INFO}%-16s${COLOR_RESET} %s\n", helpCommand, helpMessage; \ 28 | } \ 29 | } \ 30 | { lastLine = $$0 }' $(MAKEFILE_LIST) 31 | 32 | ## Build the application by running preparation tasks such as composer install 33 | build: 34 | composer install --dev 35 | 36 | ## Prepare the application to be ready to run 37 | deploy: 38 | make build 39 | 40 | ## Run a development web server 41 | run_dev_server: 42 | COMPOSER_PROCESS_TIMEOUT=9999999 composer run web 43 | 44 | ## Regenerate the cache by re-visiting all pages that were already visited by bots (does not force regenerate) 45 | regenerate_cached_pages: 46 | bash ./bin/reclick-cache.sh 47 | 48 | ## Run application test suites 49 | test: 50 | ./vendor/bin/phpunit -vvv 51 | 52 | ## Build x86_64 image 53 | build@x86_64: 54 | sudo docker build . -f ./Dockerfile.x86_64 -t wolnosciowiec/webproxy 55 | 56 | ## Build arm7hf image 57 | build@arm7hf: 58 | sudo docker build . -f ./Dockerfile.arm7hf -t wolnosciowiec/webproxy:arm7hf 59 | 60 | ## Push x86_64 image to registry 61 | push@x86_64: 62 | sudo docker push wolnosciowiec/webproxy 63 | 64 | ## Push arm7hf image to registry 65 | push@arm7hf: 66 | sudo docker push wolnosciowiec/webproxy:arm7hf 67 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: vendor/bin/heroku-php-nginx web/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Wolnościowiec Web Proxy 2 | ======================= 3 | 4 | Notice: This project is looking for a maintainer 5 | ------------------------------------------------ 6 | 7 | [![Build Status](https://travis-ci.org/Wolnosciowiec/web-proxy.svg?branch=master)](https://travis-ci.org/Wolnosciowiec/web-proxy) 8 | [![Code quality](https://scrutinizer-ci.com/g/Wolnosciowiec/webproxy/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/Wolnosciowiec/webproxy/) 9 | [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/Wolnosciowiec/web-proxy) 10 | 11 | Anonymous HTTP proxy that forwards all requests through the PHP application on server side. 12 | 13 | Features: 14 | - Redirect all traffic hide behind the server where the Wolnościowiec WebProxy is set up 15 | - Redirect all traffic through external web proxies using providers (the list of proxies is updated automatically from external provider) 16 | - Forward all headers and cookies 17 | 18 | ``` 19 | /* 20 | * Wolnościowiec / WebProxy 21 | * ------------------------ 22 | * 23 | * Web Proxy passing through all traffic on port 80 24 | * A part of an anarchist portal - wolnosciowiec.net 25 | * 26 | * Wolnościowiec is a project to integrate the movement 27 | * of people who strive to build a society based on 28 | * solidarity, freedom, equality with a respect for 29 | * individual and cooperation of each other. 30 | * 31 | * We support human rights, animal rights, feminism, 32 | * anti-capitalism (taking over the production by workers), 33 | * anti-racism, and internationalism. We negate 34 | * the political fight and politicians at all. 35 | * 36 | * http://wolnosciowiec.net/en 37 | * 38 | * License: LGPLv3 39 | */ 40 | ```` 41 | 42 | Installation 43 | ============ 44 | 45 | ``` 46 | # you can also create the "config.custom.php" with ` 'your-api-key'];` to have the key stored permanently without having to pass it through shell 47 | export WW_TOKEN="your-api-key-here" 48 | composer install 49 | php -S 0.0.0.0:8081 ./web/index.php 50 | ``` 51 | 52 | To have a permanent configuration file create a file named "config.custom.php" in the main directory, it will be ignored by git. 53 | Example syntax: 54 | 55 | ``` 56 | 'FreeProxyListProvider', // use http://free-proxy-list.net as a provider 60 | 'connectionTimeout' => 10, 61 | 'apiKey' => 'something', 62 | 63 | // cache stored in the filesystem 64 | 'cache' => new \Doctrine\Common\Cache\FilesystemCache(__DIR__ . '/var/cache'), 65 | 'cacheTtl' => 360, // cache live time, refresh every eg. 360 seconds (the list of external proxy addresses is cached) 66 | 67 | // turn off the cache 68 | // 'cache' => new \Doctrine\Common\Cache\VoidCache(), 69 | 70 | // fixtures, example: Detect Facebook captcha and return a 500 response, convert all 404 to 500 error codes 71 | 'fixtures' => 'FacebookCaptchaTo500,NotFoundTo500', 72 | 73 | // 74 | // Feature: Content processor 75 | // When the HTML page is downloaded, then we can replace JS and CSS urls, so the will also be proxied 76 | // 77 | 'contentProcessingEnabled' => true, 78 | 79 | // 80 | // Feature: External IP providers 81 | // Use external proxies randomly to provide a huge amount of IP addresses, best option to scrap a big amount of data 82 | // from pages such as Facebook, Google which are blocking very quickly by showing a captcha 83 | // 84 | 'externalProxyProviders' => 'HideMyNameProvider,FreeProxyListProvider,GatherProxyProvider,ProxyListOrgProvider', 85 | 86 | // Wait 15 seconds for the connection 87 | 'connectionTimeout' => 15, 88 | 89 | // 90 | // Feature: One-time access tokens 91 | // Imagine you can display an IFRAME on your page that will allow users to browse the URLs you allow 92 | // So, on server side you can prepare a token, encrypt it with AES + base64 and give to the user 93 | // then a user can view the specific URL through the proxy using this token 94 | // 95 | // Token format: {"url": "http://some-allowed-url", "expires": "2017-05-05 10:20:30", "process": true, "stripHeaders": "X-Frame-Options"} 96 | // GET parameter to pass token: __wp_one_time_token 97 | // @see Implementation at https://github.com/Wolnosciowiec/news-feed-provider/blob/master/src/WebProxyBundle/Service/OneTimeViewUrlGenerator.php 98 | // 99 | 'encryptionKey' => 'some-key', 100 | 'oneTimeTokenStaticFilesLifeTime' => '+2 minutes', 101 | 102 | // 103 | // Feature: Chromium/PhantomJS prerenderer 104 | // Use an external service - Wolnościowiec Prerenderer to send requests using a real browser like Chromium or PhantomJS 105 | // 106 | 'prerendererUrl' => 'http://my-prerenderer-host', 107 | 'prerendererEnabled' => true 108 | ]; 109 | ``` 110 | 111 | #### External providers list 112 | 113 | To redirect incoming traffic through an external proxy server you can set an external proxy provider. 114 | This will fetch a list of IP addresses of proxy servers that will be used to redirect the traffic. 115 | 116 | Use `externalProxyProviders` configuration parameter, or `WW_EXTERNAL_PROXIES` environment variable. 117 | 118 | - FreeProxyCzProvider 119 | - FreeProxyListProvider 120 | - GatherProxyProvider 121 | - ProxyListOrgProvider 122 | - HideMyNameProvider 123 | - UsProxyOrgProvider 124 | 125 | To make sure that the proxy list is ALWAYS UP TO DATE you can put into crontab a script: 126 | `./bin/rebuild-proxy-list` 127 | 128 | ``` 129 | # fetch the list of proxy IP addresses from providers selected in configuration 130 | # and verify all proxy addresses one-by-one to make sure that everything is fresh 131 | */8 * * * * php ./bin/rebuild-proxy-list 132 | ``` 133 | 134 | How to use 135 | ========== 136 | 137 | Make a request, just as usual. For example POST facebook.com, but move the target url to the header "WW_TARGET_URL" 138 | and as a URL temporarily set your proxy address. 139 | 140 | So, the `web-proxy` will redirect all headers, parameters and body you will send to it except the `WW_` prefixed. 141 | 142 | ##### Example request 143 | 144 | ``` 145 | GET / HTTP/1.1 146 | ww-target-url: http://facebook.com/ZSP-Związek-Wielobranżowy-Warszawa-290681631074873 147 | ww-token: your-api-key-here 148 | ww-no-external-proxy: false 149 | 150 | ``` 151 | 152 | ##### Example request through Chromium/PhantomJS + external proxy 153 | 154 | - External proxy is used (from various providers) eg. a proxy from Proxy-List.org 155 | - Output is rendered by Chromium or PhantomJS using the [Wolnościowiec Prerenderer](https://github.com/Wolnosciowiec/frontend-prerenderer) (requires configuration + hosting) 156 | 157 | ``` 158 | GET /__webproxy/render HTTP/1.1 159 | Host: webproxy.localhost 160 | ww-token: your-api-key-here 161 | ww-url: https://facebook.com 162 | ww-process-output: false 163 | 164 | ``` 165 | 166 | ##### Example request with Chromium/PhantomJS without external proxy 167 | 168 | - A webproxy service IP address is used 169 | - Output is rendered by Chromium/PhantomJS 170 | 171 | ``` 172 | GET /__webproxy/render HTTP/1.1 173 | Host: webproxy.localhost 174 | ww-token: your-api-key-here 175 | ww-url: https://facebook.com 176 | ww-process-output: false 177 | ww-no-external-proxy: true 178 | 179 | ``` 180 | 181 | ##### Example request to get only external proxy details 182 | 183 | ``` 184 | GET /__webproxy/get-ip HTTP/1.1 185 | Host: webproxy.localhost 186 | ww-token: your-api-key-here 187 | 188 | ``` 189 | 190 | Deployment 191 | ========== 192 | 193 | To build and run a fresh image: 194 | ``` 195 | sudo docker build . -t webproxy 196 | sudo docker run -p 7001:80 webproxy:latest 197 | curl http://localhost:7001 198 | ``` 199 | 200 | With docker hub: 201 | ``` 202 | sudo docker run -p 7001:80 wolnosciowiec/web-proxy:latest 203 | ``` 204 | 205 | CURL example 206 | ============ 207 | 208 | ``` 209 | $headers = [/* ... */]; 210 | $headers[] = 'ww-token: my-proxy-token' 211 | $headers[] = 'ww-target-url: http://google.com'; 212 | 213 | curl_setopt($curlHandle, CURLOPT_URL, 'https://proxy-address'); 214 | curl_setopt($curlHandle, CURLOPT_HTTPHEADER, $headers); 215 | curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 15); 216 | curl_setopt($curlHandle, CURLOPT_TIMEOUT, 15); 217 | curl_setopt($curlHandle, CURLOPT_PROXY, ''); 218 | ``` 219 | 220 | Fixtures 221 | ======== 222 | 223 | Fixtures are response fixing middlewares. 224 | Example fixture is `FacebookCaptchaTo500` which is detecting the captcha on facebook.com, if its present then HTTP response status code will 225 | be changed to `500`. 226 | 227 | Example of enabling a fixture using an environment variable: 228 | ``` 229 | export WW_FIXTURES="FacebookCaptchaTo500,SomethingElse" 230 | ``` 231 | 232 | Example using config: 233 | ``` 234 | return [ 235 | 'fixtures' => 'FacebookCaptchaTo500', 236 | ]; 237 | ``` 238 | 239 | [Read more about the fixtures](./docs/Fixtures.md) 240 | 241 | Special endpoints 242 | ================= 243 | 244 | ``` 245 | ProxySelector 246 | ------------- 247 | Returns the IP address with port of a proxy which normally would be used to redirect the traffic 248 | Token is required to use the endpoint. 249 | 250 | Useful when need to render a page using eg. Chromium, so the browser could be spawn with proper arguments. 251 | See: https://github.com/Wolnosciowiec/frontend-prerenderer 252 | 253 | GET /__webproxy/get-ip 254 | ``` 255 | 256 | ``` 257 | Renderer 258 | -------- 259 | Renders the page with Chromium/PhantomJS using an external service Wolnościowiec Prerenderer. 260 | See: https://github.com/Wolnosciowiec/frontend-prerenderer 261 | 262 | GET /__webproxy/render HTTP/1.1 263 | Host: webproxy.localhost 264 | ww-token: your-api-key-here 265 | ww-url: https://facebook.com 266 | ww-process-output: false 267 | ``` 268 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wolnosciowiec Webproxy", 3 | "description": "Anonymous HTTP proxy that forwards all requests through the PHP application on server side.", 4 | "repository": "https://github.com/Wolnosciowiec/webproxy", 5 | "logo": "https://avatars0.githubusercontent.com/u/22785395", 6 | "keywords": ["webproxy", "proxy", "https", "http"], 7 | 8 | "env": { 9 | "WW_TOKEN": { 10 | "description": "Secret API authorization token that needs to be passed using 'ww-token' header", 11 | "required": true, 12 | "generator": "secret" 13 | }, 14 | 15 | "WW_ENCRYPTION_KEY": { 16 | "description": "Encryption key for one-time-tokens", 17 | "required": true, 18 | "generator": "secret" 19 | }, 20 | 21 | "WW_ONE_TIME_TOKEN_LIFE_TIME": { 22 | "description": "One-time-tokens life time", 23 | "required": true, 24 | "value": "+2 minutes" 25 | }, 26 | 27 | "WW_EXTERNAL_PROXIES": { 28 | "description": "List of external IP providers, leave empty to use this machine IP", 29 | "value": "FreeProxyListProvider,GatherProxyProvider,HideMyNameProvider,ProxyListOrgProvider", 30 | "required": false 31 | }, 32 | 33 | "WW_FIXTURES": { 34 | "description": "List of response fixing rules", 35 | "value": "FacebookCaptchaTo500", 36 | "required": false 37 | }, 38 | 39 | "WW_FIXTURES_MAPPING": { 40 | "description": "Mapping for non-standard fixtures (provided by external libraries or files). Json format: Key is fixture name, value is a class name.", 41 | "value": "{\"ExampleFixture\": \"\\\\Wolnosciowiec\\\\CustomFixtures\\\\ExampleFixture\"}", 42 | "required": false 43 | }, 44 | 45 | "WW_TIMEOUT": { 46 | "description": "Timeout for all connections made by proxy", 47 | "value": "10", 48 | "required": true 49 | }, 50 | 51 | "WW_CACHE_TTL": { 52 | "description": "Cache life time in seconds", 53 | "value": "360", 54 | "required": true 55 | }, 56 | 57 | "WW_DEBUG": { 58 | "description": "Debugging mode", 59 | "value": "1", 60 | "required": false 61 | }, 62 | 63 | "WW_PROCESS_CONTENT": { 64 | "description": "Process the CSS/JS urls in the HTML content", 65 | "value": "1", 66 | "required": false 67 | }, 68 | 69 | "WW_PRERENDER_URL": { 70 | "description": "Use web browser such as Chromium or PhantomJS to prerender the page", 71 | "value": "http://prerender", 72 | "required": false 73 | }, 74 | 75 | "WW_PRERENDER_ENABLED": { 76 | "description": "Enable usage of prerenderer", 77 | "value": "1", 78 | "required": false 79 | }, 80 | 81 | "WW_TOR_PROXIES_VIRTUAL_COUNT": { 82 | "description": "Sum the total count of TOR proxy servers on the list of proxies to use", 83 | "value": "5", 84 | "required": false 85 | }, 86 | 87 | "WW_TOR_PROXIES": { 88 | "description": "List of TOR proxy servers (regular HTTP proxy servers that uses TOR), comma separated without spaces", 89 | "value": "", 90 | "required": false 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /bin/rebuild-proxy-list: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | get(ProxyCacheBuilder::class); 12 | 13 | $addresses = $service->rebuildListCache(); 14 | $service->spawnVerificationProcesses($addresses); 15 | $service->logSummary(); 16 | -------------------------------------------------------------------------------- /bin/verify-proxy-address: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | get(ProxyCacheBuilder::class); 12 | 13 | /** 14 | * @var ProxyCacheBuilder $service 15 | */ 16 | $service->performProxyVerification($_SERVER['argv'][1] ?? ''); 17 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wolnosciowiec/wolnosciowiec-webproxy", 3 | "license": "LGPLv3", 4 | "type": "project", 5 | 6 | "require": { 7 | "php": ">=7.0", 8 | "ext-curl": "*", 9 | "jenssegers/proxy": "dev-master", 10 | "guzzlehttp/guzzle": "^6.0", 11 | "zendframework/zend-diactoros": "^1.3", 12 | "php-di/php-di": "^5.4", 13 | "fabpot/goutte": "^3.2", 14 | "doctrine/cache": "^1.6", 15 | "doctrine/collections": "^1.4", 16 | "monolog/monolog": "^1.22", 17 | "symfony/var-dumper": "*", 18 | "relay/relay": "1.*", 19 | "blocktrail/cryptojs-aes-php": "^0.1.0", 20 | "symfony/event-dispatcher": "^4.0" 21 | }, 22 | 23 | "scripts": { 24 | "post-install-cmd": [ 25 | "composer dump-autoload -o", 26 | "rm ./var/cache/* -rf" 27 | ], 28 | "web": [ 29 | "php -S 0.0.0.0:8009 web/index.php" 30 | ] 31 | }, 32 | 33 | "autoload": { 34 | "psr-4": { 35 | "Wolnosciowiec\\WebProxy\\": "src/", 36 | "Wolnosciowiec\\CustomFixtures\\": "custom-fixtures/", 37 | "Tests\\": "tests/" 38 | } 39 | }, 40 | 41 | "require-dev": { 42 | "phpunit/phpunit": "^5.6" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /config.php: -------------------------------------------------------------------------------- 1 | getenv('WW_TOKEN') ?: 'your-api-key-here', 13 | 'externalProxyProviders' => getenv('WW_EXTERNAL_PROXIES') !== false ? getenv('WW_EXTERNAL_PROXIES') : '', 14 | 'cache' => new \Doctrine\Common\Cache\FilesystemCache(__DIR__ . '/var/cache'), 15 | 'cacheTtl' => getenv('WW_CACHE_TTL') !== false ? (int)getenv('WW_CACHE_TTL') : 360, 16 | 'connectionTimeout' => getenv('WW_TIMEOUT') !== false ? (int)getenv('WW_TIMEOUT') : 10, 17 | 18 | // post-process: fixtures are eg. adding headers, modifying responses, customizing things generally 19 | 'fixtures' => getenv('WW_FIXTURES') !== false ? getenv('WW_FIXTURES') : '', 20 | 'fixtures_mapping' => getenv('WW_FIXTURES_MAPPING') !== false ? getenv('WW_FIXTURES_MAPPING') : '', 21 | 22 | // security: one-time-token is a possibility to grant access to the webproxy for a specific URL and until given time (url, expiration) 23 | 'encryptionKey' => getenv('WW_ENCRYPTION_KEY') !== false ? getenv('WW_ENCRYPTION_KEY') : 'your-encryption-key-here', 24 | 'oneTimeTokenStaticFilesLifeTime' => getenv('WW_ONE_TIME_TOKEN_LIFE_TIME') !== false ? getenv('WW_ONE_TIME_TOKEN_LIFE_TIME') : '+2 minutes', 25 | 26 | // post-process: replacing external links with proxied links 27 | 'contentProcessingEnabled' => getenv('WW_PROCESS_CONTENT') === '1' || getenv('WW_PROCESS_CONTENT') === false /* Enabled by default */, 28 | 29 | // use an external service - Wolnościowiec Prerenderer to send requests using a real browser like Chromium or PhantomJS 30 | 'prerendererUrl' => getenv('WW_PRERENDER_URL') ?: 'http://prerender', 31 | 'prerendererEnabled' => getenv('WW_PRERENDER_ENABLED') === '1' || getenv('WW_PRERENDER_ENABLED') === 'true', 32 | 33 | // examples 34 | #'externalProxyProviders' => 'FreeProxyListProvider', 35 | #'fixtures' => 'FacebookCaptchaTo500', 36 | 37 | 'torProxies' => getenv('WW_TOR_PROXIES') !== false ? getenv('WW_TOR_PROXIES') : '', 38 | 'torVirtualProxiesNum' => getenv('WW_TOR_PROXIES_VIRTUAL_COUNT') !== false ? (int)getenv('WW_TOR_PROXIES_VIRTUAL_COUNT') : 5 39 | ]; 40 | 41 | if (is_file(__DIR__ . '/config.custom.php')) { 42 | $settings = array_merge( 43 | $settings, 44 | require __DIR__ . '/config.custom.php' 45 | ); 46 | } 47 | 48 | return $settings; 49 | -------------------------------------------------------------------------------- /crontab.conf: -------------------------------------------------------------------------------- 1 | # rebuild proxy list every 8 minutes 2 | */8 * * * * www-data cd /var/www && ./bin/rebuild-proxy-list >> /var/log/cron.log 2>&1 3 | -------------------------------------------------------------------------------- /custom-fixtures/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riotkit-org/web-proxy/2e586aeff18b6706b61b31f2dddbef6e27004540/custom-fixtures/.gitkeep -------------------------------------------------------------------------------- /custom-fixtures/ExampleFixture.php: -------------------------------------------------------------------------------- 1 | withHeader('X-Message', 'Hello'); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /docs/Fixtures.md: -------------------------------------------------------------------------------- 1 | Fixtures 2 | ======== 3 | 4 | Fixtures are providing a possibility to edit a response that goes to the end user (eg. to your crawler). 5 | There are many cases when this is helpful for example a multiple instance of webproxy for crawling behind a load balancer. 6 | A load balancer could be configured to use `ip_hash` strategy so, always keep awake only one webproxy in the cloud, unless it will be 7 | not usable. In this case we can tell the load balancer, that for example a captcha was found, so the webproxy for the moment is not usable anymore, let's switch to other node. 8 | 9 | ![example](./images/fixture-example.png) 10 | 11 | #### Enabling fixtures 12 | 13 | Example of enabling a fixture using an environment variable: 14 | ``` 15 | export WW_TOKEN="FacebookCaptchaTo500,SomethingElse" 16 | ``` 17 | 18 | Example using config: 19 | ``` 20 | return [ 21 | 'fixtures' => 'FacebookCaptchaTo500', 22 | ]; 23 | ``` 24 | 25 | #### Custom fixtures 26 | 27 | 1. Should be available for the composer autoload, so a `custom-fixtures` or vendor is a good place. 28 | Easiest way is to create a fixture in `custom-fixtures` directory with a `Wolnosciowiec\CustomFixtures\` namespace. 29 | 30 | 2. Every fixture should implement `Wolnosciowiec\WebProxy\Fixtures\FixtureInterface`. 31 | 3. A registration is necessary through configuration variable or environment variable 32 | 33 | Example (environment variable): 34 | ``` 35 | export WW_FIXTURES="MyFixture,NotFoundTo500" 36 | export WW_FIXTURES_MAPPING='{"MyFixture": "\\Wolnosciowiec\\CustomFixtures\\MyFixture"}' 37 | ``` 38 | 39 | Example (configuration file): 40 | ``` 41 | 'MyFixture', 45 | 'fixtures_mapping' => '{"MyFixture": "\\Wolnosciowiec\\CustomFixtures\\MyFixture"}', 46 | ]; 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/images/fixture-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riotkit-org/web-proxy/2e586aeff18b6706b61b31f2dddbef6e27004540/docs/images/fixture-example.png -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 14 | 15 | 16 | 17 | ./tests 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | ./ 28 | 29 | ./Resources 30 | ./Tests 31 | ./vendor 32 | ./config.php 33 | ./web 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/Controllers/BaseController.php: -------------------------------------------------------------------------------- 1 | getHeader('ww-no-external-proxy')[0] ?? ''; 14 | 15 | return $this->getBooleanValue($headerValue); 16 | } 17 | 18 | private function getBooleanValue(string $value) 19 | { 20 | return in_array($value, ['1', 'true'], true); 21 | } 22 | 23 | protected function createConnectionAddressString(bool $withExternalProxy, ProxySelector $selector): ?string 24 | { 25 | if (!$withExternalProxy) { 26 | return null; 27 | } 28 | 29 | $address = $selector->getHTTPProxy(); 30 | 31 | if ($address instanceof ProxyServerAddress) { 32 | $address->prepare(); 33 | return $address->getFormatted(); 34 | } 35 | 36 | return null; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Controllers/PassThroughController.php: -------------------------------------------------------------------------------- 1 | maxRetries = $maxRetries; 57 | $this->clientFactory = $clientFactory; 58 | $this->logger = $logger; 59 | $this->fixturesManager = $fixturesManager; 60 | } 61 | 62 | /** 63 | * @param ForwardableRequest $request 64 | * @throws HttpException 65 | * @return string 66 | */ 67 | private function getRequestedURL(ForwardableRequest $request) 68 | { 69 | $url = $request->getDestinationUrl(); 70 | 71 | if (!$url) { 72 | throw new HttpException('Missing target URL, did you provided a one-time token, WW-URL header or WW_URL environment variable?', Codes::HTTP_MISSING_URL); 73 | } 74 | 75 | return $url; 76 | } 77 | 78 | /** 79 | * @param ForwardableRequest $request 80 | * @throws \Exception 81 | * @return Response 82 | */ 83 | public function executeAction(ForwardableRequest $request): ResponseInterface 84 | { 85 | try { 86 | $request = $request->withProtocolVersion('1.1'); 87 | 88 | } catch (\Exception $e) { 89 | $this->logger->error('Invalid request: ' . $e->getMessage()); 90 | 91 | return new Response(400, [], json_encode([ 92 | 'success' => false, 93 | 'message' => $e->getMessage(), 94 | ])); 95 | } 96 | 97 | try { 98 | $this->logger->notice('Forwarding to "' . $this->getRequestedURL($request) . '"'); 99 | 100 | // forward the request and get the response. 101 | $response = $this->clientFactory->create(!$this->hasDisabledExternalProxy($request)) 102 | ->forward($request) 103 | ->to($this->getRequestedURL($request)); 104 | 105 | $response = $response->withHeader('X-Wolnosciowiec-Proxy', $this->clientFactory->getProxyIPAddress(!$this->hasDisabledExternalProxy($request))); 106 | 107 | } catch (RequestException $e) { 108 | 109 | // try again in case of connection failure 110 | if ( 111 | ($e instanceof ConnectException || $e instanceof ServerException) 112 | && $this->maxRetries > $this->retries 113 | ) { 114 | $this->retries++; 115 | 116 | $this->logger->error('Retrying request(' . $this->retries . '/' . $this->maxRetries . ')'); 117 | return $this->executeAction($request); 118 | } 119 | 120 | $response = $e->getResponse(); 121 | 122 | if (!$response instanceof Response) { 123 | $response = new JsonResponse(['error' => $e->getMessage()], 500); 124 | $this->logger->notice('Error response: ' . $e->getMessage()); 125 | } 126 | } 127 | 128 | // apply fixtures 129 | $response = $this->fixturesManager->fix($request, $response); 130 | 131 | // add optional headers 132 | $response = $response->withHeader('X-Target-Url', $request->getDestinationUrl()); 133 | $response = $response->withHeader('X-Powered-By', 'Wolnosciowiec WebProxy'); 134 | 135 | return $this->fixResponseHeaders($response); 136 | } 137 | 138 | /** 139 | * @param ResponseInterface $response 140 | * @return ResponseInterface|static 141 | */ 142 | private function fixResponseHeaders(ResponseInterface $response) 143 | { 144 | // fix: empty response if page is using gzip (Zend Diactoros is trying to do the same, but it's doing it incorrectly) 145 | if (!$response->hasHeader('Content-Length')) { 146 | $response = $response->withAddedHeader('Content-Length', strlen((string)$response->getBody())); 147 | } 148 | 149 | // we are not using any encoding at the output 150 | $response = $response->withoutHeader('Transfer-Encoding'); 151 | 152 | return $response; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/Controllers/ProxySelectorController.php: -------------------------------------------------------------------------------- 1 | proxySelector = $proxySelector; 20 | } 21 | 22 | public function executeAction(RequestInterface $request): ResponseInterface 23 | { 24 | return new JsonResponse([ 25 | 'address' => $this->createConnectionAddressString(true, $this->proxySelector) 26 | ]); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Controllers/RenderController.php: -------------------------------------------------------------------------------- 1 | proxySelector = $proxySelector; 44 | $this->prerenderer = $prerenderer; 45 | $this->enabled = $enabled; 46 | $this->fixturesManager = $fixturesManager; 47 | } 48 | 49 | public function executeAction(RequestInterface $request): ResponseInterface 50 | { 51 | if (!$this->enabled) { 52 | return new Response(500, [], 'The prerender functionality was not enabled.'); 53 | } 54 | 55 | $proxyAddress = $this->createConnectionAddressString( 56 | !$this->hasDisabledExternalProxy($request), 57 | $this->proxySelector 58 | ); 59 | $targetUrl = (string) ($request->getHeader('ww-url')[0] ?? ''); 60 | 61 | $response = new Response( 62 | 200, 63 | [ 64 | 'X-Wolnosciowiec-Proxy' => $proxyAddress, 65 | 'X-Target-Url' => $targetUrl 66 | ], 67 | $this->prerenderer->render($targetUrl, $proxyAddress) 68 | ); 69 | 70 | // run all fixtures (middlewares) to process the end result 71 | $response = $this->fixturesManager->fix($request, $response); 72 | 73 | return $this->validateResponse($response); 74 | } 75 | 76 | private function validateResponse(ResponseInterface $response): ResponseInterface 77 | { 78 | if (trim($response->getBody()->getContents()) === '') { 79 | return new Response(503, $response->getHeaders(), 'Proxy error, got empty HTML'); 80 | } 81 | 82 | return $response; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/DependencyInjection/LibraryServices.php: -------------------------------------------------------------------------------- 1 | function () { 12 | $guzzle = new \GuzzleHttp\Client([ 13 | 'timeout' => 30 14 | ]); 15 | 16 | $client = new Goutte\Client(); 17 | $client->setClient($guzzle); 18 | 19 | return $client; 20 | }, 21 | 22 | Cache::class => function (Container $container) { 23 | $cache = $container->get('config')->get('cache'); 24 | 25 | if (!$cache instanceof Cache) { 26 | throw new \Exception('"cache" configuration key should be an instance of \Doctrine\Common\Cache\Cache'); 27 | } 28 | 29 | return $cache; 30 | }, 31 | 32 | LoggerInterface::class => function () { 33 | $log = new Logger('wolnosciowiec.webproxy'); 34 | $log->pushHandler(new StreamHandler(__DIR__ . '/../../var/app.log', Logger::INFO)); 35 | 36 | if (PHP_SAPI === 'cli') { 37 | $log->pushHandler(new StreamHandler("php://stdout", Logger::DEBUG)); 38 | } 39 | 40 | return $log; 41 | }, 42 | ]; 43 | -------------------------------------------------------------------------------- /src/DependencyInjection/Services.php: -------------------------------------------------------------------------------- 1 | function () { 33 | return new ArrayCollection(require __DIR__ . '/../../config.php'); 34 | }, 35 | 36 | FixturesManager::class => function (Container $container) { 37 | return new FixturesManager( 38 | $container->get('config')->get('fixtures'), 39 | $container->get('config')->get('fixtures_mapping') 40 | ); 41 | }, 42 | 43 | ProxySelector::class => function (Container $container) { 44 | return new ProxySelector($container->get(ProxyProviderInterface::class)); 45 | }, 46 | 47 | ProxyProviderFactory::class => function (Container $container) { 48 | /** @var ArrayCollection $config */ 49 | $config = $container->get('config'); 50 | 51 | return new ProxyProviderFactory( 52 | (string) $config->get('externalProxyProviders'), 53 | $container 54 | ); 55 | }, 56 | 57 | RequestFactory::class => function () { 58 | return new RequestFactory(); 59 | }, 60 | 61 | ProxyClientFactory::class => function (Container $container) { 62 | return new ProxyClientFactory( 63 | $container->get(ProxySelector::class), 64 | (int)$container->get('config')->get('connectionTimeout') 65 | ); 66 | }, 67 | 68 | Prerenderer::class => function (Container $container) { 69 | return new Prerenderer( 70 | new Client(), 71 | (string) $container->get('config')->get('prerendererUrl') 72 | ); 73 | }, 74 | 75 | 76 | // controllers 77 | PassThroughController::class => function (Container $container) { 78 | return new PassThroughController( 79 | (int)$container->get('config')->get('maxRetries'), 80 | $container->get(ProxyClientFactory::class), 81 | $container->get(LoggerInterface::class), 82 | $container->get(FixturesManager::class) 83 | ); 84 | }, 85 | 86 | ProxySelectorController::class => function (Container $container) { 87 | return new ProxySelectorController($container->get(ProxySelector::class)); 88 | }, 89 | 90 | RenderController::class => function (Container $container) { 91 | return new RenderController( 92 | $container->get(ProxySelector::class), 93 | $container->get(Prerenderer::class), 94 | $container->get('config')->get('prerendererEnabled'), 95 | $container->get(FixturesManager::class) 96 | ); 97 | }, 98 | 99 | // providers 100 | FreeProxyListProvider::class => function (Container $container) { 101 | return new FreeProxyListProvider( 102 | $container->get(Goutte\Client::class), 103 | $container->get(LoggerInterface::class) 104 | ); 105 | }, 106 | 107 | HideMyNameProvider::class => function (Container $container) { 108 | return new HideMyNameProvider( 109 | $container->get(Goutte\Client::class), 110 | $container->get(LoggerInterface::class) 111 | ); 112 | }, 113 | 114 | GatherProxyProvider::class => function (Container $container) { 115 | return new GatherProxyProvider( 116 | $container->get(Goutte\Client::class), 117 | $container->get(LoggerInterface::class) 118 | ); 119 | }, 120 | 121 | ProxyListOrgProvider::class => function (Container $container) { 122 | return new ProxyListOrgProvider( 123 | $container->get(Goutte\Client::class), 124 | $container->get(LoggerInterface::class) 125 | ); 126 | }, 127 | 128 | CachedProvider::class => function (Container $container) { 129 | return new CachedProvider( 130 | $container->get(Cache::class), 131 | $container->get(ProxyProviderFactory::class)->create(), 132 | (int) ($container->get(Config::class)->get('cacheTtl') ?? 360) 133 | ); 134 | }, 135 | 136 | ProxyProviderInterface::class => function (Container $container) { 137 | return $container->get(CachedProvider::class); 138 | }, 139 | 140 | '\Wolnosciowiec\WebProxy\Providers\Proxy\TorProxyProvider' => function (Container $container) { 141 | $config = $container->get(Config::class); 142 | 143 | return new TorProxyProvider( 144 | explode(',', $config->get('torProxies') ?: ''), 145 | (int) $config->get('torVirtualProxiesNum') ?: 5, 146 | $container->get(\Goutte\Client::class), 147 | $container->get(LoggerInterface::class) 148 | ); 149 | }, 150 | 151 | ForwardableRequest::class => function (RequestFactory $factory) { 152 | return $factory->createFromGlobals(); 153 | }, 154 | 155 | AuthenticationMiddleware::class => function (Container $container) { 156 | return new AuthenticationMiddleware([ 157 | $container->get(OneTimeBrowseTokenChecker::class), 158 | $container->get(TokenAuthChecker::class) 159 | ]); 160 | }, 161 | 162 | OneTimeTokenParametersConversionMiddleware::class => function (Container $container) { 163 | return new OneTimeTokenParametersConversionMiddleware($container->get(Config::class)->get('encryptionKey')); 164 | }, 165 | 166 | ProxyStaticContentMiddleware::class => function (Container $container) { 167 | return new ProxyStaticContentMiddleware( 168 | $container->get(ContentProcessor::class), 169 | $container->get(Config::class) 170 | ); 171 | }, 172 | 173 | ApplicationMiddleware::class => function (Container $container) { 174 | return new ApplicationMiddleware( 175 | $container->get(PassThroughController::class), 176 | $container->get(ProxySelectorController::class), 177 | $container->get(RenderController::class) 178 | ); 179 | }, 180 | 181 | ContentProcessor::class => function (Container $container) { 182 | return new ContentProcessor([ 183 | $container->get(HtmlProcessor::class), 184 | $container->get(CssProcessor::class) 185 | ]); 186 | }, 187 | 188 | OneTimeBrowseTokenChecker::class => function (Container $container) { 189 | return new OneTimeBrowseTokenChecker($container->get(Config::class)); 190 | }, 191 | 192 | OneTimeTokenUrlGenerator::class => function (Container $container) { 193 | return new OneTimeTokenUrlGenerator($container->get(Config::class)); 194 | }, 195 | 196 | Config::class => function (Container $container) { 197 | return new Config(require __DIR__ . '/../../config.php'); 198 | } 199 | ]; 200 | -------------------------------------------------------------------------------- /src/Entity/ForwardableRequest.php: -------------------------------------------------------------------------------- 1 | forwardToUrl = $headers[InputParams::HEADER_TARGET_URL][0] ?? ($queryParams[InputParams::QUERY_TARGET_URL] ?? ''); 29 | $this->token = $headers[InputParams::HEADER_TOKEN][0] ?? ($queryParams[InputParams::QUERY_TOKEN] ?? ''); 30 | $this->processOutput = count( 31 | array_filter([ 32 | in_array(($headers[InputParams::HEADER_CAN_PROCESS][0] ?? ''), ['true', '1'], true), 33 | in_array(($queryParams[InputParams::QUERY_CAN_PROCESS] ?? ''), ['true', '1'], true), 34 | ]) 35 | ) > 0; 36 | 37 | if (in_array(($headers[InputParams::HEADER_CAN_PROCESS][0] ?? ''), ['false', '0'], true)) { 38 | $this->processOutput = false; 39 | } 40 | } 41 | 42 | /** 43 | * Gets the URL we are forwarding request 44 | * 45 | * @return string 46 | */ 47 | public function getDestinationUrl(): string 48 | { 49 | return $this->forwardToUrl; 50 | } 51 | 52 | /** 53 | * @param string $url 54 | * @return ForwardableRequest 55 | */ 56 | public function withNewDestinationUrl(string $url): ForwardableRequest 57 | { 58 | $request = clone $this; 59 | $request->forwardToUrl = $url; 60 | 61 | return $request; 62 | } 63 | 64 | /** 65 | * @return bool 66 | */ 67 | public function canOutputBeProcessed(): bool 68 | { 69 | return $this->processOutput; 70 | } 71 | 72 | /** 73 | * @param bool $processOutput 74 | * @return ForwardableRequest 75 | */ 76 | public function withOutputProcessing(bool $processOutput): ForwardableRequest 77 | { 78 | $request = clone $this; 79 | $request->processOutput = $processOutput; 80 | 81 | return $request; 82 | } 83 | 84 | /** 85 | * @return string 86 | */ 87 | public function getToken(): string 88 | { 89 | return $this->token; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Entity/ProxyServerAddress.php: -------------------------------------------------------------------------------- 1 | address = $address; 32 | return $this; 33 | } 34 | 35 | /** 36 | * @param int $port 37 | * @return ProxyServerAddress 38 | */ 39 | public function setPort(int $port): ProxyServerAddress 40 | { 41 | $this->port = $port; 42 | return $this; 43 | } 44 | 45 | /** 46 | * @param string $schema 47 | * @return ProxyServerAddress 48 | */ 49 | public function setSchema(string $schema): ProxyServerAddress 50 | { 51 | $this->schema = $schema; 52 | return $this; 53 | } 54 | 55 | /** 56 | * @return bool 57 | */ 58 | public function isSecure(): bool 59 | { 60 | return $this->getSchema() === 'https'; 61 | } 62 | 63 | /** 64 | * @return string 65 | */ 66 | public function getAddress(): string 67 | { 68 | return $this->address; 69 | } 70 | 71 | /** 72 | * @return int 73 | */ 74 | public function getPort(): int 75 | { 76 | return $this->port; 77 | } 78 | 79 | /** 80 | * @return string 81 | */ 82 | public function getSchema(): string 83 | { 84 | return $this->schema; 85 | } 86 | 87 | /** 88 | * @return string 89 | */ 90 | public function getFormatted(): string 91 | { 92 | return 'http://' . $this->getAddress() . ':' . $this->getPort(); 93 | } 94 | 95 | public function __toString(): string 96 | { 97 | return $this->getFormatted(); 98 | } 99 | 100 | /** 101 | * Decides if the address requires a verification or not by the background process 102 | * For example the TOR address would not require a verification as it is handing the 103 | * proxy freshness by itself 104 | * 105 | * @return bool 106 | */ 107 | public function requiresVerification(): bool 108 | { 109 | return true; 110 | } 111 | 112 | public function prepare() 113 | { 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/Entity/TorProxyServerAddress.php: -------------------------------------------------------------------------------- 1 | torManagementPort = $torManagementPort; 23 | $this->authPassword = $authPassword; 24 | } 25 | 26 | /** 27 | * When the request is going to be executed then 28 | * the exit node needs to be switched 29 | */ 30 | public function prepare() 31 | { 32 | $fp = fsockopen($this->getAddress(), $this->torManagementPort, $errNo, $errStr, 30); 33 | 34 | if ($this->authPassword) { 35 | fwrite($fp, 'AUTHENTICATE "' . $this->authPassword . "\"\r\n"); 36 | } 37 | 38 | fwrite($fp, "SIGNAL NEWNYM\r\n"); 39 | fclose($fp); 40 | } 41 | 42 | /** 43 | * TOR network handles this inside of the network itself 44 | * 45 | * @return bool 46 | */ 47 | public function requiresVerification(): bool 48 | { 49 | return false; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Exception/AccessDeniedException.php: -------------------------------------------------------------------------------- 1 | proxySelector = $proxySelector; 35 | $this->connectionTimeout = $connectionTimeout; 36 | } 37 | 38 | /** 39 | * @param bool $withExternalProxy 40 | * 41 | * @return Proxy 42 | */ 43 | public function create(bool $withExternalProxy = true) 44 | { 45 | return new Proxy(new GuzzleAdapter( 46 | new Client($this->getClientOptions($withExternalProxy)) 47 | )); 48 | } 49 | 50 | /** 51 | * @param bool $withExternalProxy 52 | * 53 | * @return array 54 | */ 55 | public function getClientOptions(bool $withExternalProxy): array 56 | { 57 | if (empty($this->options)) { 58 | $this->options = array_filter([ 59 | 'proxy' => $this->createConnectionAddressString($withExternalProxy, $this->proxySelector), 60 | 'connect_timeout' => $this->connectionTimeout, 61 | 'read_timeout' => $this->connectionTimeout, 62 | 'timeout' => $this->connectionTimeout, 63 | ]); 64 | } 65 | 66 | return $this->options; 67 | } 68 | 69 | private function createConnectionAddressString(bool $withExternalProxy, ProxySelector $selector): ?string 70 | { 71 | if (!$withExternalProxy) { 72 | return null; 73 | } 74 | 75 | $address = $selector->getHTTPProxy(); 76 | 77 | if ($address instanceof ProxyServerAddress) { 78 | $address->prepare(); 79 | return $address->getFormatted(); 80 | } 81 | 82 | return null; 83 | } 84 | 85 | /** 86 | * @param bool $withExternalProxy 87 | * 88 | * @return string 89 | */ 90 | public function getProxyIPAddress(bool $withExternalProxy): string 91 | { 92 | return $this->getClientOptions($withExternalProxy)['proxy']['http'] ?? ''; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/Factory/ProxyProviderFactory.php: -------------------------------------------------------------------------------- 1 | providersNames = $providerNames; 29 | $this->container = $container; 30 | } 31 | 32 | public function create(): ProxyProviderInterface 33 | { 34 | return new ChainProvider($this->buildProviders()); 35 | } 36 | 37 | /** 38 | * @throws \Exception 39 | * @return ProxyProviderInterface[] 40 | */ 41 | public function buildProviders() 42 | { 43 | $providers = []; 44 | $names = str_replace(' ', '', $this->providersNames); 45 | $names = array_filter(explode(',', $names)); 46 | 47 | $defaultNamespace = '\\Wolnosciowiec\\WebProxy\\Providers\\Proxy\\'; 48 | 49 | foreach ($names as $name) { 50 | if (class_exists($defaultNamespace . $name)) { 51 | $fullName = $defaultNamespace . $name; 52 | 53 | } elseif (class_exists($name)) { 54 | $fullName = $name; 55 | 56 | } else { 57 | throw new \Exception('Invalid provider name "' . $name . '", please check the configuration. ' . 58 | 'Looked at: "' . $defaultNamespace . $name . '"'); 59 | } 60 | 61 | $providers[$fullName] = $this->container->get($fullName); 62 | } 63 | 64 | return $providers; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Factory/RequestFactory.php: -------------------------------------------------------------------------------- 1 | rewriteRequestToOwnRequest($request); 32 | $currentHost = $request->getUri()->getHost(); 33 | 34 | $requestedUrl = new Uri($destinationUrl); 35 | $requestedUrl = $requestedUrl->withPath(''); 36 | 37 | $request = $request->withUri($requestedUrl); 38 | 39 | if ($currentHost === $request->getUri()->getHost()) { // @codeCoverageIgnore 40 | throw new HttpException('Cannot make a request to the same host as we are'); // @codeCoverageIgnore 41 | } 42 | 43 | return $request; 44 | } 45 | 46 | /** 47 | * Creates the request object basing on globals (in this case it's a HTTP header accessible from global variable) 48 | * 49 | * @throws HttpException 50 | * @return ForwardableRequest 51 | */ 52 | public function createFromGlobals(): ForwardableRequest 53 | { 54 | return $this->create((string) ($_SERVER['HTTP_WW_TARGET_URL'] ?? '')); 55 | } 56 | 57 | private function rewriteRequestToOwnRequest(ServerRequest $request): ForwardableRequest 58 | { 59 | return new ForwardableRequest( 60 | $_SERVER, 61 | $request->getUploadedFiles(), 62 | $request->getUri()->__toString(), 63 | $request->getMethod(), 64 | $request->getBody(), 65 | $request->getHeaders(), 66 | $request->getCookieParams(), 67 | $request->getQueryParams(), 68 | $request->getParsedBody(), 69 | $request->getProtocolVersion() 70 | ); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Fixtures/FacebookCaptchaTo500.php: -------------------------------------------------------------------------------- 1 | findHost($request); 20 | 21 | // enable only on *facebook.com 22 | if (strpos($host, 'facebook.com') === false) { 23 | return $response; 24 | } 25 | 26 | if (strpos((string) $response->getBody(), 'id="captcha_submit"') !== false) { 27 | $response = $response->withStatus(500, 'Captcha found'); 28 | } 29 | 30 | return $response; 31 | } 32 | 33 | protected function findHost(RequestInterface $request) 34 | { 35 | if (count($request->getHeader('ww-url')) > 0) { 36 | return parse_url($request->getHeader('ww-url')[0], PHP_URL_HOST); 37 | } 38 | 39 | if (count($request->getHeader('Host')) > 0) { 40 | return $request->getHeader('Host')[0]; 41 | } 42 | 43 | return parse_url($_SERVER['HTTP_WW_TARGET_URL'], PHP_URL_HOST); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Fixtures/FixtureInterface.php: -------------------------------------------------------------------------------- 1 | getStatusCode() === 404) { 20 | $response = $response->withStatus(500, 'Bad output status code'); 21 | } 22 | 23 | return $response; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/InputParams.php: -------------------------------------------------------------------------------- 1 | passThroughController = $passThroughController; 38 | $this->selectorController = $selectorController; 39 | $this->renderController = $renderController; 40 | } 41 | 42 | /** 43 | * @param ForwardableRequest $request 44 | * @param ResponseInterface $response 45 | * @param callable $next 46 | * 47 | * @throws \Exception 48 | * @return \GuzzleHttp\Psr7\Response 49 | */ 50 | public function __invoke(ForwardableRequest $request, ResponseInterface $response, callable $next) 51 | { 52 | // remove header that should not be passed to the destination server 53 | $request = $request->withoutHeader(InputParams::HEADER_TARGET_URL); 54 | 55 | // REQUEST_URI is a non-rewritten URI, original that was passed as a request to the webproxy 56 | if (($_SERVER['REQUEST_URI'] ?? '') === '/__webproxy/get-ip') { 57 | return $next($request, $this->selectorController->executeAction($request)); 58 | 59 | } elseif (($_SERVER['REQUEST_URI'] ?? '') === '/__webproxy/render') { 60 | return $next($request, $this->renderController->executeAction($request)); 61 | } 62 | 63 | return $next($request, $this->passThroughController->executeAction($request)); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Middleware/AuthenticationMiddleware.php: -------------------------------------------------------------------------------- 1 | securityCheckers = $securityCheckers; 27 | } 28 | 29 | /** 30 | * @param ForwardableRequest $request 31 | * @param ResponseInterface $response 32 | * @param callable $next 33 | * 34 | * @throws \Wolnosciowiec\WebProxy\Exception\AccessDeniedException 35 | * @return ResponseInterface 36 | */ 37 | public function __invoke(ForwardableRequest $request, ResponseInterface $response, callable $next) 38 | { 39 | foreach ($this->securityCheckers as $checker) { 40 | if ($checker->canHandle($request) && $checker->isValid($request)) { 41 | return $next( 42 | $request->withoutHeader(InputParams::HEADER_TOKEN), // the token header is no longer needed 43 | $response 44 | ); 45 | } 46 | } 47 | 48 | throw new AccessDeniedException(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Middleware/OneTimeTokenParametersConversionMiddleware.php: -------------------------------------------------------------------------------- 1 | encryptionKey = $encryptionKey; 24 | } 25 | 26 | /** 27 | * @param ForwardableRequest $request 28 | * @param ResponseInterface $response 29 | * @param callable $next 30 | * 31 | * @return mixed 32 | */ 33 | public function __invoke(ForwardableRequest $request, ResponseInterface $response, callable $next) 34 | { 35 | $oneTimeToken = $this->unescape($request->getQueryParams()[InputParams::QUERY_ONE_TIME_TOKEN] ?? ''); 36 | 37 | if (!$oneTimeToken) { 38 | return $next($request, $response); 39 | } 40 | 41 | $decrypted = CryptoJSAES::decrypt($oneTimeToken, $this->encryptionKey); 42 | 43 | try { 44 | $decoded = \GuzzleHttp\json_decode($decrypted, true); 45 | 46 | } catch (\InvalidArgumentException $exception) { 47 | return new Response(403, [], json_encode([ 48 | 'message' => 'The one-time-token cannot be decoded. ' . 49 | 'Was it properly encoded with a proper passphrase and in proper format?' 50 | ])); 51 | } 52 | 53 | 54 | if ($decoded[InputParams::ONE_TIME_TOKEN_PROCESS] ?? false) { 55 | $request = $request->withOutputProcessing((bool) $decoded[InputParams::ONE_TIME_TOKEN_PROCESS]); 56 | } 57 | 58 | return $next( 59 | $request->withNewDestinationUrl($decoded[InputParams::ONE_TIME_TOKEN_PROPERTY_URL] ?? ''), 60 | $response 61 | ); 62 | } 63 | 64 | /** 65 | * @param string $queryParameter 66 | * @return mixed 67 | */ 68 | private function unescape(string $queryParameter) 69 | { 70 | return str_replace(' ', '+', $queryParameter); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Middleware/ProxyStaticContentMiddleware.php: -------------------------------------------------------------------------------- 1 | processor = $processor; 34 | $this->enabled = $config->getOptional('contentProcessingEnabled', false); 35 | } 36 | 37 | /** 38 | * @param ForwardableRequest $request 39 | * @param ResponseInterface $response 40 | * @param callable $next 41 | * 42 | * @throws \Exception 43 | * @return ResponseInterface 44 | */ 45 | public function __invoke(ForwardableRequest $request, ResponseInterface $response, callable $next) 46 | { 47 | $mimeType = $this->getMimeType($response); 48 | 49 | if (!$this->isEnabled($request) || !$this->processor->canProcess($mimeType)) { 50 | return $next($request, $response); 51 | } 52 | 53 | $processedBody = $this->processor->process($request, (string) $response->getBody(), $mimeType); 54 | 55 | // append processed body 56 | $body = new Stream('php://temp', 'wb+'); 57 | $body->write($processedBody); 58 | $body->rewind(); 59 | $response = $response->withBody($body); 60 | $response = $response->withHeader('Content-Length', strlen($processedBody)); 61 | 62 | // add information helpful for debugging 63 | $response = $response->withHeader('X-Processed-With', 'Wolnosciowiec'); 64 | 65 | return $next($request, $response); 66 | } 67 | 68 | /** 69 | * @param ResponseInterface $response 70 | * @return string 71 | */ 72 | private function getMimeType(ResponseInterface $response) 73 | { 74 | $parts = explode(';', $response->getHeader('Content-Type')[0] ?? ''); 75 | return strtolower($parts[0] ?? ''); 76 | } 77 | 78 | /** 79 | * @param ForwardableRequest $request 80 | * @return bool 81 | */ 82 | private function isEnabled(ForwardableRequest $request): bool 83 | { 84 | // in every request there must be a parameter explicitly defined 85 | if (!$request->canOutputBeProcessed()) { 86 | return false; 87 | } 88 | 89 | return $this->enabled; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Providers/Proxy/BaseProvider.php: -------------------------------------------------------------------------------- 1 | client = $client; 23 | $this->logger = $logger; 24 | } 25 | 26 | /** 27 | * @param string $time eg. "2 seconds", "1 minute", "2 minutes" 28 | * @return bool 29 | */ 30 | protected function isEnoughFresh(string $time): bool 31 | { 32 | $parts = explode(' ', $time); 33 | $multiply = 60 * 60 * 24 * 2; 34 | 35 | if (in_array($parts[1], ['minute', 'minutes'])) { 36 | $multiply = 60; 37 | } 38 | elseif (in_array($parts[1], ['hour', 'hours'])) { 39 | $multiply = 60 * 60; 40 | } 41 | elseif (in_array($parts[1], ['day', 'days'])) { 42 | $multiply = 60 * 60 * 24; 43 | } 44 | elseif (in_array($parts[1], ['second', 'seconds'])) { 45 | $multiply = 1; 46 | } 47 | 48 | $this->logger->debug('Freshness: ' . (int)$parts[0] * $multiply); 49 | 50 | return ((int)$parts[0] * $multiply) <= (60 * 8); // 8 minutes 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Providers/Proxy/CachedProvider.php: -------------------------------------------------------------------------------- 1 | cache = $cache; 36 | $this->provider = $provider; 37 | $this->ttl = $ttl; 38 | } 39 | 40 | /** 41 | * @inheritdoc 42 | */ 43 | public function collectAddresses(): array 44 | { 45 | if ($this->cache->contains(self::CACHE_KEY)) { 46 | $addresses = $this->getFromCache(); 47 | 48 | if (!empty($addresses)) { 49 | return $addresses; 50 | } 51 | } 52 | 53 | $addresses = $this->provider->collectAddresses(); 54 | $this->cacheResult($addresses); 55 | 56 | return $addresses; 57 | } 58 | 59 | public function getFromCache(): array 60 | { 61 | $data = unserialize($this->cache->fetch(self::CACHE_KEY)); 62 | 63 | if ($data['expiration'] <= time()) { 64 | return []; 65 | } 66 | 67 | return $data['data']; 68 | } 69 | 70 | public function cacheResult(array $addresses) 71 | { 72 | $this->cache->save(self::CACHE_KEY, serialize([ 73 | 'data' => $addresses, 74 | 'expiration' => time() + $this->getExpirationTime(), 75 | ])); 76 | } 77 | 78 | protected function getExpirationTime(): int 79 | { 80 | return $this->ttl; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/Providers/Proxy/ChainProvider.php: -------------------------------------------------------------------------------- 1 | providers = $providers; 23 | return $this; 24 | } 25 | 26 | /** 27 | * @param array $providers 28 | */ 29 | public function __construct(array $providers = []) 30 | { 31 | $this->setProviders($providers); 32 | } 33 | 34 | /** 35 | * @return array 36 | */ 37 | public function collectAddresses(): array 38 | { 39 | $addresses = []; 40 | 41 | foreach ($this->providers as $provider) { 42 | $addresses = array_merge($addresses, $provider->collectAddresses()); 43 | } 44 | 45 | return $addresses; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Providers/Proxy/DummyProvider.php: -------------------------------------------------------------------------------- 1 | mode === self::RETURN_NONE) { 23 | return []; 24 | } 25 | 26 | return [ 27 | (new ProxyServerAddress()) 28 | ->setSchema('https') 29 | ->setPort(443) 30 | ->setAddress('localhost'), 31 | (new ProxyServerAddress()) 32 | ->setSchema('http') 33 | ->setPort(80) 34 | ->setAddress('wolnosciowiec.local'), 35 | 36 | (new ProxyServerAddress()) 37 | ->setSchema('http') 38 | ->setPort(8080) 39 | ->setAddress('zsp.net.pl'), 40 | ]; 41 | } 42 | 43 | /** 44 | * @param int $mode 45 | * @return DummyProvider 46 | */ 47 | public function setMode(int $mode): DummyProvider 48 | { 49 | $this->mode = $mode; 50 | return $this; 51 | } 52 | } -------------------------------------------------------------------------------- /src/Providers/Proxy/FreeProxyCzProvider.php: -------------------------------------------------------------------------------- 1 | minUptime = $minUptime; 28 | $this->maxPing = $maxPing; 29 | } 30 | 31 | /** 32 | * @inheritdoc 33 | */ 34 | public function collectAddresses(): array 35 | { 36 | $addresses = []; 37 | $crawler = $this->client->request('GET', 'http://free-proxy.cz/en/proxylist/country/all/https/ping/level1/1', [], [], [ 38 | 'HTTP_USER_AGENT' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/51.0.2704.79 Chrome/51.0.2704.79 Safari/537.36' 39 | ]); 40 | $rowsInTheTable = $crawler->filterXPath('//table[@id="proxy_list"]/tbody[1]/tr'); 41 | 42 | if (!$rowsInTheTable->count()) { 43 | throw new \Exception('The crawler for free-proxy.cz seems to not work anymore'); 44 | } 45 | 46 | foreach ($rowsInTheTable as $element) { 47 | // 0 => IP Address 48 | // 1 => Port 49 | // 2 => Protocol 50 | // 8 => Uptime 51 | // 9 => Response 52 | 53 | try { 54 | $data = [ 55 | 'ip' => $element->ownerDocument->saveHTML($element->childNodes[0]), 56 | 'port' => $element->ownerDocument->saveHTML($element->childNodes[1]), 57 | 'proto' => $element->ownerDocument->saveHTML($element->childNodes[2]), 58 | 'uptime' => $element->ownerDocument->saveHTML($element->childNodes[8]), 59 | 'ping' => $element->ownerDocument->saveHTML($element->childNodes[9]) 60 | ]; 61 | 62 | $data = array_map(function (string $str) { return strip_tags($str); }, $data); 63 | 64 | } catch (\Throwable $exception) { 65 | $this->logger->debug($exception); 66 | continue; 67 | } 68 | 69 | preg_match('/([0-9\.]+)\%/', $data['uptime'], $uptimeMatches); 70 | preg_match('/(\d+) ms/', $data['ping'], $pingMatches); 71 | 72 | if (!$uptimeMatches || (int) $uptimeMatches[1] < $this->minUptime) { 73 | $this->logger->debug( 74 | $data['ip'] . ' has uptime < ' . $this->minUptime, 75 | [$uptimeMatches[1] ?? ''] 76 | ); 77 | continue; 78 | } 79 | 80 | if (!$pingMatches || (int) $pingMatches[1] > $this->maxPing) { 81 | $this->logger->debug( 82 | $data['ip'] . ' has response time higher than ' . $this->maxPing . ' ms', 83 | [$pingMatches[1] ?? ''] 84 | ); 85 | } 86 | 87 | $address = new ProxyServerAddress(); 88 | $address->setAddress($data['ip']) 89 | ->setPort((int) $data['port']) 90 | ->setSchema('https'); 91 | 92 | $addresses[] = $address; 93 | } 94 | 95 | return $addresses; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/Providers/Proxy/FreeProxyListProvider.php: -------------------------------------------------------------------------------- 1 | client->request('GET', 'http://free-proxy-list.net'); 16 | $rows = $crawler->filter('#proxylisttable tr'); 17 | 18 | $addresses = $rows->each(function (Crawler $node) { 19 | $collection = $node->filter('td'); 20 | 21 | // cells mapping 22 | $lastVerificationTime = @$collection->getNode(7)->textContent; 23 | $proxyType = @$collection->getNode(4)->textContent; 24 | $proxyPort = (int)@$collection->getNode(1)->textContent; 25 | $proxyIP = @$collection->getNode(0)->textContent; 26 | $proxySchema =@ $collection->getNode(6)->textContent == 'yes' ? 'https' : 'http'; 27 | 28 | if (!$proxyIP || !$proxyPort || strlen($lastVerificationTime) === 0 || !$proxyType) { 29 | return null; 30 | } 31 | 32 | if ($this->isEnoughFresh($lastVerificationTime) === false) { 33 | $this->logger->notice('[free-proxy-list.net] The proxy is old'); 34 | return null; 35 | } 36 | 37 | if ($proxyType !== 'elite proxy' || $proxySchema !== 'https') { 38 | return null; 39 | } 40 | 41 | $address = new ProxyServerAddress(); 42 | $address->setAddress($proxyIP); 43 | $address->setPort($proxyPort); 44 | $address->setSchema('https'); 45 | 46 | return $address; 47 | }); 48 | 49 | $addresses = array_filter($addresses); 50 | 51 | if (!$addresses) { 52 | $this->logger->critical('Error in data collection from free-proxy-list.net, cannot get IP or port or verification time or proxy type'); 53 | } 54 | 55 | return array_filter($addresses); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Providers/Proxy/GatherProxyProvider.php: -------------------------------------------------------------------------------- 1 | client->request('GET', 'http://www.gatherproxy.com/'); 18 | $content = $response->html(); 19 | 20 | // collect 21 | preg_match_all('/gp\.insertPrx\((.*)\)\;/i', $content, $matches); 22 | $addresses = array_map(function ($row) { return json_decode($row, true); }, $matches[1]); 23 | 24 | // build internal objects 25 | $addresses = array_map(function ($data) use ($provider) { 26 | 27 | if (!$provider->replacePort($data['PROXY_PORT'])) { 28 | $provider->logger->info('[GatherProxy] Unrecognized/unmapped port'); 29 | return null; 30 | } 31 | 32 | if ($data['PROXY_TYPE'] !== 'Elite') { 33 | return null; 34 | } 35 | 36 | $proxyAddress = new ProxyServerAddress(); 37 | $proxyAddress->setAddress($data['PROXY_IP']); 38 | $proxyAddress->setPort($this->replacePort($data['PROXY_PORT'])); 39 | $proxyAddress->setSchema('http'); 40 | 41 | return $proxyAddress; 42 | 43 | }, $addresses); 44 | 45 | return array_filter($addresses); 46 | } 47 | 48 | private function replacePort(string $port) 49 | { 50 | $mapping = [ 51 | '1F90' => 8080, 52 | 'C38' => 3128, 53 | '50' => 80, 54 | '22B8' => 8888, 55 | 'C3A' => 3130, 56 | '1F91' => 8081, 57 | '1FB6' => 8118, 58 | '115C' => 4444, 59 | ]; 60 | 61 | return $mapping[$port] ?? ''; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Providers/Proxy/HideMyNameProvider.php: -------------------------------------------------------------------------------- 1 | client->request('GET', 'https://hidemy.name/en/proxy-list/?maxtime=1000&type=s&anon=4#list'); 16 | $rows = $crawler->filterXPath('//*[@id="content-section"]/section[1]/div/table/tbody/tr'); 17 | 18 | $addresses = $rows->each(function (Crawler $node) { 19 | $collection = $node->filter('td'); 20 | 21 | $proxyIP = @$collection->getNode(0)->textContent; 22 | $proxyPort = (int)@$collection->getNode(1)->textContent; 23 | $lastVerificationTime = @$collection->getNode(6)->textContent; 24 | 25 | if (!$proxyIP || !$proxyPort || strlen($lastVerificationTime) === 0) { 26 | $this->logger->critical('Error in data collection from hidemy.name, cannot get IP or port or verification time or proxy type'); 27 | return null; 28 | } 29 | 30 | if (!$this->isEnoughFresh($lastVerificationTime)) { 31 | $this->logger->notice('[hidemy.name] The address is not enough fresh'); 32 | return null; 33 | } 34 | 35 | $address = new ProxyServerAddress(); 36 | $address->setAddress($proxyIP); 37 | $address->setPort($proxyPort); 38 | $address->setSchema('https'); 39 | 40 | return $address; 41 | }); 42 | 43 | return array_filter($addresses); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Providers/Proxy/ProxyListOrgProvider.php: -------------------------------------------------------------------------------- 1 | client->request('GET', 'https://proxy-list.org/english/search.php?search=elite.ssl-yes&country=any&type=elite&port=any&ssl=yes'); 18 | $content = $response->html(); 19 | 20 | // collect 21 | preg_match_all("/Proxy\('(.*)'\)/i", $content, $matches); 22 | $addresses = array_map(function ($row) { return base64_decode($row); }, $matches[1]); 23 | 24 | // build internal objects 25 | $addresses = array_map(function ($data) use ($provider) { 26 | 27 | $parts = explode(':', $data); 28 | 29 | if ((int)$parts[1] === 0) { 30 | return null; 31 | } 32 | 33 | $proxyAddress = new ProxyServerAddress(); 34 | $proxyAddress->setAddress($parts[0]); 35 | $proxyAddress->setPort((int)$parts[1]); 36 | $proxyAddress->setSchema('https'); 37 | 38 | return $proxyAddress; 39 | 40 | }, $addresses); 41 | 42 | return array_filter($addresses); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Providers/Proxy/ProxyProviderInterface.php: -------------------------------------------------------------------------------- 1 | torProxies = array_filter($torProxies); 32 | $this->serversNum = $serversNum; 33 | } 34 | 35 | /** 36 | * @inheritdoc 37 | */ 38 | public function collectAddresses(): array 39 | { 40 | if ($this->serversNum < 1 || !$this->getRandomServer()) { 41 | return []; 42 | } 43 | 44 | $proxies = []; 45 | 46 | foreach (range(1, $this->serversNum) as $num) { 47 | $virtualAddress = $this->getRandomServer(); 48 | $split = explode('@', $virtualAddress); 49 | 50 | // example TOR proxy string: http://tor_proxy:8118@9051@some_passphrase 51 | $serverAddress = $split[0]; 52 | $torManagementPort = (int) ($split[1] ?? 9051); 53 | $passphrase = $split[2] ?? ''; 54 | 55 | $this->logger->info( 56 | 'Registering TOR server "' . $serverAddress . '" with management port at ' . $torManagementPort 57 | ); 58 | 59 | $address = new TorProxyServerAddress($torManagementPort, $passphrase); 60 | $address->setSchema('http') 61 | ->setPort(parse_url($serverAddress, PHP_URL_PORT) ?: 80) 62 | ->setAddress(parse_url($serverAddress, PHP_URL_HOST)); 63 | 64 | $proxies[] = $address; 65 | } 66 | 67 | return $proxies; 68 | } 69 | 70 | private function getRandomServer(): string 71 | { 72 | if (!$this->torProxies) { 73 | return ''; 74 | } 75 | 76 | return $this->torProxies[array_rand($this->torProxies)]; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/Providers/Proxy/UsProxyOrgProvider.php: -------------------------------------------------------------------------------- 1 | client->request('GET', 'https://www.us-proxy.org/', [], [], [ 19 | 'HTTP_USER_AGENT' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/51.0.2704.79 Chrome/51.0.2704.79 Safari/537.36' 20 | ]); 21 | $rowsInTheTable = $crawler->filterXPath('//table[@id="proxylisttable"]/tbody[1]/tr'); 22 | 23 | if (!$rowsInTheTable->count()) { 24 | throw new \Exception('The crawler for us-proxy.org seems to not work anymore'); 25 | } 26 | 27 | foreach ($rowsInTheTable as $element) { 28 | try { 29 | $data = [ 30 | 'ip' => trim($element->ownerDocument->saveHTML($element->childNodes[0])), 31 | 'port' => trim($element->ownerDocument->saveHTML($element->childNodes[1])), 32 | 'https' => trim($element->ownerDocument->saveHTML($element->childNodes[6])), 33 | 'last_checked' => trim($element->ownerDocument->saveHTML($element->childNodes[7])) 34 | ]; 35 | 36 | $data = array_map(function (string $str) { return strip_tags($str); }, $data); 37 | 38 | } catch (\Throwable $exception) { 39 | $this->logger->debug($exception); 40 | continue; 41 | } 42 | 43 | if ($data['https'] !== 'yes' || !$this->isEnoughFresh($data['last_checked'])) { 44 | continue; 45 | } 46 | 47 | $address = new ProxyServerAddress(); 48 | $address->setAddress($data['ip']) 49 | ->setPort((int) $data['port']) 50 | ->setSchema('https'); 51 | 52 | $addresses[] = $address; 53 | } 54 | 55 | return $addresses; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Service/Config.php: -------------------------------------------------------------------------------- 1 | values = $values; 21 | } 22 | 23 | /** 24 | * @param string $keyName 25 | * @return array|string|int|float 26 | */ 27 | public function get(string $keyName) 28 | { 29 | if (!array_key_exists($keyName, $this->values)) { 30 | throw new \InvalidArgumentException($keyName . ' was not defined in the config.php'); 31 | } 32 | 33 | return $this->values[$keyName]; 34 | } 35 | 36 | /** 37 | * @param string $keyName 38 | * @param string|int|float|array $default 39 | * 40 | * @return array|float|int|string 41 | */ 42 | public function getOptional(string $keyName, $default) 43 | { 44 | try { 45 | return $this->get($keyName); 46 | } catch (\InvalidArgumentException $exception) { 47 | return $default; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Service/ContentProcessor/ContentProcessor.php: -------------------------------------------------------------------------------- 1 | processors = $processors; 20 | } 21 | 22 | /** 23 | * @inheritdoc 24 | */ 25 | public function process(ForwardableRequest $request, string $input, string $mimeType = null): string 26 | { 27 | if (!$mimeType) { 28 | throw new \InvalidArgumentException('$mimeType must be provided to the ContentProcessor'); 29 | } 30 | 31 | foreach ($this->processors as $processor) { 32 | if ($processor->canProcess($mimeType)) { 33 | return $processor->process($request, $input); 34 | } 35 | } 36 | 37 | return ''; 38 | } 39 | 40 | /** 41 | * @inheritdoc 42 | */ 43 | public function canProcess(string $mimeType): bool 44 | { 45 | $processorsThatCanHandleTheProcessing = array_filter( 46 | $this->processors, 47 | function (ProcessorInterface $processor) use ($mimeType) { 48 | return $processor->canProcess($mimeType); 49 | } 50 | ); 51 | 52 | return count($processorsThatCanHandleTheProcessing) > 0; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Service/ContentProcessor/CssProcessor.php: -------------------------------------------------------------------------------- 1 | urlGenerator = $urlGenerator; 25 | } 26 | 27 | /** 28 | * @inheritdoc 29 | */ 30 | public function process(ForwardableRequest $request, string $input, string $mimeType = null): string 31 | { 32 | $output = $input; 33 | $matches = explode('url(', $input); 34 | array_shift($matches); 35 | 36 | foreach ($matches as $match) { 37 | $splitToEnding = explode(')', $match); 38 | $rawUrl = $splitToEnding[0] ?? ''; 39 | $cleanUrl = trim($rawUrl, '" \''); 40 | 41 | // do not support inline links 42 | if (strpos($cleanUrl, 'data:') === 0) { 43 | continue; 44 | } 45 | 46 | $output = str_replace($cleanUrl, $this->urlGenerator->generateUrl($request, $cleanUrl), $output); 47 | } 48 | 49 | return $output; 50 | } 51 | 52 | /** 53 | * @inheritdoc 54 | */ 55 | public function canProcess(string $mimeType): bool 56 | { 57 | return strtolower($mimeType) === 'text/css'; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Service/ContentProcessor/HtmlProcessor.php: -------------------------------------------------------------------------------- 1 | ['src'], 16 | 'a' => ['href'], 17 | 'link' => ['href'], 18 | 'base' => ['href'], 19 | 'script' => ['src'], 20 | 'form' => ['action'] 21 | ]; 22 | 23 | /** 24 | * @var OneTimeTokenUrlGenerator $urlGenerator 25 | */ 26 | private $urlGenerator; 27 | 28 | /** 29 | * @var CssProcessor $cssProcessor 30 | */ 31 | private $cssProcessor; 32 | 33 | /** 34 | * @param OneTimeTokenUrlGenerator $urlGenerator 35 | */ 36 | public function __construct(OneTimeTokenUrlGenerator $urlGenerator, CssProcessor $cssProcessor) 37 | { 38 | $this->urlGenerator = $urlGenerator; 39 | $this->cssProcessor = $cssProcessor; 40 | } 41 | 42 | /** 43 | * @inheritdoc 44 | */ 45 | public function process(ForwardableRequest $request, string $input, string $mimeType = null): string 46 | { 47 | $dom = new \DOMDocument(); 48 | @$dom->loadHTML($input); 49 | 50 | foreach (self::ELEMENTS_MAPPING as $tagName => $attributeNames) { 51 | foreach ($attributeNames as $attributeName) { 52 | /** 53 | * @var \DOMElement[] $tags 54 | */ 55 | $tags = $dom->getElementsByTagName($tagName); 56 | 57 | if (!$tags) { 58 | continue; 59 | } 60 | 61 | if ($tagName === 'img') { 62 | $this->processScalableImages($request, $tags); 63 | } 64 | 65 | foreach ($tags as $tag) { 66 | if ($tag->hasAttribute($attributeName)) { 67 | $tag->setAttribute( 68 | $attributeName, 69 | $this->rewriteRawUrlToProxiedUrl($request, $tag->getAttribute($attributeName)) 70 | ); 71 | } 72 | } 73 | } 74 | } 75 | 76 | $this->handleStyleTags($request, $dom); 77 | 78 | return $dom->saveHTML(); 79 | } 80 | 81 | private function handleStyleTags(ForwardableRequest $request, \DOMDocument $dom) 82 | { 83 | /** 84 | * @var \DOMElement[] $styleTags 85 | */ 86 | $styleTags = $dom->getElementsByTagName('style'); 87 | 88 | foreach ($styleTags as $styleTag) { 89 | $html = $this->_domNodeToHTML($styleTag); 90 | $processedHtml = $this->cssProcessor->process($request, $html, 'text/css'); 91 | $this->_domReplaceHTMLContent($styleTag, $processedHtml); 92 | } 93 | } 94 | 95 | private function _domNodeToHTML(\DOMElement $element) 96 | { 97 | return array_reduce( 98 | iterator_to_array($element->childNodes), 99 | function ($carry, \DOMNode $child) { 100 | return $carry . $child->ownerDocument->saveHTML($child); 101 | } 102 | ); 103 | } 104 | 105 | private function _domReplaceHTMLContent(\DOMElement $element, string $newContent) 106 | { 107 | $fragment = $element->ownerDocument->createDocumentFragment(); 108 | $fragment->appendXML($newContent); 109 | 110 | while ($element->hasChildNodes()) { 111 | $element->removeChild($element->firstChild); 112 | } 113 | 114 | $element->appendChild($fragment); 115 | } 116 | 117 | /** 118 | * @url https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images 119 | * 120 | * @param ForwardableRequest $request 121 | * @param \DOMElement[]|\DOMNodeList $domElements 122 | */ 123 | private function processScalableImages(ForwardableRequest $request, \DOMNodeList $domElements) 124 | { 125 | foreach ($domElements as $image) { 126 | if ($image->hasAttribute('srcset')) { 127 | $scalableImages = explode(',', $image->getAttribute('srcset')); 128 | 129 | foreach ($scalableImages as $key => $scalableImage) { 130 | $scalableImages[$key] = $this->replaceUrlInScalableImageElement($request, $scalableImage); 131 | } 132 | 133 | $image->setAttribute('srcset', implode(',', $scalableImages)); 134 | } 135 | } 136 | } 137 | 138 | /** 139 | * @param ForwardableRequest $request 140 | * @param string $element 141 | * 142 | * @return string 143 | */ 144 | private function replaceUrlInScalableImageElement(ForwardableRequest $request, string $element) 145 | { 146 | [$url, $size] = explode(' ', trim($element)); 147 | return $this->rewriteRawUrlToProxiedUrl($request, $url) . ' ' . $size; 148 | } 149 | 150 | /** 151 | * @param ForwardableRequest $request 152 | * @param string $url 153 | * 154 | * @return string 155 | */ 156 | private function rewriteRawUrlToProxiedUrl(ForwardableRequest $request, string $url) 157 | { 158 | return $this->urlGenerator->generateUrl($request, $url); 159 | } 160 | 161 | /** 162 | * @inheritdoc 163 | */ 164 | public function canProcess(string $mimeType): bool 165 | { 166 | return $mimeType === 'text/html'; 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/Service/ContentProcessor/ProcessorInterface.php: -------------------------------------------------------------------------------- 1 | load( 35 | explode(',', $fixturesNames), 36 | json_decode($mapping, true) ?? [] 37 | ); 38 | } 39 | 40 | /** 41 | * Load all fixtures, internal and external 42 | * 43 | * @param array $names 44 | * @param array $mapping 45 | * 46 | * @throws InvalidConfigurationException 47 | */ 48 | protected function load(array $names, array $mapping) 49 | { 50 | $names = array_filter($names); 51 | 52 | foreach ($names as $name) { 53 | $className = $this->getFullClassName($name, $mapping); 54 | $this->fixtures[] = new $className(); 55 | } 56 | } 57 | 58 | /** 59 | * Get class name for a fixture 60 | * looking at first in internal fixtures, then into mapped fixtures 61 | * from libraries/external files accessible via composer's autoloader 62 | * 63 | * @param string $name 64 | * @param array $mapping 65 | * 66 | * @throws InvalidConfigurationException 67 | * @return string 68 | */ 69 | protected function getFullClassName(string $name, array $mapping) 70 | { 71 | $className = 'Wolnosciowiec\\WebProxy\\Fixtures\\' . $name; 72 | 73 | if (class_exists($className)) { 74 | return $className; 75 | } 76 | 77 | if (!isset($mapping[$name])) { 78 | throw new InvalidConfigurationException( 79 | '"' . $name . '" fixture not found in standard namespace, 80 | you probably should define it in mapping. Use "fixtures_mapping" configuration variable 81 | or WW_FIXTURES_MAPPING environment variable.' 82 | ); 83 | } 84 | 85 | return $mapping[$name]; 86 | } 87 | 88 | /** 89 | * @return FixtureInterface[] 90 | */ 91 | public function getFixtures() 92 | { 93 | return $this->fixtures; 94 | } 95 | 96 | /** 97 | * Apply all enabled fixtures 98 | * 99 | * @param ResponseInterface $response 100 | * @return ResponseInterface 101 | */ 102 | public function fix(RequestInterface $request, ResponseInterface $response): ResponseInterface 103 | { 104 | foreach ($this->fixtures as $fixture) { 105 | $response = $fixture->fixResponse($request, $response); 106 | } 107 | 108 | return $response; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/Service/Prerenderer.php: -------------------------------------------------------------------------------- 1 | client = $client; 27 | $this->serviceUrl = $prerenderUrl; 28 | } 29 | 30 | /** 31 | * Use Wolnościowiec Prerenderer service to fetch page contents 32 | * 33 | * @param string $url 34 | * @param string $proxyUrl 35 | * 36 | * @return string 37 | */ 38 | public function render(string $url, string $proxyUrl = null): string 39 | { 40 | try { 41 | return $this->client->get($this->serviceUrl, [ 42 | 'headers' => array_filter([ 43 | 'X-Render-Url' => $url, 44 | 'X-Proxy-Address' => $proxyUrl 45 | ]) 46 | ])->getBody()->getContents(); 47 | 48 | } catch (RequestException $exception) { 49 | 50 | if (!$exception->getResponse()) { 51 | return 'Render error: ' . $exception->getMessage(); 52 | } 53 | 54 | return 'Render error: ' . $exception->getResponse()->getBody()->getContents(); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Service/Proxy.php: -------------------------------------------------------------------------------- 1 | request)) 17 | { 18 | throw new UnexpectedValueException('Missing request instance.'); 19 | } 20 | 21 | $target = new Uri($target); 22 | 23 | // Overwrite target scheme and host. 24 | $uri = $this->request->getUri() 25 | ->withScheme($target->getScheme()) 26 | ->withHost($target->getHost()); 27 | 28 | // Check for custom port. 29 | if ($port = $target->getPort()) { 30 | $uri = $uri->withPort($port); 31 | } 32 | 33 | // Check for subdirectory. 34 | if ($path = $target->getPath()) { 35 | // this line was fixed - it causes an issue in Wolnościowiec WebProxy, all links are getting "/" at the end 36 | // which makes the proxy useless 37 | $uri = $uri->withPath(rtrim($path, '/')); 38 | } 39 | 40 | $request = $this->request->withUri($uri); 41 | 42 | $stack = $this->filters; 43 | 44 | $stack[] = function (RequestInterface $request, ResponseInterface $response, callable $next) 45 | { 46 | return $next($request, $this->adapter->send($request)); 47 | }; 48 | 49 | $relay = (new RelayBuilder())->newInstance($stack); 50 | 51 | return $relay($request, new Response()); 52 | } 53 | } -------------------------------------------------------------------------------- /src/Service/Proxy/ProxySelector.php: -------------------------------------------------------------------------------- 1 | addresses = $provider->collectAddresses(); 21 | } 22 | 23 | public function getHTTPProxy(): ?ProxyServerAddress 24 | { 25 | if (!$this->addresses) { 26 | return null; 27 | } 28 | 29 | shuffle($this->addresses); 30 | return $this->addresses[0]; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Service/ProxyCacheBuilder.php: -------------------------------------------------------------------------------- 1 | factory = $factory; 54 | $this->logger = $logger; 55 | $this->provider = $provider; 56 | $this->client = $client; 57 | $this->connectionTimeout = $connectionTimeout; 58 | } 59 | 60 | /** 61 | * Fetches a new list 62 | * 63 | * @return ProxyServerAddress[] 64 | * 65 | * @throws \Exception 66 | */ 67 | public function rebuildListCache(): array 68 | { 69 | $providers = $this->factory->buildProviders(); 70 | $addresses = []; 71 | 72 | foreach ($providers as $provider) { 73 | $this->logger->info('Processing ' . get_class($provider)); 74 | $addresses = array_merge($addresses, $provider->collectAddresses()); 75 | } 76 | 77 | $this->logger->info('Saving ' . count($addresses) . ' addresses to cache'); 78 | $this->provider->cacheResult($addresses); 79 | 80 | return $addresses; 81 | } 82 | 83 | /** 84 | * @param ProxyServerAddress[] $addresses 85 | */ 86 | public function spawnVerificationProcesses(array $addresses) 87 | { 88 | foreach ($addresses as $address) { 89 | $command = '/bin/bash -c "' . __DIR__ . '/../../bin/verify-proxy-address ' . $address->getFormatted() . '" &'; 90 | 91 | $this->logger->info('Spawning "' . $command . '"'); 92 | passthru($command); 93 | sleep(1); 94 | } 95 | } 96 | 97 | /** 98 | * Connects to a proxy to check if the proxy is valid 99 | * 100 | * @param string $address 101 | * 102 | * @return bool 103 | */ 104 | public function performProxyVerification(string $address): bool 105 | { 106 | if (!$address) { 107 | $this->logger->info('No address passed'); 108 | return false; 109 | } 110 | 111 | $sitesToTest = [ 112 | 'https://duckduckgo.com/', 113 | 'https://github.com/', 114 | 'http://iwa-ait.org/' 115 | ]; 116 | 117 | try { 118 | $response = $this->client->request('GET', $sitesToTest[array_rand($sitesToTest)], [ 119 | 'proxy' => $address, 120 | 'connect_timeout' => $this->connectionTimeout, 121 | 'read_timeout' => $this->connectionTimeout, 122 | 'timeout' => $this->connectionTimeout 123 | ]); 124 | 125 | if ($response->getBody()->getContents() === '') { 126 | throw new HttpException('Invalid proxy response, that proxy seems not to be working'); 127 | } 128 | 129 | } catch (ConnectException | RequestException | ClientException | HttpException $exception) { 130 | $this->logger->info('Exception: ' . $exception->getMessage()); 131 | $this->logger->info('The proxy "' . $address . '" is not valid anymore, removing from cache'); 132 | 133 | $this->removeFromCache($address); 134 | return false; 135 | } 136 | 137 | $this->logger->info('The proxy "' . $address . '" looks OK.'); 138 | return true; 139 | } 140 | 141 | public function logSummary() 142 | { 143 | $this->logger->info('In the summary there are "' . \count($this->provider->getFromCache()) . ' working proxies'); 144 | } 145 | 146 | private function removeFromCache(string $address) 147 | { 148 | $addresses = $this->provider->getFromCache(); 149 | $withoutSpecificAddress = array_filter( 150 | $addresses, 151 | function (ProxyServerAddress $cached) use ($address) { 152 | return $cached->getFormatted() !== $address; 153 | } 154 | ); 155 | 156 | $this->provider->cacheResult($withoutSpecificAddress); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/Service/Security/AuthCheckerInterface.php: -------------------------------------------------------------------------------- 1 | encryptionKey = $config->get('encryptionKey'); 27 | } 28 | 29 | private function unescape(string $queryParameter) 30 | { 31 | return str_replace(' ', '+', $queryParameter); 32 | } 33 | 34 | /** 35 | * Checks only if encrypted one time token is valid 36 | * 37 | * @param ForwardableRequest $request 38 | * @return bool 39 | */ 40 | public function isValid(ForwardableRequest $request): bool 41 | { 42 | try { 43 | $decrypted = CryptoJSAES::decrypt( 44 | $this->unescape($request->getQueryParams()[InputParams::QUERY_ONE_TIME_TOKEN] ?? ''), 45 | $this->encryptionKey 46 | ); 47 | 48 | $array = \GuzzleHttp\json_decode($decrypted, true); 49 | 50 | if (!isset($array[InputParams::ONE_TIME_TOKEN_PROPERTY_URL])) { 51 | return false; 52 | } 53 | 54 | // token can have expiration time 55 | if (isset($array[InputParams::ONE_TIME_TOKEN_PROPERTY_EXPIRES])) { 56 | $expiration = new \DateTime($array[InputParams::ONE_TIME_TOKEN_PROPERTY_EXPIRES]); 57 | 58 | if ($expiration <= new \DateTime()) { 59 | return false; 60 | } 61 | } 62 | 63 | } catch (\Exception $exception) { 64 | return false; 65 | } 66 | 67 | return true; 68 | } 69 | 70 | /** 71 | * @inheritdoc 72 | */ 73 | public function canHandle(ForwardableRequest $request): bool 74 | { 75 | return isset($request->getQueryParams()[InputParams::QUERY_ONE_TIME_TOKEN]); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Service/Security/OneTimeTokenUrlGenerator.php: -------------------------------------------------------------------------------- 1 | encryptionKey = $config->get('encryptionKey'); 25 | $this->expirationTime = $config->getOptional('oneTimeTokenStaticFilesLifeTime', '+1 minute'); 26 | } 27 | 28 | /** 29 | * @param ForwardableRequest $request 30 | * @param string $relativeOrAbsoluteUrl 31 | * 32 | * @return string 33 | */ 34 | public function generateUrl(ForwardableRequest $request, string $relativeOrAbsoluteUrl) 35 | { 36 | $absoluteUrl = $this->makeAbsoluteUrl($request, $relativeOrAbsoluteUrl); 37 | 38 | $oneTimeToken = $this->encrypt([ 39 | InputParams::ONE_TIME_TOKEN_PROPERTY_EXPIRES => (new \DateTime())->modify($this->expirationTime)->format('Y-m-d H:i:s'), 40 | InputParams::ONE_TIME_TOKEN_PROPERTY_URL => $absoluteUrl, 41 | InputParams::ONE_TIME_TOKEN_PROCESS => true, 42 | ]); 43 | 44 | return '?' . InputParams::QUERY_ONE_TIME_TOKEN . '=' . $oneTimeToken; 45 | } 46 | 47 | /** 48 | * @param array $data 49 | * @return string 50 | */ 51 | private function encrypt(array $data): string 52 | { 53 | return CryptoJSAES::encrypt(json_encode($data), $this->encryptionKey); 54 | } 55 | 56 | /** 57 | * @param ForwardableRequest $request 58 | * @param string $relativeOrAbsoluteUrl 59 | * 60 | * @return string 61 | */ 62 | private function makeAbsoluteUrl(ForwardableRequest $request, string $relativeOrAbsoluteUrl) 63 | { 64 | $absoluteUrlBeginsWith = [ 65 | 'http://', 'https://', '://', 66 | ]; 67 | 68 | foreach ($absoluteUrlBeginsWith as $prefix) { 69 | if (strpos($relativeOrAbsoluteUrl, $prefix) === 0) { 70 | return $relativeOrAbsoluteUrl; 71 | } 72 | } 73 | 74 | $parsed = parse_url($request->getDestinationUrl()); 75 | $rootUrl = $parsed['scheme'] . '://' . $parsed['host']; 76 | 77 | if (isset($parsed['port'])) { 78 | $rootUrl .= ':' . $parsed['port']; 79 | } 80 | 81 | return $rootUrl . '/' . $relativeOrAbsoluteUrl; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/Service/Security/TokenAuthChecker.php: -------------------------------------------------------------------------------- 1 | get('apiKey'); 26 | 27 | if (!is_array($keys)) { 28 | $keys = [$keys]; 29 | } 30 | 31 | $this->apiKeys = $keys; 32 | } 33 | 34 | /** 35 | * @inheritdoc 36 | */ 37 | public function isValid(ForwardableRequest $request): bool 38 | { 39 | return in_array( 40 | $request->getToken(), 41 | $this->apiKeys, 42 | true 43 | ); 44 | } 45 | 46 | /** 47 | * @inheritdoc 48 | */ 49 | public function canHandle(ForwardableRequest $request): bool 50 | { 51 | return strlen($request->getToken()) > 0; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/bootstrap.php: -------------------------------------------------------------------------------- 1 | useAnnotations(false); 16 | $builder->useAutowiring(true); 17 | $builder->addDefinitions(new DefinitionFile(__DIR__ . '/../src/DependencyInjection/Services.php')); 18 | $builder->addDefinitions(new DefinitionFile(__DIR__ . '/../src/DependencyInjection/LibraryServices.php')); 19 | $container = $builder->build(); 20 | 21 | // for PhpUnit 22 | $GLOBALS['container'] = $container; 23 | 24 | return $container; 25 | 26 | // @codeCoverageIgnoreEnd 27 | -------------------------------------------------------------------------------- /tests/Factory/ProxyClientFactoryTest.php: -------------------------------------------------------------------------------- 1 | create(); 25 | 26 | $this->assertInstanceOf(Proxy::class, $guzzleProxy); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tests/Factory/ProxyProviderFactoryTest.php: -------------------------------------------------------------------------------- 1 | getContainer() 25 | ); 26 | } 27 | 28 | /** 29 | * @see ProxyProviderFactory::create() 30 | */ 31 | public function testCreate() 32 | { 33 | $provider = $this->getFactory()->create(); 34 | 35 | // get chain provider from the inside 36 | $ref = new \ReflectionObject($provider); 37 | $property = $ref->getProperty('provider'); 38 | $property->setAccessible(true); 39 | $chainProvider = $property->getValue($provider); 40 | 41 | // get a list of providers from the chain provider 42 | $ref = new \ReflectionObject($chainProvider); 43 | $property = $ref->getProperty('providers'); 44 | $property->setAccessible(true); 45 | $providers = $property->getValue($chainProvider); 46 | 47 | $this->assertInstanceOf(ProxyProviderInterface::class, $provider); 48 | $this->assertInstanceOf(ProxyProviderInterface::class, $chainProvider); 49 | 50 | foreach ($providers as $provider) { 51 | $this->assertInstanceOf(ProxyProviderInterface::class, $provider); 52 | } 53 | } 54 | 55 | /** 56 | * @expectedException \Exception 57 | * @expectedExceptionMessage Invalid provider name "NonExistingProvider", please check the configuration 58 | */ 59 | public function testValidationCreate() 60 | { 61 | $this->getFactory('NonExistingProvider')->create(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tests/Factory/RequestFactoryTest.php: -------------------------------------------------------------------------------- 1 | create('https://wolnosciowiec.net'); 20 | 21 | $this->assertSame('wolnosciowiec.net', $request->getUri()->getHost()); 22 | $this->assertNotContains('ww-target-url', $request->getHeaders()); 23 | $this->assertNotContains('ww-token', $request->getHeaders()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/Fixtures/FacebookCaptchaTo500Test.php: -------------------------------------------------------------------------------- 1 | '); 22 | 23 | $newResponse = (new FacebookCaptchaTo500())->fixResponse($request, $response); 24 | $this->assertSame(500, $newResponse->getStatusCode()); 25 | } 26 | 27 | /** 28 | * @see FacebookCaptchaTo500::fix() 29 | */ 30 | public function testNotMatchingDomain() 31 | { 32 | $request = new Request('GET', 'https://not-a-facebook-domain.org/'); 33 | $response = new Response(200, [], '
'); 34 | 35 | $newResponse = (new FacebookCaptchaTo500())->fixResponse($request, $response); 36 | $this->assertSame(200, $newResponse->getStatusCode()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/Fixtures/NotFoundTo500Test.php: -------------------------------------------------------------------------------- 1 | fixResponse($request, $response); 24 | $this->assertSame(500, $newResponse->getStatusCode()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/Middleware/OneTimeTokenParametersConversionMiddlewareTest.php: -------------------------------------------------------------------------------- 1 | true, 26 | InputParams::ONE_TIME_TOKEN_PROPERTY_URL => 'http://iwa-ait.org' 27 | ]), 28 | 29 | 'kick-off-bosses-power-to-the-grassroots-workers' 30 | ); 31 | 32 | $middleware = new OneTimeTokenParametersConversionMiddleware('kick-off-bosses-power-to-the-grassroots-workers'); 33 | $middleware( 34 | new ForwardableRequest( 35 | [], [], 'http://localhost', 36 | 'GET', 'php://input', [], [], [ 37 | InputParams::QUERY_ONE_TIME_TOKEN => $token 38 | ]), 39 | new Response(), 40 | function (ForwardableRequest $request, $response) { 41 | $this->assertSame('http://iwa-ait.org', $request->getDestinationUrl()); 42 | $this->assertTrue($request->canOutputBeProcessed()); 43 | } 44 | ); 45 | } 46 | 47 | /** 48 | * @see OneTimeTokenParametersConversionMiddleware 49 | */ 50 | public function test_invalid_token_not_decoded() 51 | { 52 | $token = CryptoJSAES::encrypt( 53 | json_encode([ 54 | InputParams::ONE_TIME_TOKEN_PROCESS => true, 55 | InputParams::ONE_TIME_TOKEN_PROPERTY_URL => 'http://zsp.net.pl' 56 | ]), 57 | 58 | 'this-passphrase-is-different-than-the-server-uses' 59 | ); 60 | 61 | $middleware = new OneTimeTokenParametersConversionMiddleware('long-live-anarchosyndicalism'); 62 | $middleware( 63 | new ForwardableRequest( 64 | [], [], 'http://localhost', 65 | 'GET', 'php://input', [], [], [ 66 | InputParams::QUERY_ONE_TIME_TOKEN => $token 67 | ]), 68 | new Response(), 69 | function (ForwardableRequest $request, ResponseInterface $response) { 70 | $this->assertNull($request->getDestinationUrl()); 71 | $this->assertFalse($request->canOutputBeProcessed()); 72 | $this->assertSame(403, $response->getStatusCode()); 73 | } 74 | ); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/CachedProviderTest.php: -------------------------------------------------------------------------------- 1 | setMode(DummyProvider::RETURN_NONE), 21 | ]); 22 | 23 | return new CachedProvider(new ArrayCache(), $chain); 24 | } 25 | 26 | /** 27 | * @see CachedProvider::collectAddresses() 28 | */ 29 | public function testCollectAddressesTwice() 30 | { 31 | parent::testCollectAddresses(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/ChainProviderTest.php: -------------------------------------------------------------------------------- 1 | setMode(DummyProvider::RETURN_NONE), 20 | ]); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/FreeProxyCzProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer()->get(FreeProxyCzProvider::class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/FreeProxyListProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer()->get(FreeProxyListProvider::class); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/GatherProxyProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer()->get(GatherProxyProvider::class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/ProxyListOrgProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer()->get(ProxyListOrgProvider::class); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/Providers/Proxy/TestProxyProviderInterfaceImplementation.php: -------------------------------------------------------------------------------- 1 | getProvider(); 29 | $addresses = $provider->collectAddresses(); 30 | 31 | foreach ($addresses as $address) { 32 | $this->assertInstanceOf(ProxyServerAddress::class, $address); 33 | $this->assertRegExp('/(http|https)\:\/\/(.*)\:([0-9]+)/i', $address->getFormatted()); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /tests/Providers/Proxy/UsProxyOrgProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer()->get(UsProxyOrgProvider::class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/Service/Controllers/PassThroughControllerTest.php: -------------------------------------------------------------------------------- 1 | 'your-api-key-here', 23 | InputParams::QUERY_TARGET_URL => 'http://wolnywroclaw.pl', 24 | ]); 25 | 26 | $controller = $this->getContainer()->get(PassThroughController::class); 27 | $response = (string)$controller->executeAction($request)->getBody(); 28 | 29 | $this->assertContains('Federacja Anarchistyczna', $response); 30 | } 31 | 32 | /** 33 | * Test HTTP 404 response 34 | */ 35 | public function testInvalidUrl() 36 | { 37 | $request = new ForwardableRequest($_SERVER, [], null, null, 'php://input', [], [], [ 38 | InputParams::QUERY_TOKEN => 'your-api-key-here', 39 | InputParams::QUERY_TARGET_URL => 'https://github.com/this_should_not_exist_fegreiuhwif', 40 | ]); 41 | 42 | $controller = $this->getContainer()->get(PassThroughController::class); 43 | $response = $controller->executeAction($request); 44 | 45 | $this->assertSame(404, $response->getStatusCode()); 46 | } 47 | 48 | /** 49 | * Test of catching connection errors 50 | * ---------------------------------- 51 | * Expecting: cURL error 6: Could not resolve host: this-domain-should-not-exists (see http://curl.haxx.se/libcurl/c/libcurl-errors.html) 52 | */ 53 | public function testHttpErrorUrl() 54 | { 55 | $request = new ForwardableRequest($_SERVER, [], null, null, 'php://input', [], [], [ 56 | InputParams::QUERY_TOKEN => 'your-api-key-here', 57 | InputParams::QUERY_TARGET_URL => 'http://1.2.3.4', 58 | ]); 59 | 60 | $controller = $this->getContainer()->get(PassThroughController::class); 61 | $response = $controller->executeAction($request); 62 | 63 | $this->assertSame(500, $response->getStatusCode()); 64 | } 65 | 66 | /** 67 | * Check if there are required headers in the request 68 | */ 69 | public function testRequestValidation() 70 | { 71 | $this->expectException(HttpException::class); 72 | 73 | $request = new ForwardableRequest($_SERVER, [], null, null, 'php://input', [], [], [ 74 | InputParams::QUERY_TOKEN => 'your-api-key-here', 75 | ]); 76 | 77 | $controller = $this->getContainer()->get(PassThroughController::class); 78 | $controller->executeAction($request); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /tests/Service/FixturesManagerTest.php: -------------------------------------------------------------------------------- 1 | assertInstanceOf(NotFoundTo500::class, $manager->getFixtures()[0]); 24 | } 25 | 26 | /** 27 | * Case: Using a mapping to attach fixtures from external sources 28 | * 29 | * @see FixturesManager::getFixtures() 30 | */ 31 | public function testGetFixturesWithMapping() 32 | { 33 | $manager = new FixturesManager( 34 | 'NotFoundTo500,ExampleFixture', 35 | '{"ExampleFixture": "\\\Wolnosciowiec\\\CustomFixtures\\\ExampleFixture"}' 36 | ); 37 | 38 | $this->assertInstanceOf(NotFoundTo500::class, $manager->getFixtures()[0]); 39 | $this->assertInstanceOf(ExampleFixture::class, $manager->getFixtures()[1]); 40 | } 41 | 42 | /** 43 | * @see FixturesManager::fix() 44 | */ 45 | public function testFix() 46 | { 47 | $manager = new FixturesManager( 48 | 'NotFoundTo500,ExampleFixture', 49 | '{"ExampleFixture": "\\\Wolnosciowiec\\\CustomFixtures\\\ExampleFixture"}' 50 | ); 51 | 52 | $request = new Request('GET', 'https://static.wolnosciowiec.net/test'); 53 | $response = new Response(404); 54 | 55 | $newResponse = $manager->fix($request, $response); 56 | 57 | $this->assertSame(500, $newResponse->getStatusCode()); 58 | $this->assertArrayHasKey('X-Message', $newResponse->getHeaders()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /tests/Service/Proxy/ProxySelectorTest.php: -------------------------------------------------------------------------------- 1 | getValidProvider()); 26 | 27 | $this->assertRegExp('/(http|https)\:\/\/(.*)\:([0-9]+)/i', $proxySelector->getHTTPProxy()); 28 | } 29 | 30 | /** 31 | * @see ProxySelector::getHTTPProxy() 32 | */ 33 | public function testAddressesAreRandomlyReturned() 34 | { 35 | $proxySelector = new ProxySelector($this->getValidProvider()); 36 | $addresses = []; 37 | 38 | // DummyProvider is providing at least 3 different addresses, the probability that only one of them 39 | // will be returned in 100000 iterations is near 0 40 | for ($i = 0; $i <= 100000; $i++) { 41 | $addresses[] = $proxySelector->getHTTPProxy(); 42 | } 43 | 44 | $this->assertGreaterThan(2, array_unique($addresses)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /tests/Service/Security/TokenAuthCheckerTest.php: -------------------------------------------------------------------------------- 1 | 'test-token'])); 18 | $request = new ForwardableRequest($_SERVER, [], null, null, 'php://input', [], [], [InputParams::QUERY_TOKEN => 'this is an invalid key']); 19 | 20 | $this->assertFalse($authChecker->isValid($request)); 21 | } 22 | 23 | public function testValidToken() 24 | { 25 | $authChecker = new TokenAuthChecker(new Config(['apiKey' => 'test-token'])); 26 | $request = new ForwardableRequest($_SERVER, [], null, null, 'php://input', [], [], [InputParams::QUERY_TOKEN => 'test-token']); 27 | 28 | $this->assertTrue($authChecker->isValid($request)); 29 | } 30 | } -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | newInstance([ 42 | $container->get(AuthenticationMiddleware::class), 43 | $container->get(OneTimeTokenParametersConversionMiddleware::class), 44 | $container->get(ApplicationMiddleware::class), 45 | $container->get(ProxyStaticContentMiddleware::class) 46 | ]); 47 | 48 | try { 49 | $request = $container->get(RequestFactory::class)->createFromGlobals(); 50 | $response = $dispatcher( 51 | $request, 52 | new Response() 53 | ); 54 | 55 | } catch (HttpException $httpException) { 56 | $response = new Response\JsonResponse([ 57 | 'error' => $httpException->getMessage(), 58 | 'code' => $httpException->getCode(), 59 | ], $httpException->getCode() >= 400 ? $httpException->getCode() : 500); 60 | } 61 | 62 | $emitter = new Zend\Diactoros\Response\SapiEmitter(); 63 | $emitter->emit($response); 64 | --------------------------------------------------------------------------------