├── .gitignore ├── .vscode └── settings.json ├── CNAME ├── LICENSE ├── README.md ├── _config.yaml ├── droplet.tf ├── network.tf ├── project.tf ├── provider.tf ├── user-data.tftpl └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/terraform,visualstudiocode 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=terraform,visualstudiocode 3 | 4 | ### Terraform ### 5 | # Local .terraform directories 6 | **/.terraform/* 7 | 8 | # Teraform lock file 9 | .terraform.lock.hcl 10 | 11 | # .tfstate files 12 | *.tfstate 13 | *.tfstate.* 14 | 15 | # Crash log files 16 | crash.log 17 | crash.*.log 18 | 19 | # Exclude all .tfvars files, which are likely to contain sensitive data, such as 20 | # password, private keys, and other secrets. These should not be part of version 21 | # control as they are data points which are potentially sensitive and subject 22 | # to change depending on the environment. 23 | *.tfvars 24 | *.tfvars.json 25 | 26 | # Ignore override files as they are usually used to override resources locally and so 27 | # are not checked in 28 | override.tf 29 | override.tf.json 30 | *_override.tf 31 | *_override.tf.json 32 | 33 | # Include override files you do wish to add to version control using negated pattern 34 | # !example_override.tf 35 | 36 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 37 | # example: *tfplan* 38 | 39 | # Ignore CLI configuration files 40 | .terraformrc 41 | terraform.rc 42 | 43 | ### VisualStudioCode ### 44 | .vscode/* 45 | !.vscode/settings.json 46 | !.vscode/tasks.json 47 | !.vscode/launch.json 48 | !.vscode/extensions.json 49 | !.vscode/*.code-snippets 50 | 51 | # Local History for Visual Studio Code 52 | .history/ 53 | 54 | # Built Visual Studio Code Extensions 55 | *.vsix 56 | 57 | ### VisualStudioCode Patch ### 58 | # Ignore all local history of files 59 | .history 60 | .ionide 61 | 62 | # End of https://www.toptal.com/developers/gitignore/api/terraform,visualstudiocode 63 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[terraform]": { 3 | "editor.formatOnSave": true, 4 | "editor.defaultFormatter": "hashicorp.terraform", 5 | "editor.tabSize": 2 6 | }, 7 | "[terraform-vars]": { 8 | "editor.formatOnSave": true, 9 | "editor.defaultFormatter": "hashicorp.terraform", 10 | "editor.tabSize": 2 11 | }, 12 | "[markdown]": { 13 | "editor.quickSuggestions": { 14 | "other": true, 15 | "comments": true, 16 | "strings": true 17 | }, 18 | "editor.unicodeHighlight.ambiguousCharacters": false, 19 | "editor.unicodeHighlight.invisibleCharacters": false, 20 | "diffEditor.ignoreTrimWhitespace": false, 21 | "editor.wordWrap": "on", 22 | "editor.defaultFormatter": "esbenp.prettier-vscode", 23 | "editor.formatOnSave": true 24 | }, 25 | "[jsonc]": { 26 | "editor.quickSuggestions": { 27 | "strings": true 28 | }, 29 | "editor.suggest.insertMode": "replace", 30 | "editor.tabSize": 4, 31 | "editor.formatOnSave": true, 32 | "editor.defaultFormatter": "vscode.json-language-features" 33 | }, 34 | "files.exclude": { 35 | "**/.git": true, 36 | "**/.svn": true, 37 | "**/.hg": true, 38 | "**/CVS": true, 39 | "**/.DS_Store": true, 40 | "**/Thumbs.db": true, 41 | "**/.terraform": true, 42 | "**/.terraform.lock.hcl": true, 43 | "**/terraform.tfstate.d": true, 44 | "**/.terraform.tfstate.lock.info": true, 45 | "**/terraform.tfstate": true 46 | } 47 | } -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | proxyforge.tbhaxor.com -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Gurkirat Singh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Proxy Forge 2 | 3 | Proxy Forge is a free and open-source tool that helps you to set up OSI level 4 proxies with rotating IPs on the Digital Ocean platform. It is a powerful and innovative solution that addresses one of the most pressing challenges faced by developers and businesses in today's digital landscape: IP throttling and blacklisting. Proxy Forge empowers you to conquer these obstacles by providing a seamless, reliable, and easy-to-deploy solution. 4 | 5 | In simpler terms, Proxy Forge helps you to create a pool of rotating IP addresses that you can use to access the internet. This can be useful for a variety of purposes, such as avoiding IP bans, scraping websites, and testing web applications. 6 | 7 | Proxy Forge is a powerful tool that can be used by both developers and businesses. It is easy to use and deploy, and it provides a number of features that make it a valuable tool for anyone who needs to access the internet anonymously or avoid IP bans. 8 | 9 | ## Features 10 | 11 | - **Rotating IPs:** Automatically rotates IP addresses from a set of deployed droplets to avoid detection and blacklisting. 12 | - **Terraform Scripts:** Provides Terraform scripts for straightforward setup and management. 13 | - **Scalability:** Easily scale your proxy fleet up or down to handle your specific needs. 14 | - **Customizable Configuration:** Fine-tune proxy settings to match your requirements. (check [below](#customizations)) 15 | - **IP Throttling Avoidance:** Effectively bypass IP throttling mechanisms used by websites and services. 16 | - **Detailed Logging:** Comprehensive logging for monitoring and troubleshooting. 17 | - **Open Source:** Proxy Forge is open-source, allowing for community contributions and customization. 18 | 19 | ## Motivation 20 | 21 | I had to find a solution for one of my clients where certain eCommerce websites were supposed to be scraped but only allowed IP addresses from the UK region. I purchased a VPC from a cloud provider in the UK, but I later discovered that the target website implements IP throttling, which temporarily disables it. As a result, I built this solution manually (no Terraform scripts) and without load balancers. 22 | 23 | I've been learning Terraform since then, and automated its setup for you. Now it takes less than 5 minutes to complete. 24 | 25 | 26 | ## Why DigitalOcean? 27 | 28 | There are many reasons why you might choose to use DigitalOcean over other cloud providers. Some of the key benefits include: 29 | 30 | - **Simplicity:** DigitalOcean is known for its simple and user-friendly interface. It is easy to get started with DigitalOcean, even if you are new to cloud computing. 31 | - **Affordability:** DigitalOcean offers competitive pricing for its cloud services. Additionally, it provides a substantial 2 TB of free bandwidth each month. 32 | - **Performance:** DigitalOcean uses high-performance hardware for its Droplets, so you can be sure that your applications will run smoothly. 33 | - **Reliability:** DigitalOcean has a strong track record of reliability and uptime. 34 | - **Community:** DigitalOcean has a large and active community of users and developers. This means that there is a wealth of resources available to help you get the most out of your DigitalOcean account. 35 | 36 | ## Proof of Concept 37 | 38 |
39 | 40 | [![asciicast](https://asciinema.org/a/0XtgYX87bPJgwORDU9jKDKsrB.svg)](https://asciinema.org/a/0XtgYX87bPJgwORDU9jKDKsrB) 41 | 42 |
43 | 44 | ## Up and Running 45 | 46 | ### Prerequisites 47 | 48 | Before using Proxy Forge, ensure you have the following prerequisites in place: 49 | 50 | 1. [Digital Ocean Account](https://www.digitalocean.com/?refcode=a7587e994b7e&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge): If you do not have an account, I would recommend creating it through my referral to get **200 USD credits for 60 days**. In this time you can test this solution. 51 | 52 | 2. [Terraform](https://docs.digitalocean.com/reference/terraform/getting-started/) installed on your local machine 53 | 54 | ### Steps 55 | 56 | 1. Clone the repository 57 | 58 | ```console 59 | git clone --depth=1 --branch=main https://github.com/tbhaxor/ProxyForge.git proxyforge 60 | cd proxyforge 61 | ``` 62 | 63 | 2. Create a file for terraform variables. 64 | 65 | ```console 66 | touch terraform.tfvars 67 | ``` 68 | 69 | > **Note** Skip it, if you wish to provide the variables and values from [environment config](https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_var_name). 70 | 71 | 3. Setup at least `token` and `region` in the file. See [variables](#variables) section below. 72 | 73 | 4. Initialize the provider, plan and apply the changes 74 | 75 | ```console 76 | terraform init 77 | terraform plan 78 | terraform apply 79 | ``` 80 | 81 | 5. Wait for some time for squid proxy to setup and load balancer to initialize routing. 82 | 6. (Optional) Test the deployment 83 | 84 | ```sh 85 | LOAD_BALANCER_IP=$(terraform output -raw lb-ip) 86 | SQUID_USERNAME=proxyforge # assuming you did not change squid-credentials.username in the tfvars 87 | SQUID_PASSWORD=proxyforge # assuming you did not change squid-credentials.password in the tfvars 88 | 89 | while true; do curl -x "http://$SQUID_USERNAME:$SQUID_PASSWORD@$LOAD_BALANCER_IP:80" https://ifconfig.me; echo; done 90 | ``` 91 | 92 | ### Variables 93 | 94 | | Variable Name | Default | Description | 95 | | :-----------------: | :----------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 96 | | `token` | N/A | **Required** API token with read/write permissions. [see more](https://docs.digitalocean.com/reference/api/create-personal-access-token/) | 97 | | `region` | N/A | **Required** Datacenter region to deploy all the resources. [see more](https://docs.digitalocean.com/products/platform/availability-matrix/) | 98 | | `ssh-fingerprint` | _`null`_ | SSH fingerprint id for droplets to use. If this is ommited, it will send you one-time-password on the email. Can be obtained from [security tab](https://i.imgur.com/TNTj7D8.png) of [accounts](https://cloud.digitalocean.com/account/security) page. [see also](https://docs.digitalocean.com/products/droplets/how-to/add-ssh-keys/) | 99 | | `prefix` | _pf_ | A prefix to quickly identify proxy-forge resources. | 100 | | `slave-count` | _2_ | Number of instances on which squid proxy will be installed. | 101 | | `lb-count` | _1_ | Number of master nodes to setup for load balancer, min `1` is required. | 102 | | `tag-name` | _proxy-forge-slave_ | Tag name to group slave droplets. | 103 | | `droplet-size` | _{ slave = "s-1vcpu-1gb-amd" }_ | Droplet size to use. | 104 | | `project` | _Proxy Forge_ | Name of the project to associate all the resources. | 105 | | `squid-credentials` | _{ password = "proxyforge", username = "proxyforge" }_ | Squid proxy HTTP basic authentication.credentials | 106 | 107 | ## Customizations 108 | 109 | Configuring ProxyForge to suit your specific needs is a straightforward process. All you need to do is update the settings in the `terraform.tfvars` file to align with your requirements. This file serves as a central configuration hub, allowing you to tailor ProxyForge precisely to your preferences without diving deep into complex setup procedures. 110 | 111 | > **Note** After you choose to make any changes from below, make sure to apply it on digitalocean 112 | > 113 | > ```console 114 | > terraform plan 115 | > terraform apply 116 | > ``` 117 | 118 | ### Change the Squid Credentials 119 | 120 | ```terraform 121 | squid-credentials = { 122 | username = "YOUR NEW USERNAME" 123 | password = "Y0ur53cr37P@55W0rD" 124 | } 125 | ``` 126 | 127 | ### Increase or Decrease Proxy Droplets 128 | 129 | ```terraform 130 | slave-count = 5 131 | ``` 132 | 133 | ## Future Plans 134 | 135 | - Add a user friendly dashboard for administration of squid accounts 136 | - Make a master service to destroy and spin-up new droplet in same region 137 | - Multiple regions support 138 | 139 | ## License 140 | 141 | Proxy Forge is licensed under the **MIT License**. See the [LICENSE](LICENSE) file for details. 142 | 143 | ## Contact Me 144 | 145 | Thank you for choosing Proxy Forge! I hope this tool helps you overcome IP throttling and blacklisting challenges, making your web-related tasks smoother and more efficient. If you have any questions or encounter issues, feel free to reach out to me. Happy proxying! 146 | 147 | Website: https://tbhaxor.com
148 | Email Address: tbhaxor@gmail.com
149 | LinkedIn: [@tbhaxor](https://linkedin.com/in/tbhaxor/)
150 | Twitter: [@tbhaxor](https://x.com/@tbhaxor)
151 | Reddit: [@tbhaxor](https://www.reddit.com/user/tbhaxor)
152 | Discord: @tbhaxor.com
153 | -------------------------------------------------------------------------------- /_config.yaml: -------------------------------------------------------------------------------- 1 | remote_theme: pages-themes/hacker@v0.2.0 2 | plugins: 3 | - jekyll-remote-theme 4 | 5 | title: ProxyForge 6 | description: Solving IP Throttling with Rotating Proxies 7 | 8 | author: 9 | name: tbhaxor 10 | email: tbhaxor@gmail.com 11 | 12 | minima: 13 | social_links: 14 | - platform: github 15 | user_url: "https://github.com/tbhaxor/ProxyForge" 16 | - platform: twitter 17 | user_url: "https://twitter.com/@tbhaxor" 18 | -------------------------------------------------------------------------------- /droplet.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | slave_urns = [ 3 | for instance in digitalocean_droplet.slave : 4 | instance.urn 5 | ] 6 | } 7 | 8 | resource "random_integer" "slave-suffix" { 9 | min = 10000 10 | max = 99999 11 | seed = count.index 12 | count = var.slave-count 13 | } 14 | 15 | resource "digitalocean_droplet" "slave" { 16 | name = "${var.prefix}-${var.region}-proxy-slave-${random_integer.slave-suffix[count.index].result}" 17 | tags = [var.tag-name] 18 | ssh_keys = var.ssh-fingerprint != null ? [var.ssh-fingerprint] : [] 19 | vpc_uuid = digitalocean_vpc.main.id 20 | region = var.region 21 | count = var.slave-count 22 | monitoring = true 23 | droplet_agent = true 24 | image = "debian-12-x64" 25 | size = var.droplet-size.slave 26 | user_data = templatefile("user-data.tftpl", { loadbalancer-ip = digitalocean_loadbalancer.lb.ip, squid-credentials = var.squid-credentials }) 27 | } 28 | -------------------------------------------------------------------------------- /network.tf: -------------------------------------------------------------------------------- 1 | resource "digitalocean_vpc" "main" { 2 | name = "${var.prefix}-${var.region}-vpc" 3 | region = var.region 4 | } 5 | 6 | 7 | resource "digitalocean_loadbalancer" "lb" { 8 | name = "${var.prefix}-lb" 9 | droplet_tag = var.tag-name 10 | vpc_uuid = digitalocean_vpc.main.id 11 | region = var.region 12 | size_unit = var.lb-count 13 | 14 | forwarding_rule { 15 | entry_port = 80 16 | entry_protocol = "tcp" 17 | target_port = 3128 18 | target_protocol = "tcp" 19 | } 20 | 21 | healthcheck { 22 | protocol = "tcp" 23 | port = 3128 24 | } 25 | } 26 | 27 | resource "digitalocean_firewall" "slave-lb-firewall" { 28 | name = "${var.prefix}-fw" 29 | tags = [var.tag-name] 30 | 31 | inbound_rule { 32 | protocol = "tcp" 33 | port_range = "22" 34 | source_addresses = ["0.0.0.0/0", "::/0"] 35 | } 36 | 37 | inbound_rule { 38 | protocol = "tcp" 39 | port_range = "all" 40 | source_load_balancer_uids = [digitalocean_loadbalancer.lb.id] 41 | } 42 | 43 | outbound_rule { 44 | destination_addresses = ["0.0.0.0/0", "::/0"] 45 | protocol = "tcp" 46 | port_range = "all" 47 | } 48 | 49 | outbound_rule { 50 | destination_addresses = ["0.0.0.0/0", "::/0"] 51 | protocol = "udp" 52 | port_range = "all" 53 | } 54 | 55 | outbound_rule { 56 | destination_addresses = ["0.0.0.0/0", "::/0"] 57 | protocol = "icmp" 58 | } 59 | } 60 | 61 | output "lb-ip" { 62 | description = "IP Address of the load balancer" 63 | value = digitalocean_loadbalancer.lb.ip 64 | } 65 | 66 | -------------------------------------------------------------------------------- /project.tf: -------------------------------------------------------------------------------- 1 | resource "digitalocean_project" "pf-project" { 2 | name = var.project 3 | resources = [digitalocean_loadbalancer.lb.urn] 4 | depends_on = [digitalocean_droplet.slave, digitalocean_loadbalancer.lb] 5 | } 6 | 7 | resource "digitalocean_project_resources" "pf-project-move-resources" { 8 | project = digitalocean_project.pf-project.id 9 | resources = concat([digitalocean_loadbalancer.lb.urn], local.slave_urns) 10 | depends_on = [digitalocean_droplet.slave] 11 | } 12 | -------------------------------------------------------------------------------- /provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | digitalocean = { 4 | source = "digitalocean/digitalocean" 5 | version = "~> 2.0" 6 | } 7 | 8 | random = { 9 | source = "hashicorp/random" 10 | version = "3.5.1" 11 | } 12 | } 13 | } 14 | 15 | 16 | provider "digitalocean" { 17 | token = var.token 18 | } 19 | 20 | provider "random" { 21 | 22 | } 23 | 24 | -------------------------------------------------------------------------------- /user-data.tftpl: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | SQUID_USERNAME="${squid-credentials.username}" 6 | SQUID_PASSWORD="${squid-credentials.password}" 7 | # shellcheck disable=SC2154 8 | LOAD_BALANCER_IP="${loadbalancer-ip}" 9 | 10 | apt update 11 | DEBIAN_FRONTEND=noninteractive apt install squid apache2-utils -y 12 | htpasswd -bc /etc/squid/passwords "$SQUID_USERNAME" "$SQUID_PASSWORD" 13 | 14 | cat < /etc/squid/squid.conf 15 | http_port 3128 16 | 17 | acl lb src $LOAD_BALANCER_IP 18 | acl SSL_ports port 443 19 | acl Safe_ports port 80 20 | acl Safe_ports port 21 21 | acl Safe_ports port 443 22 | http_access allow lb 23 | 24 | http_access deny CONNECT !SSL_ports 25 | 26 | auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwords 27 | auth_param basic realm Squid proxy-caching web server 28 | acl authenticated_users proxy_auth REQUIRED 29 | http_access allow Safe_ports authenticated_users 30 | 31 | http_access deny all 32 | EOF_SQUID 33 | 34 | squid -k parse 35 | service squid restart -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "token" { 2 | description = "API token with read/write permissions" 3 | type = string 4 | sensitive = true 5 | } 6 | 7 | variable "region" { 8 | description = "Datacenter region to deploy all the resources" 9 | type = string 10 | } 11 | 12 | variable "prefix" { 13 | description = "Resource prefix" 14 | type = string 15 | default = "pf" 16 | } 17 | 18 | variable "slave-count" { 19 | description = "Number of proxy slaves to deploy" 20 | type = number 21 | default = 2 22 | } 23 | 24 | 25 | variable "lb-count" { 26 | description = "Number of master nodes to setup for load balancer, min 1 is required" 27 | type = number 28 | default = 1 29 | } 30 | 31 | variable "tag-name" { 32 | description = "Tag name to group slave droplets" 33 | type = string 34 | default = "proxy-forge-slave" 35 | } 36 | 37 | variable "droplet-size" { 38 | description = "Droplet size to use" 39 | type = object({ 40 | slave = string 41 | }) 42 | default = { 43 | slave = "s-1vcpu-1gb-amd" 44 | } 45 | } 46 | 47 | variable "project" { 48 | description = "Name of the project to associate all the resources" 49 | type = string 50 | default = "Proxy Forge" 51 | } 52 | 53 | variable "ssh-fingerprint" { 54 | description = "SSH fingerprint id for droplets to use. If this is ommited, it will send you one-time-password on the email." 55 | type = string 56 | sensitive = true 57 | default = null 58 | } 59 | 60 | variable "squid-credentials" { 61 | description = "Squid proxy HTTP basic authentication credentials" 62 | type = object({ username = string, password = string }) 63 | default = { password = "proxyforge", username = "proxyforge" } 64 | } 65 | --------------------------------------------------------------------------------