├── .gitignore ├── src ├── defense │ ├── blocklist.cr │ ├── safelist.cr │ ├── allow2ban.cr │ ├── store.cr │ ├── handler.cr │ ├── throttle.cr │ ├── memory_store.cr │ ├── redis_store.cr │ └── fail2ban.cr └── defense.cr ├── shard.lock ├── shard.yml ├── .github └── workflows │ ├── linters.yml │ └── tests.yml ├── spec ├── spec_helper.cr └── integration │ ├── safelist_spec.cr │ ├── blocklist_spec.cr │ ├── throttle_spec.cr │ ├── fail2ban_spec.cr │ └── allow2ban_spec.cr └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /docs/ 2 | /lib/ 3 | /bin/ 4 | /.shards/ 5 | *.dwarf 6 | sentry_cli 7 | -------------------------------------------------------------------------------- /src/defense/blocklist.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | private class Blocklist 3 | getter :name, :block 4 | 5 | def initialize(@name : String, &@block : (HTTP::Request) -> Bool) 6 | end 7 | 8 | def matched_by?(request : HTTP::Request) : Bool 9 | block.call(request) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /src/defense/safelist.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | private class Safelist 3 | getter :name, :block 4 | 5 | def initialize(@name : String, &@block : (HTTP::Request) -> Bool) 6 | end 7 | 8 | def matched_by?(request : HTTP::Request) : Bool 9 | block.call(request) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /shard.lock: -------------------------------------------------------------------------------- 1 | version: 2.0 2 | shards: 3 | ameba: 4 | git: https://github.com/crystal-ameba/ameba.git 5 | version: 1.5.0 6 | 7 | db: 8 | git: https://github.com/crystal-lang/crystal-db.git 9 | version: 0.12.0 10 | 11 | redis: 12 | git: https://github.com/jgaskins/redis.git 13 | version: 0.8.0 14 | 15 | -------------------------------------------------------------------------------- /shard.yml: -------------------------------------------------------------------------------- 1 | name: defense 2 | version: 0.5.0 3 | 4 | authors: 5 | - Florin Lipan 6 | - Rodrigo Pinto 7 | 8 | crystal: "1.0.0, < 2.0.0" 9 | 10 | dependencies: 11 | redis: 12 | github: jgaskins/redis 13 | version: ~> 0.8.0 14 | 15 | development_dependencies: 16 | ameba: 17 | github: crystal-ameba/ameba 18 | version: ~> 1.5.0 19 | -------------------------------------------------------------------------------- /src/defense/allow2ban.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | class Allow2Ban < Fail2Ban 3 | private def self.fail!(discriminator : String, maxretry : Int32, findtime : Int32, bantime : Int32) : Bool 4 | count = store.increment("#{prefix}:count:#{discriminator}", findtime) 5 | 6 | if count >= maxretry 7 | ban!(discriminator, bantime) 8 | end 9 | 10 | false 11 | end 12 | 13 | private def self.prefix 14 | "allow2ban" 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /src/defense/store.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | abstract class Store 3 | abstract def exists?(unprefixed_key : String) : Bool 4 | abstract def increment(unprefixed_key : String, expires_in : Int32) : Int64 5 | abstract def read(unprefixed_key : String) : Int64 | Nil 6 | abstract def reset 7 | 8 | def prefix 9 | "defense" 10 | end 11 | 12 | def prefix_key(unprefixed_key : String) : String 13 | "#{prefix}:#{unprefixed_key}" 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /src/defense/handler.cr: -------------------------------------------------------------------------------- 1 | require "http/server/handler" 2 | 3 | module Defense 4 | class Handler 5 | include HTTP::Handler 6 | 7 | def initialize 8 | end 9 | 10 | def call(context : HTTP::Server::Context) 11 | if Defense.safelisted?(context.request) 12 | call_next(context) 13 | elsif Defense.blocklisted?(context.request) 14 | Defense.blocklisted_response.call(context.response) 15 | elsif Defense.throttled?(context.request) 16 | Defense.throttled_response.call(context.response) 17 | else 18 | call_next(context) 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /src/defense/throttle.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | private class Throttle 3 | getter :name, :limit, :period, :block 4 | 5 | def initialize(@name : String, @limit : Int32, @period : Int32, &@block : (HTTP::Request) -> String?) 6 | end 7 | 8 | def matched_by?(request : HTTP::Request) : Bool 9 | discriminator = block.call(request) 10 | return false unless discriminator 11 | 12 | count = store.increment("#{prefix}:#{discriminator}", period) 13 | 14 | count > limit 15 | end 16 | 17 | private def store 18 | Defense.store 19 | end 20 | 21 | private def prefix 22 | "throttle:#{name}" 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - "*" 10 | 11 | jobs: 12 | linters: 13 | name: Linters 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Download source 17 | uses: actions/checkout@v3 18 | 19 | - name: Install Crystal 20 | uses: crystal-lang/install-crystal@v1 21 | 22 | - name: Cache shards 23 | uses: actions/cache@v2 24 | with: 25 | path: lib 26 | key: ${{ runner.os }}-shards-${{ hashFiles('**/shard.lock') }} 27 | restore-keys: ${{ runner.os }}-shards- 28 | 29 | - name: Install shards 30 | run: shards update 31 | 32 | - name: Run linter 33 | run: bin/ameba 34 | -------------------------------------------------------------------------------- /src/defense/memory_store.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | class MemoryStore < Store 3 | def initialize 4 | @data = Hash(String, Hash(String, Int64)).new 5 | end 6 | 7 | def increment(unprefixed_key : String, expires_in : Int32) : Int64 8 | current_time = Time.utc.to_unix_ms 9 | 10 | key = prefix_key(unprefixed_key) 11 | 12 | if exists?(unprefixed_key) 13 | @data[key]["count"] += 1 14 | else 15 | @data[key] = {"count" => 1i64, "expires_at" => (current_time + expires_in * 1000)} 16 | end 17 | 18 | @data[key]["count"] 19 | end 20 | 21 | def exists?(unprefixed_key : String) : Bool 22 | key = prefix_key(unprefixed_key) 23 | @data.has_key?(key) && @data[key]["expires_at"] > Time.utc.to_unix_ms 24 | end 25 | 26 | def read(unprefixed_key : String) : Int64 | Nil 27 | if exists?(unprefixed_key) 28 | @data[prefix_key(unprefixed_key)]["count"] 29 | end 30 | end 31 | 32 | def reset 33 | @data.clear 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - "*" 10 | 11 | jobs: 12 | tests: 13 | name: Tests 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | store: 18 | - memory 19 | - redis 20 | services: 21 | redis: 22 | image: redis 23 | ports: 24 | - 6379:6379 25 | steps: 26 | - name: Download source 27 | uses: actions/checkout@v3 28 | 29 | - name: Install Crystal 30 | uses: crystal-lang/install-crystal@v1 31 | 32 | - name: Cache shards 33 | uses: actions/cache@v2 34 | with: 35 | path: lib 36 | key: ${{ runner.os }}-shards-${{ hashFiles('**/shard.lock') }} 37 | restore-keys: ${{ runner.os }}-shards- 38 | 39 | - name: Install shards 40 | run: shards update 41 | 42 | - name: Run tests 43 | run: crystal spec --verbose 44 | env: 45 | STORE: ${{ matrix.store }} 46 | -------------------------------------------------------------------------------- /src/defense/redis_store.cr: -------------------------------------------------------------------------------- 1 | require "redis" 2 | 3 | module Defense 4 | class RedisStore < Store 5 | def initialize(url : String? = nil) 6 | if !url.nil? 7 | @redis = Redis::Client.new(URI.parse(url)) 8 | elsif ENV.has_key?("REDIS_URL") 9 | @redis = Redis::Client.from_env("REDIS_URL") 10 | else 11 | @redis = Redis::Client.new 12 | end 13 | end 14 | 15 | def increment(unprefixed_key : String, expires_in : Int32) : Int64 16 | key = prefix_key(unprefixed_key) 17 | 18 | @redis.multi do |r| 19 | r.incr(key) 20 | r.expire(key, expires_in) 21 | end.first.as(Int64) 22 | end 23 | 24 | def exists?(unprefixed_key : String) : Bool 25 | @redis.exists(prefix_key(unprefixed_key)) == 1 26 | end 27 | 28 | def read(unprefixed_key : String) : Int64 | Nil 29 | @redis.get(prefix_key(unprefixed_key)).try(&.to_i64) 30 | end 31 | 32 | def reset 33 | keys = @redis.keys("#{prefix}:*") 34 | return if keys.empty? 35 | @redis.del(keys.map(&.to_s)) 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /src/defense/fail2ban.cr: -------------------------------------------------------------------------------- 1 | module Defense 2 | class Fail2Ban 3 | def self.filter(discriminator : String, maxretry : Int32 = 0, findtime : Int32 = 0, bantime : Int32 = 0, &) : Bool 4 | if banned?(discriminator) 5 | true 6 | elsif yield 7 | fail!(discriminator, maxretry, findtime, bantime) 8 | else 9 | false 10 | end 11 | end 12 | 13 | private def self.banned?(discriminator : String) : Bool 14 | store.exists?("#{prefix}:ban:#{discriminator}") 15 | end 16 | 17 | private def self.fail!(discriminator : String, maxretry : Int32, findtime : Int32, bantime : Int32) : Bool 18 | count = store.increment("#{prefix}:count:#{discriminator}", findtime) 19 | 20 | if count >= maxretry 21 | ban!(discriminator, bantime) 22 | end 23 | 24 | true 25 | end 26 | 27 | private def self.ban!(discriminator : String, bantime : Int32) 28 | store.increment("#{prefix}:ban:#{discriminator}", bantime) 29 | end 30 | 31 | private def self.store 32 | Defense.store 33 | end 34 | 35 | private def self.prefix 36 | "fail2ban" 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/spec_helper.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | require "../src/defense" 3 | 4 | if ENV["STORE"]? == "memory" 5 | Defense.store = Defense::MemoryStore.new 6 | end 7 | 8 | # Reopen the class so we can have access to protected/private methods inside a test run. 9 | module Defense 10 | ORIGINAL_THROTTLED_RESPONSE = throttled_response 11 | ORGINAL_BLOCKLISTED_RESPONSE = blocklisted_response 12 | 13 | def self.reset_for_tests 14 | reset 15 | throttles.clear 16 | blocklists.clear 17 | safelists.clear 18 | Defense.throttled_response = ORIGINAL_THROTTLED_RESPONSE 19 | Defense.blocklisted_response = ORGINAL_BLOCKLISTED_RESPONSE 20 | end 21 | end 22 | 23 | Spec.before_each do 24 | Defense.reset_for_tests 25 | end 26 | 27 | module Helper 28 | def self.call_handler(request : HTTP::Request, response_io : IO = IO::Memory.new) : HTTP::Client::Response 29 | response = HTTP::Server::Response.new(response_io) 30 | ctx = HTTP::Server::Context.new(request, response) 31 | 32 | handler = Defense::Handler.new 33 | handler.next = ->(_ctx : HTTP::Server::Context) {} 34 | handler.call(ctx) 35 | 36 | ctx.response.close 37 | response_io.rewind 38 | 39 | HTTP::Client::Response.from_io(response_io, decompress: false) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/integration/safelist_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | 3 | describe "Defense.safelist" do 4 | it "if the request is blocked but the safelisted block matches, the request is not blocked" do 5 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 6 | 7 | Defense.safelist { |req| req.headers["user-agent"]? == "bot" } 8 | Defense.blocklist { true } 9 | 10 | response = Helper.call_handler(request) 11 | response.status.should eq(HTTP::Status::OK) 12 | end 13 | 14 | it "if the request is blocked and one of several safelisted blocks matches, the request is not blocked" do 15 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 16 | 17 | Defense.safelist { |req| req.headers["user-agent"]? == "not-a-bot" } 18 | Defense.safelist { |req| req.headers["user-agent"]? == "bot" } 19 | Defense.blocklist { true } 20 | 21 | response = Helper.call_handler(request) 22 | response.status.should eq(HTTP::Status::OK) 23 | end 24 | 25 | it "if the request is blocked and the safelisted block doesn't match, the request is blocked" do 26 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 27 | 28 | Defense.safelist { |req| req.headers["user-agent"]? == "not-a-bot" } 29 | Defense.blocklist { true } 30 | 31 | response = Helper.call_handler(request) 32 | response.status.should eq(HTTP::Status::FORBIDDEN) 33 | response.body.should eq("Forbidden\n") 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /spec/integration/blocklist_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | 3 | describe "Defense.blocklist" do 4 | it "does not block the request if the block doesn't match the request" do 5 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 6 | 7 | Defense.blocklist { |req| req.headers["user-agent"]? == "not-a-bot" } 8 | 9 | response = Helper.call_handler(request) 10 | response.status.should eq(HTTP::Status::OK) 11 | end 12 | 13 | it "blocks the request if the block matches the request" do 14 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 15 | 16 | Defense.blocklist { |req| req.headers["user-agent"]? == "bot" } 17 | 18 | response = Helper.call_handler(request) 19 | response.status.should eq(HTTP::Status::FORBIDDEN) 20 | response.body.should eq("Forbidden\n") 21 | end 22 | 23 | it "blocks the request if one of several blocks matches the request" do 24 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 25 | 26 | Defense.blocklist { |req| req.headers["user-agent"]? == "not-a-bot" } 27 | Defense.blocklist { |req| req.headers["user-agent"]? == "bot" } 28 | 29 | response = Helper.call_handler(request) 30 | response.status.should eq(HTTP::Status::FORBIDDEN) 31 | response.body.should eq("Forbidden\n") 32 | end 33 | 34 | it "adapts the blocklisted response based on the value of Defense.blocklisted_response" do 35 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 36 | 37 | Defense.blocklist { |req| req.headers["user-agent"]? == "bot" } 38 | Defense.blocklisted_response = ->(response : HTTP::Server::Response) do 39 | response.status = HTTP::Status::UNAUTHORIZED 40 | response.content_type = "application/json" 41 | response.puts("{'hello':'world'}") 42 | end 43 | 44 | response = Helper.call_handler(request) 45 | response.status.should eq(HTTP::Status::UNAUTHORIZED) 46 | response.body.should eq("{'hello':'world'}\n") 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /src/defense.cr: -------------------------------------------------------------------------------- 1 | require "http/request" 2 | require "http/server/response" 3 | require "uuid" 4 | require "./defense/throttle" 5 | require "./defense/blocklist" 6 | require "./defense/fail2ban" 7 | require "./defense/allow2ban" 8 | require "./defense/safelist" 9 | require "./defense/store" 10 | require "./defense/memory_store" 11 | require "./defense/redis_store" 12 | require "./defense/handler" 13 | 14 | module Defense 15 | def self.throttle(name : String, limit : Int32, period : Int32, &block : (HTTP::Request) -> String?) 16 | throttles[name] = Throttle.new(name, limit, period, &block) 17 | end 18 | 19 | def self.blocklist(name : String = UUID.random.to_s, &block : (HTTP::Request) -> Bool) 20 | blocklists[name] = Blocklist.new(name, &block) 21 | end 22 | 23 | def self.safelist(name : String = UUID.random.to_s, &block : (HTTP::Request) -> Bool) 24 | safelists[name] = Safelist.new(name, &block) 25 | end 26 | 27 | def self.throttled_response=(block : (HTTP::Server::Response) -> Nil) 28 | @@throttled_response = block 29 | end 30 | 31 | def self.blocklisted_response=(block : (HTTP::Server::Response) -> Nil) 32 | @@blocklisted_response = block 33 | end 34 | 35 | def self.reset 36 | store.reset 37 | end 38 | 39 | def self.store : Store 40 | @@store ||= RedisStore.new 41 | end 42 | 43 | def self.store=(store : Store) 44 | @@store = store 45 | end 46 | 47 | @@throttled_response : (HTTP::Server::Response) -> Nil = ->(response : HTTP::Server::Response) do 48 | response.status = HTTP::Status::TOO_MANY_REQUESTS 49 | response.content_type = "text/plain" 50 | response.puts("Retry later\n") 51 | end 52 | 53 | protected def self.throttled_response 54 | @@throttled_response 55 | end 56 | 57 | @@blocklisted_response : (HTTP::Server::Response) -> Nil = ->(response : HTTP::Server::Response) do 58 | response.status = HTTP::Status::FORBIDDEN 59 | response.content_type = "text/plain" 60 | response.puts("Forbidden\n") 61 | end 62 | 63 | protected def self.blocklisted_response 64 | @@blocklisted_response 65 | end 66 | 67 | protected def self.throttles 68 | @@throttles ||= Hash(String, Throttle).new 69 | end 70 | 71 | protected def self.blocklists 72 | @@blocklists ||= Hash(String, Blocklist).new 73 | end 74 | 75 | protected def self.safelists 76 | @@safelists ||= Hash(String, Safelist).new 77 | end 78 | 79 | protected def self.throttled?(request) 80 | throttles.any? do |_, throttle| 81 | throttle.matched_by?(request) 82 | end 83 | end 84 | 85 | protected def self.blocklisted?(request) 86 | blocklists.any? do |_, blocklist| 87 | blocklist.matched_by?(request) 88 | end 89 | end 90 | 91 | protected def self.safelisted?(request) 92 | safelists.any? do |_, safelist| 93 | safelist.matched_by?(request) 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /spec/integration/throttle_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | 3 | describe "Defense.throttle" do 4 | it "does not block the requests before exceeding the rule" do 5 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 6 | 7 | Defense.throttle("my-throttle-rule", limit: 1, period: 60) { |req| req.headers["user-agent"]? } 8 | 9 | response = Helper.call_handler(request) 10 | response.status.should eq(HTTP::Status::OK) 11 | end 12 | 13 | it "blocks the requests that exceed the rule" do 14 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 15 | 16 | Defense.throttle("my-throttle-rule", limit: 5, period: 60) { |req| req.headers["user-agent"]? } 17 | 18 | 5.times { Helper.call_handler(request) } 19 | 20 | response = Helper.call_handler(request) 21 | response.status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 22 | response.body.should eq("Retry later\n") 23 | end 24 | 25 | it "adapts the throttled response based on the value of Defense.throttled_response" do 26 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 27 | 28 | Defense.throttle("my-throttle-rule", limit: 2, period: 60) { |req| req.headers["user-agent"]? } 29 | Defense.throttled_response = ->(response : HTTP::Server::Response) do 30 | response.status = HTTP::Status::UNAUTHORIZED 31 | response.content_type = "application/json" 32 | response.puts("{'hello':'world'}") 33 | end 34 | 35 | 2.times { Helper.call_handler(request) } 36 | 37 | response = Helper.call_handler(request) 38 | response.status.should eq(HTTP::Status::UNAUTHORIZED) 39 | response.body.should eq("{'hello':'world'}\n") 40 | end 41 | 42 | it "blocks requests only within the defined period" do 43 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 44 | 45 | Defense.throttle("my-throttle-rule", limit: 3, period: 1) { |req| req.headers["user-agent"]? } 46 | 47 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 48 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 49 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 50 | Helper.call_handler(request).status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 51 | 52 | sleep(0.1) 53 | 54 | Helper.call_handler(request).status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 55 | 56 | sleep(0.1) 57 | 58 | Helper.call_handler(request).status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 59 | 60 | sleep(1) 61 | 62 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 63 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 64 | Helper.call_handler(request).status.should eq(HTTP::Status::OK) 65 | Helper.call_handler(request).status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 66 | end 67 | 68 | it "blocks requests by IP" do 69 | request = HTTP::Request.new("GET", "/", HTTP::Headers{"user-agent" => "bot"}) 70 | 71 | Defense.throttle("my-throttle-rule", limit: 5, period: 60) { |req| req.remote_address.to_s } 72 | 73 | 5.times { Helper.call_handler(request) } 74 | 75 | response = Helper.call_handler(request) 76 | response.status.should eq(HTTP::Status::TOO_MANY_REQUESTS) 77 | response.body.should eq("Retry later\n") 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /spec/integration/fail2ban_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | 3 | private def setup 4 | Defense.blocklist do |req| 5 | Defense::Fail2Ban.filter("spec-#{req.headers["host"]}", maxretry: 2, bantime: 60, findtime: 60) do 6 | (req.query =~ /FAIL/) != nil 7 | end 8 | end 9 | end 10 | 11 | private def headers 12 | HTTP::Headers{"Host" => "1.2.3.4"} 13 | end 14 | 15 | describe "Defense.fail2ban" do 16 | describe "while the discriminator is not banned" do 17 | context "receives a valid request" do 18 | it "responds with success" do 19 | setup 20 | request = HTTP::Request.new("GET", "/", headers) 21 | 22 | response = Helper.call_handler(request) 23 | response.status.should eq(HTTP::Status::OK) 24 | end 25 | end 26 | 27 | context "receives an invalid request" do 28 | context "while maxretry is not reached" do 29 | it "responds with forbidden" do 30 | setup 31 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 32 | 33 | response = Helper.call_handler(request) 34 | response.status.should eq(HTTP::Status::FORBIDDEN) 35 | end 36 | 37 | it "increments the fail counter" do 38 | setup 39 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 40 | 41 | Helper.call_handler(request) 42 | 43 | ip = request.headers["host"] 44 | Defense.store.read("fail2ban:count:spec-#{ip}").should eq(1) 45 | end 46 | 47 | it "is not banned yet" do 48 | setup 49 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 50 | 51 | Helper.call_handler(request) 52 | 53 | ip = request.headers["host"] 54 | Defense.store.read("fail2ban:ban:spec-#{ip}").should be_nil 55 | end 56 | end 57 | 58 | context "when maxretry is reached" do 59 | it "responds with forbidden" do 60 | setup 61 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 62 | 63 | Helper.call_handler(request) 64 | last_response = Helper.call_handler(request) 65 | 66 | last_response.status.should eq(HTTP::Status::FORBIDDEN) 67 | end 68 | 69 | it "increments the fail counter" do 70 | setup 71 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 72 | 73 | 2.times { Helper.call_handler(request) } 74 | 75 | ip = request.headers["host"] 76 | Defense.store.read("fail2ban:count:spec-#{ip}").should eq(2) 77 | end 78 | 79 | it "is banned" do 80 | setup 81 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 82 | 83 | 2.times { Helper.call_handler(request) } 84 | 85 | ip = request.headers["host"] 86 | Defense.store.read("fail2ban:ban:spec-#{ip}").should eq(1) 87 | end 88 | end 89 | end 90 | end 91 | 92 | describe "when the discriminator has been banned already" do 93 | context "receives a valid request for another discriminator" do 94 | it "responds with success" do 95 | setup 96 | 97 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 98 | headers = HTTP::Headers{"Host" => "4.3.2.1"} 99 | request = HTTP::Request.new("GET", "/", headers) 100 | 101 | response = Helper.call_handler(request) 102 | response.status.should eq(HTTP::Status::OK) 103 | end 104 | end 105 | 106 | context "receives a valid request for the banned discriminator" do 107 | it "responds with forbidden" do 108 | setup 109 | 110 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 111 | request = HTTP::Request.new("GET", "/", headers) 112 | 113 | response = Helper.call_handler(request) 114 | response.status.should eq(HTTP::Status::FORBIDDEN) 115 | end 116 | end 117 | 118 | context "receives an invalid request for the banned discriminator" do 119 | it "responds with forbidden" do 120 | setup 121 | 122 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 123 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 124 | 125 | response = Helper.call_handler(request) 126 | response.status.should eq(HTTP::Status::FORBIDDEN) 127 | end 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /spec/integration/allow2ban_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | 3 | private def setup 4 | Defense.blocklist do |req| 5 | Defense::Allow2Ban.filter("spec-#{req.headers["host"]}", maxretry: 2, bantime: 60, findtime: 60) do 6 | (req.query =~ /FAIL/) != nil 7 | end 8 | end 9 | end 10 | 11 | private def headers 12 | HTTP::Headers{"Host" => "1.2.3.4"} 13 | end 14 | 15 | describe "Defense.allow2ban" do 16 | describe "while the discriminator is not banned" do 17 | context "receives a valid request" do 18 | it "responds with success" do 19 | setup 20 | request = HTTP::Request.new("GET", "/", headers) 21 | 22 | response = Helper.call_handler(request) 23 | response.status.should eq(HTTP::Status::OK) 24 | end 25 | end 26 | 27 | context "receives an invalid request" do 28 | context "while maxretry is not reached" do 29 | it "responds with success" do 30 | setup 31 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 32 | 33 | response = Helper.call_handler(request) 34 | response.status.should eq(HTTP::Status::OK) 35 | end 36 | 37 | it "increments the fail counter" do 38 | setup 39 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 40 | 41 | Helper.call_handler(request) 42 | 43 | ip = request.headers["host"] 44 | Defense.store.read("allow2ban:count:spec-#{ip}").should eq(1) 45 | end 46 | 47 | it "is not banned yet" do 48 | setup 49 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 50 | 51 | Helper.call_handler(request) 52 | 53 | ip = request.headers["host"] 54 | Defense.store.read("allow2ban:ban:spec-#{ip}").should be_nil 55 | end 56 | end 57 | 58 | context "when maxretry is exceeded" do 59 | it "responds with forbidden" do 60 | setup 61 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 62 | 63 | 2.times { Helper.call_handler(request) } 64 | last_response = Helper.call_handler(request) 65 | 66 | last_response.status.should eq(HTTP::Status::FORBIDDEN) 67 | end 68 | 69 | it "increments the fail counter" do 70 | setup 71 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 72 | 73 | 3.times { Helper.call_handler(request) } 74 | 75 | ip = request.headers["host"] 76 | Defense.store.read("allow2ban:count:spec-#{ip}").should eq(2) 77 | end 78 | 79 | it "is banned" do 80 | setup 81 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 82 | 83 | 3.times { Helper.call_handler(request) } 84 | 85 | ip = request.headers["host"] 86 | Defense.store.read("allow2ban:ban:spec-#{ip}").should eq(1) 87 | end 88 | end 89 | end 90 | end 91 | 92 | describe "when the discriminator has been banned already" do 93 | context "receives a valid request for another discriminator" do 94 | it "responds with success" do 95 | setup 96 | 97 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 98 | headers = HTTP::Headers{"Host" => "4.3.2.1"} 99 | request = HTTP::Request.new("GET", "/", headers) 100 | 101 | response = Helper.call_handler(request) 102 | response.status.should eq(HTTP::Status::OK) 103 | end 104 | end 105 | 106 | context "receives a valid request for the banned discriminator" do 107 | it "responds with forbidden" do 108 | setup 109 | 110 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 111 | request = HTTP::Request.new("GET", "/", headers) 112 | 113 | response = Helper.call_handler(request) 114 | response.status.should eq(HTTP::Status::FORBIDDEN) 115 | end 116 | end 117 | 118 | context "receives an invalid request for the banned discriminator" do 119 | it "responds with forbidden" do 120 | setup 121 | 122 | 2.times { Helper.call_handler(HTTP::Request.new("GET", "/?filter=FAIL", headers)) } 123 | request = HTTP::Request.new("GET", "/?filter=FAIL", headers) 124 | 125 | response = Helper.call_handler(request) 126 | response.status.should eq(HTTP::Status::FORBIDDEN) 127 | end 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Defense 2 | 3 | [![Build Status](https://github.com/defense-cr/defense/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/defense-cr/defense/actions/workflows/tests.yml/?branch=master) 4 | 5 | 🔮 *A Crystal HTTP handler for throttling, blocking and tracking malicious requests* 🔮 6 | 7 | ## Getting started 8 | 9 | ### Installation 10 | 11 | Add the shard as a dependency to your project's `shards.yml`: 12 | 13 | ```yaml 14 | dependencies: 15 | defense: 16 | github: defense-cr/defense 17 | ``` 18 | 19 | ...and install it: 20 | 21 | ```sh 22 | shards install 23 | ``` 24 | 25 | ### Configure the data store 26 | 27 | Defense stores its state in a **Redis** database. You can configure this by setting the `REDIS_URL` environment variable or by using the `Defense#store=` method: 28 | 29 | ```crystal 30 | Defense.store = Defense::RedisStore.new(url: "redis://localhost:6379/0") 31 | ``` 32 | 33 | For simple use cases or tests you can also use the **memory store**: 34 | 35 | ```crystal 36 | Defense.store = Defense::MemoryStore.new 37 | ``` 38 | 39 | You can always implement your own **custom store** by extending the abstract class `Defense::Store`. 40 | 41 | ### Plugging into the application 42 | 43 | Defense is built as a Crystal `HTTP::Handler`. You will need to register the `Defense::Handler` to your web application's handler chain. For more information about *handlers* and the *handler chain* follow [this link](https://crystal-lang.org/api/latest/HTTP/Server.html). 44 | 45 | Usually the earlier you register the handler to your handler chain, the better. This ensures that malicious requests are blocked early own, before other layers (handlers) of your application are reached. 46 | 47 | Here's how to plug Defense into some of the **most popular Crystal web frameworks**: 48 | 49 | #### Kemal 50 | 51 | In Kemal you would use the `add_handler` method to register the Defense handler: 52 | 53 | ```crystal 54 | require "kemal" 55 | require "defense" 56 | 57 | add_handler Defense::Handler.new 58 | 59 | # Other handlers... 60 | add_handler SomeOtherHandler.new 61 | 62 | get "/" do 63 | "hello world" 64 | end 65 | 66 | Kemal.run 67 | ``` 68 | 69 | For more details, check out the [kemal-defense-example repository](https://github.com/defense-cr/kemal-defense-example). 70 | 71 | #### Amber 72 | 73 | In Amber you register handlers as part of a pipeline in your `config/routes.cr` file: 74 | 75 | ```crystal 76 | Amber::Server.configure do |app| 77 | pipeline :web do 78 | plug Defense::Handler.new 79 | 80 | # Other handlers... 81 | plug SomeOtherHandler.new 82 | end 83 | 84 | routes :web do 85 | get "/", HomeController, :index 86 | end 87 | end 88 | ``` 89 | 90 | #### Lucky 91 | 92 | In Lucky, you would add the `Defense::Handler` within your `src/app_server.cr` file, somewhere before the `Lucky::RouteHandler`: 93 | 94 | ```crystal 95 | class AppServer < Lucky::BaseAppServer 96 | def middleware 97 | [ 98 | Defense::Handler.new, 99 | 100 | # Other handlers... 101 | SomeOtherHandler.new, 102 | 103 | Lucky::RouteHandler.new, 104 | ] 105 | end 106 | end 107 | ``` 108 | 109 | #### HTTP::Server (Standalone) 110 | 111 | When using the standard library `HTTP::Server`, any middleware is registered as part of the initializer: 112 | 113 | ```crystal 114 | require "defense" 115 | require "http/server" 116 | 117 | server = HTTP::Server.new([Defense::Handler.new]) do |context| 118 | context.response.content_type = "text/plain" 119 | context.response.print "hello world" 120 | end 121 | 122 | server.bind_tcp(8080) 123 | server.listen 124 | ``` 125 | 126 | ### Usage 127 | 128 | Defense provides a set of configurable rules that you can use to throttle, block and track malicious requests based on your own heuristics: 129 | 130 | - [Throttling](#throttling) 131 | - [Configure the throttled response](#configure-the-throttled-response) 132 | - [Blocklist](#blocklist) 133 | - [Configure the blocked response](#configure-the-blocked-response) 134 | - [Fail2Ban](#fail2ban) 135 | - [Allow2Ban](#allow2ban) 136 | - [Safelist](#safelist) 137 | 138 | #### Throttling 139 | 140 | The `Defense.throttle` method can be used to throttle clients based on a maximum number of requests (*limit*) over a given time frame specified in seconds (*period*). 141 | 142 | The method takes a block which receives the `request` as an argument. The return value of the block should either be `nil` (in which case the request will not be counted at all) or a `String` which uniquely identifies the client to throttle. A good identifier is usually the IP address. 143 | 144 | The following example throttles clients based on their IP address to a limit of 10 requests per minute: 145 | 146 | ```crystal 147 | Defense.throttle("throttle requests per minute", limit: 10, period: 60) do |request| 148 | request.remote_address.to_s 149 | end 150 | ``` 151 | 152 | The following example throttles clients in a similar way but will ignore requests coming from `127.0.0.1`: 153 | 154 | ```crystal 155 | Defense.throttle("throttle requests per minute except localhost", limit: 10, period: 60) do |request| 156 | return nil if request.remote_address.to_s == "127.0.0.1" 157 | 158 | request.remote_address.to_s 159 | end 160 | ``` 161 | 162 | #### Configure the throttled response 163 | 164 | Throttled requests are responded with: 165 | 166 | ```http 167 | HTTP/1.1 429 Too Many Requests 168 | content-type: text/plain 169 | content-length: 10 170 | 171 | Retry later 172 | ``` 173 | 174 | You can override the default response message by using the `Defense.throttled_response=` method: 175 | 176 | ```crystal 177 | Defense.throttled_response = ->(response : HTTP::Server::Response) do 178 | response.status = HTTP::Status::UNAUTHORIZED 179 | response.content_type = "application/json" 180 | response.puts("{'hello':'world'}") 181 | end 182 | ``` 183 | 184 | #### Blocklist 185 | 186 | The `Defense.blocklist` method can be used to block malicious or unwanted requests. 187 | 188 | The method takes a block which receives the `request` as an argument. The return value of the block should either be `true` - in which case the request will be blocked, or `false` - in which case the request will be allowed. 189 | 190 | The following example blocks all requests to `/admin/*`: 191 | 192 | ```crystal 193 | Defense.blocklist("block requests to the admin") do |request| 194 | request.path.starts_with?("/admin/") 195 | end 196 | ``` 197 | 198 | The following example blocks requests based on a predefined list of malicious IPs: 199 | 200 | ```crystal 201 | MALICIOUS_IPS = ["1.1.1.1", "2.2.2.2", "3.3.3.3"] 202 | 203 | Defense.blocklist("block requests from malicious ips") do |request| 204 | MALICIOUS_IPS.includes?(request.remote_address.to_s) 205 | end 206 | ``` 207 | 208 | The [Spamhaus DROP lists](https://www.spamhaus.org/drop/) are a great resource for malicious IPs to block. 209 | 210 | #### Configure the blocked response 211 | 212 | Blocked requests are responded with: 213 | 214 | ```http 215 | HTTP/1.1 403 Forbidden 216 | content-type: text/plain 217 | content-length: 9 218 | 219 | Forbidden 220 | ``` 221 | 222 | You can override the default response message by using the `Defense.blocked_response=` method: 223 | 224 | ```crystal 225 | Defense.blocked_response = ->(response : HTTP::Server::Response) do 226 | response.status = HTTP::Status::UNAUTHORIZED 227 | response.content_type = "application/json" 228 | response.puts("{'hello':'world'}") 229 | end 230 | ``` 231 | 232 | #### Fail2Ban 233 | 234 | The `Defense::Fail2Ban.filter` method can be used within a `Defense.blocklist` block to ban misbehaving clients for a given period of time (*bantime*) after a sequence of blocked requests (*maxretry*) performed over a particular time range (*findtime*). 235 | 236 | The method's first argument should be a unique identifier of the client - the IP address is usually a safe bet. It's highly recommended to namespace this identifier, in order to avoid conflicts with other `Fail2Ban` or `Allow2Ban` calls - e.g. `my-fancy-filter:#{request.remote_address.to_s}` would be a good identifier. 237 | 238 | The method also takes a block which should return `true` - in which case the request will be blocked and counted for the ban, or `false` - in which case the request will be allowed and excluded from the ban count. Note that the return value of the `#filter` block will also be used as a return value for the `#blocklist` block. 239 | 240 | The following example blocks any requests containing `/etc/passwd` inside the path and, once a particular client identified by IP has accumulated 5 requests wihin 60 seconds, it bans him for the next 24 hours: 241 | 242 | ```crystal 243 | Defense.blocklist("fail2ban pentesters") do |request| 244 | Defense::Fail2Ban.filter("pentesters:#{request.remote_address.to_s}", maxretry: 5, findtime: 60, bantime: 24 * 60 * 60) do 245 | request.path.includes?("/etc/passwd") 246 | end 247 | end 248 | ``` 249 | 250 | #### Allow2Ban 251 | 252 | The `Defense::Allow2Ban.filter` method works the same way as `Defense::Fail2Ban.filter` except that it allows requests from misbehaving clients until such time as they reach *maxretry* at which they are cut off as per normal. 253 | 254 | The following example allows all `POST /login` requests until a particular client identified by IP has accumulated 5 requests within 60 seconds, at which point it bans him for the next 24 hours: 255 | 256 | ```crystal 257 | Defense.blocklist("allow2ban too many login attempts") do |request| 258 | Defense::Allow2Ban.filter("too-many-login-attempts:#{request.remote_address.to_s}", maxretry: 5, findtime: 60, bantime: 24 * 60 * 60) do 259 | request.method == "POST" && request.path == "/login" 260 | end 261 | end 262 | ``` 263 | 264 | #### Safelist 265 | 266 | The `Defense.safelist` method can be used to exclude requests from any throttling or blocking rules. This method has precedence over all the other rules. 267 | 268 | The method takes a block which receives the `request` as an argument. The return value of the block should either be `true` - in which case the request will never be throttled or blocked, or `false` - in which case the request will be checked against the other existing rules and might potentially be throttled or blocked. 269 | 270 | The following example marks all requests originating from `127.0.0.1` as safe: 271 | 272 | ```crystal 273 | Defense.safelist("local requests are safe") do |request| 274 | request.remote_address.to_s == "127.0.0.1" 275 | end 276 | ``` 277 | 278 | ## Contributing & Development 279 | 280 | Contributions are welcome. Make sure to check the existing issues (including the closed ones) before requesting a feature, reporting a bug or opening a pull requests. 281 | 282 | ### Getting started 283 | 284 | Install dependencies: 285 | 286 | ```sh 287 | shards install 288 | ``` 289 | 290 | Run tests using Redis as a backend (requires a running Redis server): 291 | 292 | ```sh 293 | crystal spec 294 | ``` 295 | 296 | Run tests using the memory store as a backend: 297 | 298 | ```sh 299 | STORE=memory crystal spec 300 | ``` 301 | 302 | Format the code: 303 | 304 | ```sh 305 | crystal tool format 306 | ``` 307 | 308 | ### Guidelines 309 | 310 | - Keep the public interface small. Anything that doesn't have to be public, should explicitly be marked as protected or 311 | private, including classes. 312 | - Be explicit about type declaration (especially on public methods). 313 | - Use the Crystal formatter to format the code. 314 | - For now, prefer integration/system tests over unit tests. 315 | 316 | ## Maintainers 317 | 318 | - [Florin Lipan](https://github.com/lipanski) 319 | - [Rodrigo Pinto](https://github.com/rodrigopinto) 320 | 321 | ## Credits 322 | 323 | This shard is heavily inspired by [rack-attack](https://github.com/kickstarter/rack-attack) ❤ 324 | --------------------------------------------------------------------------------