├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ └── auto-update.yml ├── .gitignore ├── LICENSE ├── README.md ├── README_en.md ├── config ├── application-example.yaml └── application.yaml ├── pom.xml └── src └── main ├── java └── org │ └── fordes │ └── adfs │ ├── Application.java │ ├── config │ ├── Config.java │ ├── InputProperties.java │ └── OutputProperties.java │ ├── constant │ ├── Constants.java │ └── RegConstants.java │ ├── enums │ ├── HandleType.java │ └── RuleSet.java │ ├── handler │ ├── Parser.java │ ├── dns │ │ └── DnsChecker.java │ ├── fetch │ │ ├── Fetcher.java │ │ ├── HttpFetcher.java │ │ └── LocalFetcher.java │ └── rule │ │ ├── ClashHandler.java │ │ ├── DnsmasqHandler.java │ │ ├── EasylistHandler.java │ │ ├── Handler.java │ │ ├── HostsHandler.java │ │ └── SmartdnsHandler.java │ ├── model │ └── Rule.java │ ├── task │ └── Endpoint.java │ └── util │ ├── BloomFilter.java │ └── Util.java └── resources ├── application-dev.yml ├── application.yml ├── banner.txt └── logback-spring.xml /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # [Choice] Java version (use -bullseye variants on local arm64/Apple Silicon): 11, 17, 11-bullseye, 17-bullseye, 11-buster, 17-buster 2 | ARG VARIANT=21-bullseye 3 | FROM mcr.microsoft.com/vscode/devcontainers/java:${VARIANT} 4 | 5 | # [Option] Install Maven 6 | ARG INSTALL_MAVEN="true" 7 | ARG MAVEN_VERSION="3.9.0" 8 | # [Option] Install Gradle 9 | ARG INSTALL_GRADLE="false" 10 | ARG GRADLE_VERSION="" 11 | RUN if [ "${INSTALL_MAVEN}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/sdkman/bin/sdkman-init.sh && sdk install maven \"${MAVEN_VERSION}\""; fi \ 12 | && if [ "${INSTALL_GRADLE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/sdkman/bin/sdkman-init.sh && sdk install gradle \"${GRADLE_VERSION}\""; fi 13 | 14 | # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 15 | ARG NODE_VERSION="none" 16 | RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi 17 | 18 | # [Optional] Uncomment this section to install additional OS packages. 19 | # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 20 | # && apt-get -y install --no-install-recommends 21 | 22 | # [Optional] Uncomment this line to install global node packages. 23 | # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 24 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CodeSpace", 3 | "dockerFile": "Dockerfile", 4 | "extensions": [ 5 | "vscjava.vscode-java-pack", 6 | "vscjava.vscode-java-debug", 7 | "vscjava.vscode-java-test", 8 | "vscjava.vscode-spring-boot-dashboard", 9 | "vmware.vscode-spring-boot", 10 | "vscjava.vscode-maven" 11 | ] 12 | } -------------------------------------------------------------------------------- /.github/workflows/auto-update.yml: -------------------------------------------------------------------------------- 1 | name: Update Filters 2 | on: 3 | push: 4 | paths-ignore: 5 | - 'README.md' 6 | - 'README_en.md' 7 | - '.github/**' 8 | - '.devcontainer/**' 9 | - 'LICENSE' 10 | schedule: 11 | - cron: 0 */8 * * * 12 | workflow_dispatch: 13 | 14 | env: 15 | TZ: Asia/Shanghai 16 | 17 | jobs: 18 | Update_Filters: 19 | runs-on: ubuntu-latest 20 | permissions: 21 | contents: write 22 | actions: write 23 | 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | with: 28 | ref: ${{ github.head_ref }} 29 | 30 | - name: Set up JDK 21 31 | uses: actions/setup-java@v4 32 | with: 33 | java-version: '21' 34 | distribution: 'temurin' 35 | cache: maven 36 | 37 | - name: Run Jar 38 | run: mvn spring-boot:run 39 | 40 | - name: Clean up 41 | run: | 42 | find . -maxdepth 1 ! -name 'rule' ! -name '.' ! -name '.git' -exec rm -rf {} + 43 | mv rule/* . 44 | rm -rf rule 45 | 46 | - name: Commit Changes 47 | id: commit 48 | uses: stefanzweifel/git-auto-commit-action@v5 49 | with: 50 | commit_message: 🚀 CI Updated 51 | branch: release 52 | skip_dirty_check: true 53 | push_options: '--force' 54 | create_branch: true 55 | skip_fetch: true 56 | skip_checkout: true 57 | commit_options: '--allow-empty' 58 | 59 | - name: Delete Workflow Runs 60 | uses: Mattraks/delete-workflow-runs@v2 61 | with: 62 | token: ${{secrets.GITHUB_TOKEN}} 63 | repository: ${{ github.repository }} 64 | retain_days: 1 65 | keep_minimum_runs: 1 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | .idea 26 | target 27 | src/test -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 以谶 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

AD Filter Subscriber

3 |

4 | 广告过滤规则订阅器，整合不同来源的规则，帮助你快速构建属于自己的规则集~ 5 |

6 | 7 |

8 | last update 9 | forks 10 | stars 11 | open issues 12 | license 13 |

14 | 15 |

16 | 项目说明 17 | · 18 | 快速开始 19 | · 20 | 规则订阅 21 | · 22 | 问题反馈 23 |

24 |
25 | 26 | [English](./README_en.md) | 中文 27 |

📔 项目说明

28 | 29 | 本项目旨在聚合不同来源、不同格式的广告过滤规则，自由的进行转换和整合。 30 | > ⚠️ 新版不再兼容原配置格式，迁移前务必注意 31 | #### 支持的规则格式 32 | - [x] easylist 33 | - [x] dnsmasq 34 | - [x] clash 35 | - [x] smartdns 36 | - [x] hosts 37 | 38 | #### 注意事项 39 | 1. 仅支持基本规则转换，即域名、通配域名构成的规则，对形如 `||example.org^$popup` 等规则无法转换(合并、去重不受影响) 40 | 2. 接受不可避免的缩限，如 `||example.org^` 将拦截 example.org 及其所有子域，但将其转换为 hosts 格式时，将无法匹配子域名。 41 | 3. 规则有效性检测基于域名解析，因此仅支持基本规则 (只能检测当前域有效性，而无法检测其是否存在有效子域，故此功能可能存在误杀)。 42 | 43 |

🛠️ 快速开始

44 | 45 | ### 示例配置 46 | 47 | ```yaml 48 | application: 49 | rule: 50 | #远程规则订阅，path为 http、https地址 51 | remote: 52 | - name: 'Subscription 1' #可选参数: 规则名称，如无将使用 path 作为名称 53 | path: 'https://example.org/rule.txt' #必要参数: 规则url，仅支持 http/https，不限定响应内容格式 54 | type: easylist #可选参数: 规则类型：easylist (默认)、dnsmasq、clash、smartdns、hosts 55 | 56 | #本地规则，path为 操作系统支持的绝对或相对路径 57 | local: 58 | - name: 'private rule' 59 | path: '/rule/private.txt' 60 | 61 | output: 62 | #文件头配置，将自动作为注释添加至每个规则文件开始 63 | #可使用占位符 ${name}、${type}、${desc} 以及 ${date} (当前日期) 64 | file_header: | 65 | ADFS Adblock List 66 | Title: ${name} 67 | Last Modified: ${date} 68 | Homepage: https://github.com/fordes123/ad-filters-subscriber/ 69 | files: 70 | - name: easylist.txt #必要参数: 文件名 71 | type: EASYLIST #必要参数: 文件类型: easylist、dnsmasq、clash、smartdns、hosts 72 | desc: 'ADFS EasyList' #可选参数: 文件描述，可在file_header中通过 ${} 中使用 73 | filter: #可选参数: 包含规则的类型，默认全选 74 | - basic #基本规则，不包含任何控制、匹配符号, 可以转换为 hosts 75 | - wildcard #通配规则，仅使用通配符 76 | - unknown #其他规则，如使用了正则、高级修饰符号等，这表示目前无法支持 77 | ``` 78 | 79 | --- 80 | 本程序基于 `Java21` 编写，使用 `Maven` 进行构建，你可以参照[示例配置](./config/application-example.yaml)，编辑 `config/application.yaml` 81 | ，并通过以下任意一种方式快速开始： 82 | 83 | #### **本地调试** 84 | 85 | ```bash 86 | git clone https://github.com/fordes123/ad-filters-subscriber.git 87 | cd ad-filters-subscriber 88 | mvn clean 89 | mvn spring-boot:run 90 | ``` 91 | 92 | #### **Github Action** 93 | 94 | - fork 本项目 95 | - 自定义规则订阅 (可选) 96 | - 参照[示例配置](./config/application-example.yaml)，修改配置文件: `config/application.yaml` 97 | - 打开 `Github Action` 页面，选中左侧 `Update Filters` 授权 `Workflow` 定时执行(⚠ 重要步骤) 98 | - 点击 `Run workflow` 或等待自动执行。执行完成后规则将生成在 `release` 分支 99 | 100 | #### **Codespaces** 101 | 102 | - 登录 `Github`，点击本仓库右上角 `Code` 按钮，选择并创建新的 `Codespaces` 103 | - 等待 `Codespaces` 启动，即可直接对本项目进行调试 104 | 105 |

🎯 规则订阅

106 | 107 | **⚠ 本仓库不再提供规则订阅，我们更推荐 fork 本项目自行构建规则集.** 108 | 109 | 下面是使用了本项目进行构建的规则仓库，可在其中寻找合适的规则订阅: 110 |
111 | 点击查看 112 | 116 |
117 | 118 |

💬 问题反馈

119 | 120 | - 👉 [issues](https://github.com/fordes123/ad-filters-subscriber/issues) 121 | -------------------------------------------------------------------------------- /README_en.md: -------------------------------------------------------------------------------- 1 |
2 |

AD Filter Subscriber

3 |

4 | Ad Filter Rule Subscriber, integrating rules from various sources to help you quickly build your own rule set~ 5 |

6 | 7 |

8 | last update 9 | forks 10 | stars 11 | open issues 12 | license 13 |

14 | 15 |

16 | Introduction 17 | · 18 | Quick Start 19 | · 20 | Rule Subscription 21 | · 22 | Feedback 23 |

24 |
25 |
26 | 27 | English | [中文](./README.md) 28 |

📔 Introduction

29 | 30 | This project aims to aggregate ad filtering rules from different sources and in various formats, allowing for flexible 31 | conversion and integration. 32 | > ⚠️ Note: The new version is not compatible with the original configuration format, so please be cautious before 33 | > migrating. 34 | 35 | #### Supported Rule Formats 36 | 37 | - [x] easylist 38 | - [x] dnsmasq 39 | - [x] clash 40 | - [x] smartdns 41 | - [x] hosts 42 | 43 | #### Important Notes 44 | 45 | 1. Only basic rule conversions are supported, specifically rules consisting of domain names and wildcard domains. Rules 46 | such as `||example.org^$popup` cannot be converted (merging and deduplication are not affected). 47 | 2. Accept unavoidable limitations. For example, `||example.org^` will block example.org and all its subdomains, but when 48 | converted to hosts format, it will not match subdomains. 49 | 3. Rule validity check is based on domain resolution, so it only supports basic rules. 50 | 51 |

🛠️ Quick Start

52 | 53 | ### Example Configuration 54 | 55 | ```yaml 56 | application: 57 | rule: 58 | # Remote rule subscription, path is http/https address 59 | remote: 60 | - name: 'Subscription 1' # Optional parameter: Rule name, if no name is provided, the path will be used as the name. 61 | path: 'https://example.org/rule.txt' # Required parameter: Rule url. Only support http/https. 62 | type: easylist # Optional parameter: Rule type: easylist (default)、dnsmasq、clash、smartdns、hosts 63 | 64 | # Local rule, path is absolute or relative path 65 | local: 66 | - name: 'private rule' 67 | path: '/rule/private.txt' 68 | 69 | output: 70 | # File header configuration, which will be automatically added as comments at the beginning of each rule file. 71 | # You can use placeholders like ${name}, ${type}, ${desc}, and ${date} (current date). 72 | file_header: | 73 | ADFS Adblock List 74 | Title: ${name} 75 | Last Modified: ${date} 76 | Homepage: https://github.com/fordes123/ad-filters-subscriber/ 77 | files: 78 | - name: easylist.txt # Required parameter: File name 79 | type: EASYLIST # Required parameter: File type: easylist、dnsmasq、clash、smartdns、hosts 80 | desc: 'ADFS EasyList' # Optional parameter: File description, which can be used within ${} in the file_header. 81 | filter: # Optional parameter: Types of included rules, all selected by default. 82 | - basic # Basic rules: Do not contain any control or matching symbols, can be converted to hosts. 83 | - wildcard # Wildcard rules: Only use wildcard symbols. 84 | - unknown # Other rules: Such as those using regex or advanced modifiers, cannot be converted at present. 85 | ``` 86 | 87 | --- 88 | This program is written in `Java 21` and built using `Maven`. You can refer to the [example configuration](./config/application-example.yaml), 89 | edit `config/application.yaml`, and quickly get started using any of the following methods: 90 | 91 | #### **Local Debugging** 92 | 93 | ```bash 94 | git clone https://github.com/fordes123/ad-filters-subscriber.git 95 | cd ad-filters-subscriber 96 | mvn clean 97 | mvn spring-boot:run 98 | ``` 99 | 100 | #### **Github Action** 101 | 102 | - Fork this project 103 | - Customize rule subscriptions 104 | - Refer to the [example configuration](./config/application-example.yaml) and modify the configuration file: `config/application.yaml` 105 | - Open the GitHub Actions page, select Update Filters on the left side, and authorize the workflow for scheduled 106 | execution (⚠ important step) 107 | - Click Run workflow or wait for automatic execution. Once completed, the corresponding rules will be generated in the 108 | directory specified in the configuration. 109 | 110 | #### **Codespaces** 111 | 112 | - Log in to `GitHub`, click the `Code` button in the upper right corner of this repository, and select and create a 113 | new `Codespace`. 114 | - Wait for `Codespaces` to start, and you can directly debug this project. 115 | 116 |

🎯 Rule Subscription

117 | 118 | > ⚠ This repository no longer provides rule subscriptions. We highly recommend forking this project to build your own 119 | > rule set. 120 | 121 | Below are rule repositories built using this project. You can find suitable rule subscriptions in them: 122 | 123 |
124 | Click to view 125 | 129 |
130 | 131 |

💬 Feedback

132 | 133 | - 👉 [issues](https://github.com/fordes123/ad-filters-subscriber/issues) 134 | -------------------------------------------------------------------------------- /config/application-example.yaml: -------------------------------------------------------------------------------- 1 | ########################################## 2 | ## !!! 示例配置，修改无效 !!! 3 | ## !!! example config, modify invalid !!! 4 | ########################################## 5 | application: 6 | 7 | # 程序配置（可选） 8 | # ！以下为默认值，如非了解不建议修改 9 | config: 10 | expected_quantity: 2000000 #预期规则数量 11 | fault_tolerance: 0.001 #容错率 12 | warn_limit: 6 #警告阈值, 原始规则长度小于该值时会输出警告日志 13 | 14 | # 域名检测，启用时将进行解析以验证域名有效性 15 | # ！开启此功能可能导致处理时间大幅延长 16 | # ！仅支持特定规则，具体原因请查看项目主页说明 17 | domain-detect: 18 | enable: false # 是否启用 19 | timeout: 2000 # 查询超时（毫秒） 20 | concurrency: 10 # 并发查询数 21 | provider: # 使用的 DNS 服务器，仅支持 IP，多个之间具备优先级; 为空则使用默认DNS 22 | - 223.5.5.5 23 | 24 | # 排除指定内容 25 | # 在解析结束后，规则内容命中此列表将直接认定为无效 26 | exclude: 27 | - "www.example.org" 28 | 29 | 30 | # 规则源配置 31 | # remote为远程规则，local为本地规则，支持多个规则源 32 | rule: 33 | #远程订阅规则 （！使用前请删除下方示例配置, 注意缩进） 34 | remote: 35 | - name: 'Subscription 1' #可选参数: 规则名称，如无将使用 path 作为名称 36 | path: 'https://example.org/rule.txt' #必要参数: 规则url，仅支持 http/https，不限定响应内容格式 37 | type: easylist #可选参数: 规则类型：easylist (默认)、dnsmasq、clash、smartdns、hosts 38 | 39 | - name: 'Subscription 2' 40 | path: 'http://example.org/rule.txt' 41 | 42 | #本地规则文件 43 | local: 44 | - name: 'local rule' 45 | path: 'local-rule.txt' #支持绝对/相对路径 46 | 47 | 48 | #输出配置 49 | output: 50 | #文件头配置，将自动作为注释添加至每个规则文件开始 51 | #可使用占位符 ${name}、${type}、${desc} 以及 ${date} (当前日期) 52 | file_header: | 53 | ADFS AdBlock ${type} 54 | Last Modified: ${date} 55 | Homepage: https://github.com/fordes123/ad-filters-subscriber/ 56 | 57 | #输出规则文件列表 （！注意缩进，且每个类型只能输出一个文件） 58 | files: 59 | - name: easylist.txt #必要参数: 文件名 60 | type: easylist #必要参数: 文件类型: easylist、dnsmasq、clash、smartdns、hosts 61 | file_header: #可选参数: 文件头配置，将自动作为注释添加至每个规则文件开始 (此处优先于 output.file_header) 62 | desc: 'ADFS EasyList' #可选参数: 文件描述，可在file_header中通过 ${} 中使用 63 | filter: 64 | - basic #基本规则，不包含任何控制、匹配符号, 可以转换为 hosts 65 | - wildcard #通配规则，仅使用通配符 66 | - unknown #其他规则，如使用了正则、高级修饰符号等，这表示目前无法支持 67 | 68 | - name: dns.txt 69 | type: easylist 70 | filter: 71 | - basic 72 | - wildcard 73 | 74 | - name: dnsmasq.txt 75 | type: dnsmasq 76 | 77 | - name: clash.yaml 78 | type: clash 79 | 80 | - name: smartdns.txt 81 | type: smartdns 82 | 83 | - name: hosts.txt 84 | type: hosts 85 | -------------------------------------------------------------------------------- /config/application.yaml: -------------------------------------------------------------------------------- 1 | # 参考 `application-example.yaml` 并在处添加订阅规则，请注意缩进 2 | # Refer to `application-example.yaml` and add subscription rules there，please note the indentation 3 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | ad-filters-subscriber 5 | org.fordes 6 | AD Filters Subscriber 7 | 8 | 4.0.0 9 | ad-filters-subscriber 10 | 1.0.0 11 | 12 | 13 | 21 14 | UTF-8 15 | UTF-8 16 | 3.4.0 17 | 6.2.0 18 | 3.1.1 19 | 3.12.1 20 | 21 | 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter 27 | ${spring-boot.version} 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-reactor-netty 33 | ${spring-boot.version} 34 | 35 | 36 | 37 | org.springframework 38 | spring-webflux 39 | ${webflux.version} 40 | 41 | 42 | 43 | org.projectlombok 44 | lombok 45 | true 46 | 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-test 51 | test 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.springframework.boot 60 | spring-boot-dependencies 61 | ${spring-boot.version} 62 | pom 63 | import 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | ${maven-compiler-plugin.version} 74 | 75 | ${java.version} 76 | ${java.version} 77 | UTF-8 78 | 79 | -parameters 80 | 81 | 82 | 83 | 84 | org.springframework.boot 85 | spring-boot-maven-plugin 86 | ${spring-boot.version} 87 | 88 | 89 | repackage 90 | 91 | repackage 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/Application.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs; 2 | 3 | import lombok.Getter; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.context.ConfigurableApplicationContext; 7 | 8 | @SpringBootApplication 9 | public class Application { 10 | 11 | @Getter 12 | private static ConfigurableApplicationContext context; 13 | 14 | public static void main(String[] args) { 15 | context = SpringApplication.run(Application.class, args); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/config/Config.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.config; 2 | 3 | import lombok.Data; 4 | import org.fordes.adfs.model.Rule; 5 | import org.fordes.adfs.util.BloomFilter; 6 | import org.springframework.boot.context.properties.ConfigurationProperties; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | import java.util.Optional; 11 | import java.util.Set; 12 | 13 | /** 14 | * @author fordes on 2024/4/9 15 | */ 16 | @Data 17 | @Configuration 18 | @ConfigurationProperties(prefix = "application.config") 19 | public class Config { 20 | 21 | private Double faultTolerance = 0.0001; 22 | private Integer expectedQuantity = 2000000; 23 | private Integer warnLimit = 6; 24 | private Set exclude; 25 | private DomainDetect domainDetect; 26 | 27 | public record DomainDetect(Boolean enable, Integer timeout) { 28 | 29 | } 30 | 31 | @Bean 32 | public BloomFilter bloomFilter() { 33 | double falsePositiveProbability = Optional.ofNullable(faultTolerance).orElse(0.0001); 34 | int expectedNumberOfElements = Optional.ofNullable(expectedQuantity).orElse(2000000); 35 | return new BloomFilter<>(falsePositiveProbability, expectedNumberOfElements); 36 | } 37 | 38 | // @Bean("producer") 39 | // public ExecutorService producerExecutor() { 40 | // return Executors.newVirtualThreadPerTaskExecutor(); 41 | // } 42 | // 43 | // @Bean("consumer") 44 | // public ExecutorService consumerExecutor() { 45 | // return Executors.newVirtualThreadPerTaskExecutor(); 46 | // } 47 | // 48 | // @Bean("singleExecutor") 49 | // public ThreadPoolTaskExecutor taskExecutor() { 50 | // ThreadPoolTaskExecutor sentinel = new ThreadPoolTaskExecutorBuilder() 51 | // .awaitTermination(false) 52 | // .corePoolSize(1) 53 | // .queueCapacity(1) 54 | // .maxPoolSize(1) 55 | // .threadNamePrefix("sentinel-") 56 | // .build(); 57 | // sentinel.initialize(); 58 | // return sentinel; 59 | // } 60 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/config/InputProperties.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.config; 2 | 3 | import lombok.Data; 4 | import org.fordes.adfs.enums.HandleType; 5 | import org.fordes.adfs.enums.RuleSet; 6 | import org.springframework.beans.factory.InitializingBean; 7 | import org.springframework.boot.context.properties.ConfigurationProperties; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Map; 11 | import java.util.Optional; 12 | import java.util.Set; 13 | import java.util.stream.Collectors; 14 | import java.util.stream.Stream; 15 | 16 | @Data 17 | @Component 18 | @ConfigurationProperties(prefix = "application.rule") 19 | public class InputProperties implements InitializingBean { 20 | 21 | private Set remote = Set.of(); 22 | private Set local = Set.of(); 23 | 24 | public Stream> stream() { 25 | return Stream.concat(local.stream().map(e -> Map.entry(HandleType.LOCAL, e)), 26 | remote.stream().map(e -> Map.entry(HandleType.REMOTE, e))); 27 | } 28 | 29 | public boolean isEmpty() { 30 | return (remote == null || remote.isEmpty()) && (local == null || local.isEmpty()); 31 | } 32 | 33 | public void setRemote(Set remote) { 34 | this.remote = Optional.ofNullable(remote) 35 | .map(e -> e.stream().filter(p -> !p.path.isEmpty()).collect(Collectors.toSet())).orElse(Set.of()); 36 | } 37 | 38 | public void setLocal(Set local) { 39 | this.local = Optional.ofNullable(local) 40 | .map(e -> e.stream().filter(p -> !p.path.isEmpty()).collect(Collectors.toSet())).orElse(Set.of()); 41 | } 42 | 43 | @Override 44 | public void afterPropertiesSet() { 45 | if ((remote == null || remote.isEmpty()) && (local == null || local.isEmpty())) { 46 | throw new IllegalArgumentException("application.rule is required"); 47 | } 48 | } 49 | 50 | public record Prop(String name, RuleSet type, String path) { 51 | 52 | public Prop(String name, RuleSet type, String path) { 53 | this.path = Optional.ofNullable(path).filter(e -> !e.isBlank()).orElseThrow(() -> new IllegalArgumentException("application.rule.path is required")).trim(); 54 | this.type = Optional.ofNullable(type).orElse(RuleSet.EASYLIST); 55 | this.name = Optional.ofNullable(name).filter(e -> !e.isBlank()).orElse(path).trim(); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object obj) { 60 | if (obj instanceof Prop prop) { 61 | return prop.path.equals(this.path); 62 | } 63 | return false; 64 | } 65 | 66 | public int hashCode() { 67 | return path.hashCode(); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/config/OutputProperties.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.config; 2 | 3 | import lombok.Data; 4 | import org.fordes.adfs.constant.Constants; 5 | import org.fordes.adfs.enums.RuleSet; 6 | import org.fordes.adfs.handler.rule.Handler; 7 | import org.fordes.adfs.model.Rule; 8 | import org.fordes.adfs.util.Util; 9 | import org.springframework.beans.factory.InitializingBean; 10 | import org.springframework.boot.context.properties.ConfigurationProperties; 11 | import org.springframework.stereotype.Component; 12 | import org.springframework.util.StringUtils; 13 | 14 | import java.time.LocalDateTime; 15 | import java.time.format.DateTimeFormatter; 16 | import java.util.Optional; 17 | import java.util.Set; 18 | 19 | import static org.fordes.adfs.constant.Constants.*; 20 | 21 | /** 22 | * 输出配置 23 | * 24 | * @author fordes123 on 2022/9/19 25 | */ 26 | @Data 27 | @Component 28 | @ConfigurationProperties(prefix = "application.output") 29 | public class OutputProperties implements InitializingBean { 30 | 31 | private String fileHeader; 32 | private String path = "rule"; 33 | private Set files; 34 | 35 | public record OutputFile(String name, RuleSet type, Set filter, String desc, String fileHeader) { 36 | 37 | public OutputFile(String name, RuleSet type, Set filter, String desc, String fileHeader) { 38 | this.name = Optional.ofNullable(name).filter(StringUtils::hasText).orElseThrow(() -> new IllegalArgumentException("application.output.files.name is required")); 39 | this.type = Optional.ofNullable(type).orElseThrow(() -> new IllegalArgumentException("application.output.files.type is required")); 40 | this.desc = Optional.ofNullable(desc).filter(StringUtils::hasText).orElse(Constants.EMPTY); 41 | this.filter = Optional.ofNullable(filter).orElse(Set.of(Rule.Type.values())); 42 | this.fileHeader = Optional.ofNullable(fileHeader).filter(StringUtils::hasText).orElse(null); 43 | } 44 | 45 | public String displayHeader(Handler handler, String parentHeader) { 46 | StringBuilder builder = new StringBuilder(); 47 | //文件头 48 | Optional.ofNullable(Optional.ofNullable(this.fileHeader()).orElse(parentHeader)) 49 | .filter(StringUtils::hasText) 50 | .ifPresent(e -> { 51 | String header = handler.commented(e 52 | .replace(HEADER_DATE, LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))) 53 | .replace(HEADER_NAME, this.name()) 54 | .replace(HEADER_DESC, this.desc()) 55 | .replace(HEADER_TYPE, this.type().name().toLowerCase())); 56 | builder.append(header).append(System.lineSeparator()); 57 | }); 58 | 59 | //格式头 60 | Optional.ofNullable(handler.headFormat()).filter(StringUtils::hasText) 61 | .ifPresent(e -> builder.append(e).append(System.lineSeparator())); 62 | 63 | return builder.toString(); 64 | } 65 | } 66 | 67 | @Override 68 | public void afterPropertiesSet() { 69 | this.files = Optional.ofNullable(files) 70 | .filter(e -> !e.isEmpty()) 71 | .orElseThrow(() -> new IllegalArgumentException("application.output.files is required")); 72 | 73 | this.path = Optional.ofNullable(path).filter(StringUtils::hasText) 74 | .map(Util::normalizePath) 75 | .orElseThrow(() -> new IllegalArgumentException("application.output.path is required")); 76 | } 77 | 78 | public boolean isEmpty() { 79 | return files == null || files.isEmpty(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/constant/Constants.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.constant; 2 | 3 | import java.io.File; 4 | import java.util.Set; 5 | 6 | public class Constants { 7 | 8 | public static final String ROOT_PATH = System.getProperty("user.dir"); 9 | public static final String FILE_SEPARATOR = File.separator; 10 | 11 | public static final String HEADER_DATE = "${date}"; 12 | public static final String HEADER_NAME = "${name}"; 13 | public static final String HEADER_TOTAL = "${total}"; 14 | public static final String HEADER_TYPE = "${type}"; 15 | public static final String HEADER_DESC = "${desc}"; 16 | 17 | public static final String EMPTY = ""; 18 | public static final String DOT = "."; 19 | public static final String EXCLAMATION = "!"; 20 | public static final String HASH = "#"; 21 | public static final String AT = "@"; 22 | public static final String PERCENT = "%"; 23 | public static final String DOLLAR = "$"; 24 | public static final String UNDERLINE = "_"; 25 | public static final String DASH = "-"; 26 | public static final String TILDE = "~"; 27 | public static final String COMMA = ","; 28 | public static final String SLASH = "/"; 29 | public static final String LEFT_BRACKETS = "["; 30 | public static final String RIGHT_BRACKETS = "]"; 31 | public static final String OR = "||"; 32 | public static final String ASTERISK = "*"; 33 | public static final String QUESTION_MARK = "?"; 34 | public static final String A = "a"; 35 | public static final String CARET = "^"; 36 | public static final String WHITESPACE = " "; 37 | public static final String CR = "\r"; 38 | public static final String LF = "\n"; 39 | public static final String CRLF = CR + LF; 40 | public static final String QUOTE = "\""; 41 | public static final String SINGLE_QUOTE = "'"; 42 | public static final String ADD = "+"; 43 | public static final String COLON = ":"; 44 | public static final String EQUAL = "="; 45 | 46 | 47 | public static final Set LOCAL_IP = Set.of("0.0.0.0", "127.0.0.1", "::1"); 48 | public static final Set LOCAL_DOMAIN = Set.of("localhost", "localhost.localdomain", "local", "ip6-localhost", "ip6-loopback"); 49 | public static final String LOCAL_V4 = "127.0.0.1"; 50 | public static final String UNKNOWN_IP = "0.0.0.0"; 51 | public static final String LOCAL_V6 = "::1"; 52 | public static final String LOCALHOST = "localhost"; 53 | public static final String DOUBLE_AT = "@@"; 54 | public static final String IMPORTANT = "important"; 55 | public static final String DOMAIN = "domain"; 56 | public static final String TAB = "\t"; 57 | public static final String PAYLOAD = "payload"; 58 | 59 | public static final String DNSMASQ_HEADER = "address=/"; 60 | public static final String SMARTDNS_HEADER = "address /"; 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/constant/RegConstants.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.constant; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | /** 6 | * @author fordes on 2024/4/9 7 | */ 8 | public class RegConstants { 9 | 10 | public static final Pattern PATTERN_PATH_ABSOLUTE = Pattern.compile("^[a-zA-Z]:([/\\\\].*)?"); 11 | public static Pattern PATTERN_IP = Pattern.compile("((?:(?:25[0-5]|2[0-4]\\d|[01]?\\d?\\d)\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d?\\d))"); 12 | public static Pattern PATTERN_DOMAIN = Pattern.compile("(?=^.{3,255}$)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+$"); 13 | public static Pattern DOMAIN_PART = Pattern.compile("^([-a-zA-Z0-9]{0,62})+$"); 14 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/enums/HandleType.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.enums; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | public enum HandleType { 7 | 8 | LOCAL, 9 | 10 | REMOTE, 11 | 12 | ; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/enums/RuleSet.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.enums; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | import java.util.stream.Stream; 7 | 8 | @Getter 9 | @AllArgsConstructor 10 | public enum RuleSet { 11 | 12 | EASYLIST, 13 | DNSMASQ, 14 | CLASH, 15 | SMARTDNS, 16 | HOSTS, 17 | ; 18 | 19 | public static RuleSet of(String name) { 20 | return Stream.of(values()) 21 | .filter(v -> v.name().equalsIgnoreCase(name)) 22 | .findFirst() 23 | .orElseThrow(() -> new IllegalArgumentException("unknown format: " + name)); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/Parser.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.fordes.adfs.config.Config; 6 | import org.fordes.adfs.config.InputProperties; 7 | import org.fordes.adfs.enums.HandleType; 8 | import org.fordes.adfs.handler.dns.DnsChecker; 9 | import org.fordes.adfs.handler.fetch.Fetcher; 10 | import org.fordes.adfs.handler.rule.Handler; 11 | import org.fordes.adfs.model.Rule; 12 | import org.fordes.adfs.util.BloomFilter; 13 | import org.springframework.stereotype.Component; 14 | import org.springframework.util.StringUtils; 15 | import reactor.core.publisher.Flux; 16 | import reactor.core.publisher.Mono; 17 | 18 | import java.util.Optional; 19 | import java.util.Set; 20 | import java.util.concurrent.atomic.AtomicLong; 21 | 22 | @Slf4j 23 | @Component 24 | @AllArgsConstructor 25 | public class Parser { 26 | 27 | protected final BloomFilter filter; 28 | protected Config config; 29 | protected DnsChecker dnsChecker; 30 | 31 | public Flux handle(InputProperties.Prop prop, HandleType type) { 32 | 33 | AtomicLong invalid = new AtomicLong(0L); 34 | AtomicLong repeat = new AtomicLong(0L); 35 | AtomicLong effective = new AtomicLong(0L); 36 | Set exclude = Optional.ofNullable(config.getExclude()).orElseGet(Set::of); 37 | 38 | return Flux.just(prop) 39 | .flatMap(p -> { 40 | Fetcher fetcher = Fetcher.getFetcher(type); 41 | return fetcher.fetch(p.path()); 42 | }) 43 | .filter(StringUtils::hasText) 44 | .flatMap(line -> { 45 | Handler handler = Handler.getHandler(prop.type()); 46 | 47 | if (handler.isComment(line)) { 48 | return Mono.empty(); 49 | } 50 | 51 | Rule rule = handler.parse(line); 52 | if (Rule.EMPTY.equals(rule)) { 53 | invalid.incrementAndGet(); 54 | log.debug("parse fail: {}", line); 55 | return Mono.empty(); 56 | } 57 | 58 | return Mono.just(rule); 59 | }) 60 | .flatMap(e -> { 61 | 62 | if (e.getTarget() != null && exclude.contains(e.getTarget())) { 63 | log.info("exclude rule: {}", e.getOrigin()); 64 | return Mono.empty(); 65 | } 66 | 67 | if (filter.contains(e)) { 68 | log.debug("already exists rule: {}", e.getOrigin()); 69 | repeat.incrementAndGet(); 70 | return Mono.empty(); 71 | } 72 | 73 | if (e.getOrigin().length() <= config.getWarnLimit()) { 74 | log.warn("[{}] Suspicious rule => {}", prop.name(), e.getOrigin()); 75 | } 76 | 77 | return Mono.just(e); 78 | 79 | }) 80 | .onErrorResume(ex -> { 81 | log.error(ex.getMessage(), ex); 82 | return Mono.empty(); 83 | }) 84 | .flatMap(rule -> { 85 | 86 | /** 87 | * 假设有规则 ||example.org^ 88 | * 通过DNS查询 example.org 是否存在 A/AAAA/CNAME 记录作为判断依据 89 | * 不可避免的误判是，example.org 没有有效记录，而其存在有效子域如 test.example.org 90 | */ 91 | if (dnsChecker.getConfig().enable() && Rule.Type.BASIC.equals(rule.getType()) 92 | && Rule.Scope.DOMAIN.equals(rule.getScope())) { 93 | 94 | return Flux.just(rule.getTarget()) 95 | .flatMap(e -> dnsChecker.lookup(e), 1) 96 | .flatMap(e -> { 97 | if (!e) { 98 | log.debug("[{}] dns check invalid rule => {}", prop.name(), rule.getOrigin()); 99 | invalid.incrementAndGet(); 100 | return Mono.empty(); 101 | } 102 | return Mono.just(rule); 103 | }); 104 | } 105 | return Mono.just(rule); 106 | }, dnsChecker.getConfig().concurrency()) 107 | .flatMap(e -> { 108 | filter.add(e); 109 | effective.incrementAndGet(); 110 | return Mono.just(e); 111 | 112 | }) 113 | .doFinally(signal -> { 114 | log.info("[{}] parser done => invalid: {}, repeat: {}, effective: {}", prop.name(), 115 | invalid.get(), repeat.get(), effective.get()); 116 | }); 117 | 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/dns/DnsChecker.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.dns; 2 | 3 | import io.netty.channel.EventLoop; 4 | import io.netty.channel.nio.NioEventLoopGroup; 5 | import io.netty.channel.socket.nio.NioDatagramChannel; 6 | import io.netty.channel.socket.nio.NioSocketChannel; 7 | import io.netty.resolver.ResolvedAddressTypes; 8 | import io.netty.resolver.dns.*; 9 | import io.netty.util.concurrent.Future; 10 | import lombok.Data; 11 | import lombok.extern.slf4j.Slf4j; 12 | import org.fordes.adfs.constant.Constants; 13 | import org.springframework.boot.context.properties.ConfigurationProperties; 14 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 15 | import org.springframework.stereotype.Component; 16 | import reactor.core.publisher.Mono; 17 | 18 | import java.net.InetAddress; 19 | import java.net.InetSocketAddress; 20 | import java.net.UnknownHostException; 21 | import java.util.List; 22 | import java.util.Optional; 23 | import java.util.concurrent.ArrayBlockingQueue; 24 | import java.util.stream.IntStream; 25 | 26 | @Data 27 | @Slf4j 28 | @Component 29 | @EnableConfigurationProperties(DnsChecker.Config.class) 30 | public class DnsChecker { 31 | 32 | private final Config config; 33 | private final NioEventLoopGroup eventLoopGroup; 34 | private final DnsServerAddressStreamProvider provider; 35 | private final ArrayBlockingQueue resolvers; 36 | 37 | public DnsChecker(Config config) { 38 | this.config = config; 39 | if (config.enable) { 40 | this.eventLoopGroup = new NioEventLoopGroup(config.concurrency); 41 | this.resolvers = new ArrayBlockingQueue<>(config.concurrency); 42 | if (config.provider.isEmpty()) { 43 | this.provider = DnsServerAddressStreamProviders.platformDefault(); 44 | } else { 45 | InetSocketAddress[] array = config.provider.stream().map(e -> { 46 | String[] split = e.split(Constants.COLON); 47 | return new InetSocketAddress(split[0], split.length > 1 ? Integer.parseInt(split[1]) : 53); 48 | }).toArray(InetSocketAddress[]::new); 49 | 50 | this.provider = new SequentialDnsServerAddressStreamProvider(array); 51 | } 52 | 53 | final DnsCache cache = new DefaultDnsCache(5, 60, 60); 54 | final DnsCnameCache cnameCache = new DefaultDnsCnameCache(5, 60); 55 | IntStream.range(0, config.concurrency) 56 | .forEach(i -> { 57 | EventLoop eventLoop = this.eventLoopGroup.next(); 58 | DnsNameResolver resolver = new DnsNameResolverBuilder(eventLoop) 59 | .datagramChannelFactory(NioDatagramChannel::new) 60 | .socketChannelFactory(NioSocketChannel::new) 61 | .nameServerProvider(this.provider) 62 | .queryTimeoutMillis(config.timeout) 63 | .maxQueriesPerResolve(1) 64 | .resolvedAddressTypes(ResolvedAddressTypes.IPV4_PREFERRED) 65 | .resolveCache(cache) 66 | .cnameCache(cnameCache) 67 | .build(); 68 | resolvers.add(resolver); 69 | }); 70 | 71 | } else { 72 | log.warn("dns check is disabled"); 73 | this.resolvers = null; 74 | this.eventLoopGroup = null; 75 | this.provider = null; 76 | } 77 | } 78 | 79 | @ConfigurationProperties(prefix = "application.config.domain-detect") 80 | public record Config(Boolean enable, Integer timeout, Integer concurrency, List provider) { 81 | 82 | public Config(Boolean enable, Integer timeout, Integer concurrency, List provider) { 83 | this.enable = Optional.ofNullable(enable).orElse(Boolean.TRUE); 84 | this.timeout = Optional.ofNullable(timeout).orElse(1000); 85 | this.concurrency = Optional.ofNullable(concurrency).orElse(4); 86 | this.provider = Optional.ofNullable(provider).filter(e -> !e.isEmpty()).orElse(List.of()); 87 | } 88 | } 89 | 90 | public Mono lookup(String domain) { 91 | return Mono.create(sink -> { 92 | if (resolvers == null) { 93 | sink.success(true); 94 | return; 95 | } 96 | 97 | DnsNameResolver resolver; 98 | try { 99 | resolver = resolvers.take(); 100 | } catch (InterruptedException e) { 101 | sink.success(true); 102 | log.error("dns resolve interrupted", e); 103 | return; 104 | } 105 | 106 | Future> future = resolver.resolveAll(domain); 107 | future.addListener(result -> { 108 | 109 | boolean res = true; 110 | if (!result.isSuccess()) { 111 | Throwable cause = result.cause(); 112 | 113 | if (cause instanceof UnknownHostException) { 114 | res = false; 115 | } else { 116 | log.warn("dns check failed: {} => {}", domain, cause.getMessage()); 117 | } 118 | } 119 | sink.success(res); 120 | resolvers.offer(resolver); 121 | log.debug("dns check done, available: {}", resolvers.size()); 122 | }); 123 | }); 124 | } 125 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/fetch/Fetcher.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.fetch; 2 | 3 | import jakarta.annotation.Nonnull; 4 | import org.fordes.adfs.enums.HandleType; 5 | import org.springframework.core.io.buffer.DataBuffer; 6 | import org.springframework.core.io.buffer.DataBufferUtils; 7 | import reactor.core.publisher.Flux; 8 | 9 | import java.nio.charset.Charset; 10 | 11 | public abstract class Fetcher { 12 | 13 | protected @Nonnull Charset charset() { 14 | return Charset.defaultCharset(); 15 | } 16 | 17 | public abstract Flux fetch(@Nonnull String path); 18 | 19 | protected Flux fetch(Flux buffers) { 20 | final StringBuilder lineBuffer = new StringBuilder(); 21 | return buffers.flatMap(data -> { 22 | String chunk = data.toString(this.charset()); 23 | DataBufferUtils.release(data); 24 | 25 | lineBuffer.append(chunk); 26 | String full = lineBuffer.toString(); 27 | 28 | String[] lines = full.split("\\r?\\n", -1); 29 | int len = lines.length; 30 | 31 | lineBuffer.setLength(0); 32 | 33 | if (!full.endsWith("\n") && !full.endsWith("\r")) { 34 | lineBuffer.append(lines[len - 1]); 35 | len--; 36 | } 37 | 38 | return Flux.fromArray(lines).take(len); 39 | }) 40 | .concatWith(Flux.defer(() -> { 41 | if (lineBuffer.length() > 0) { 42 | return Flux.just(lineBuffer.toString()); 43 | } else { 44 | return Flux.empty(); 45 | } 46 | })); 47 | } 48 | 49 | public final static Fetcher getFetcher(HandleType type) { 50 | switch (type) { 51 | case LOCAL: 52 | return new LocalFetcher(); 53 | case REMOTE: 54 | return new HttpFetcher(); 55 | } 56 | throw new IllegalArgumentException("unsupported handle type: " + type); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/fetch/HttpFetcher.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.fetch; 2 | 3 | import io.netty.channel.ChannelOption; 4 | import io.netty.handler.timeout.ReadTimeoutHandler; 5 | import io.netty.handler.timeout.WriteTimeoutHandler; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.core.io.buffer.DataBuffer; 8 | import org.springframework.http.HttpHeaders; 9 | import org.springframework.http.client.reactive.ReactorClientHttpConnector; 10 | import org.springframework.web.reactive.function.client.ExchangeStrategies; 11 | import org.springframework.web.reactive.function.client.WebClient; 12 | import reactor.core.publisher.Flux; 13 | import reactor.netty.http.client.HttpClient; 14 | 15 | import java.net.URI; 16 | import java.nio.charset.Charset; 17 | import java.nio.charset.StandardCharsets; 18 | import java.time.Duration; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | @Slf4j 22 | public class HttpFetcher extends Fetcher { 23 | 24 | private final WebClient webClient; 25 | private Integer connectTimeout = 10_000; 26 | private Integer readTimeout = 30_000; 27 | private Integer writeTimeout = 30_000; 28 | private Integer bufferSize = 4096; 29 | 30 | public HttpFetcher() { 31 | 32 | HttpClient httpClient = HttpClient.create() 33 | .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, this.connectTimeout) // 连接超时 10 秒 34 | .responseTimeout(Duration.ofSeconds(30)) // 响应超时 30 秒 35 | .doOnConnected(conn -> conn 36 | .addHandlerLast(new ReadTimeoutHandler(this.readTimeout, TimeUnit.MILLISECONDS)) // 读超时 37 | .addHandlerLast(new WriteTimeoutHandler(this.writeTimeout, TimeUnit.MILLISECONDS)) // 写超时 38 | ); 39 | 40 | ExchangeStrategies strategies = ExchangeStrategies.builder() 41 | .codecs(configurer -> configurer 42 | .defaultCodecs() 43 | .maxInMemorySize(this.bufferSize) 44 | ) 45 | .build(); 46 | 47 | this.webClient = WebClient.builder() 48 | .clientConnector(new ReactorClientHttpConnector(httpClient)) 49 | .exchangeStrategies(strategies) 50 | .defaultHeader(HttpHeaders.CONNECTION, "keep-alive") 51 | // .defaultHeader(HttpHeaders.ACCEPT_CHARSET, this.charset().displayName()) 52 | // .defaultHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br") 53 | .build(); 54 | } 55 | 56 | public HttpFetcher(Integer connectTimeout, Integer readTimeout, Integer writeTimeout, Integer bufferSize) { 57 | this(); 58 | this.connectTimeout = connectTimeout; 59 | this.readTimeout = readTimeout; 60 | this.writeTimeout = writeTimeout; 61 | this.bufferSize = bufferSize; 62 | } 63 | 64 | @Override 65 | public Flux fetch(String path) { 66 | Flux data = webClient.get() 67 | .uri(URI.create(path)) 68 | .retrieve() 69 | .bodyToFlux(DataBuffer.class) 70 | .onErrorResume(e -> { 71 | log.error("http rule => {}, fetch failed => {}", path, e.getMessage(), e); 72 | return Flux.empty(); 73 | }); 74 | 75 | return this.fetch(data); 76 | } 77 | 78 | @Override 79 | protected Charset charset() { 80 | return StandardCharsets.UTF_8; 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/fetch/LocalFetcher.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.fetch; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.fordes.adfs.util.Util; 5 | import org.springframework.core.io.buffer.DataBuffer; 6 | import org.springframework.core.io.buffer.DataBufferUtils; 7 | import org.springframework.core.io.buffer.DefaultDataBufferFactory; 8 | import reactor.core.publisher.Flux; 9 | 10 | import java.nio.charset.Charset; 11 | import java.nio.file.Path; 12 | 13 | @Slf4j 14 | public class LocalFetcher extends Fetcher { 15 | 16 | private int bufferSize = 4096; 17 | 18 | public LocalFetcher() { 19 | super(); 20 | } 21 | 22 | public LocalFetcher(int bufferSize) { 23 | super(); 24 | this.bufferSize = bufferSize; 25 | } 26 | 27 | @Override 28 | public Flux fetch(String path) { 29 | 30 | Flux data = Flux.just(path) 31 | .map(Util::normalizePath) 32 | .map(Path::of) 33 | .flatMap(p -> DataBufferUtils.read(p, new DefaultDataBufferFactory(), this.bufferSize)) 34 | .onErrorResume(e -> { 35 | log.error("local rule => {}, read failed => {}", path, e.getMessage(), e); 36 | return Flux.empty(); 37 | }); 38 | 39 | return this.fetch(data); 40 | } 41 | 42 | @Override 43 | protected Charset charset() { 44 | return super.charset(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/ClashHandler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.fordes.adfs.enums.RuleSet; 5 | import org.fordes.adfs.model.Rule; 6 | import org.fordes.adfs.util.Util; 7 | import org.springframework.beans.factory.InitializingBean; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Optional; 11 | import java.util.Set; 12 | import java.util.stream.Collectors; 13 | 14 | import static org.fordes.adfs.constant.Constants.*; 15 | import static org.fordes.adfs.constant.RegConstants.PATTERN_DOMAIN; 16 | 17 | /** 18 | * @author fordes123 on 2024/5/27 19 | */ 20 | @Slf4j 21 | @Component 22 | public final class ClashHandler extends Handler implements InitializingBean { 23 | 24 | @Override 25 | public Rule parse(String line) { 26 | //跳过文件头 27 | if (line.startsWith(PAYLOAD)) { 28 | return Rule.EMPTY; 29 | } 30 | 31 | Rule rule = new Rule(); 32 | rule.setOrigin(line); 33 | rule.setSource(RuleSet.EASYLIST); 34 | 35 | //只匹配 domain 规则，ipcidr、classical 规则暂不支持 36 | String content = (line.startsWith(DASH) ? line.substring(DASH.length()) : line).trim(); 37 | if (content.startsWith(SINGLE_QUOTE)) { 38 | content = Util.subBetween(line, SINGLE_QUOTE, SINGLE_QUOTE).trim(); 39 | } else if (content.startsWith(QUOTE)) { 40 | content = Util.subBetween(line, QUOTE, QUOTE).trim(); 41 | } 42 | 43 | //通配符 * 一次只能匹配一级域名，无法转换为easylist 44 | if (content.startsWith(ASTERISK)) { 45 | rule.setType(Rule.Type.UNKNOWN); 46 | return rule; 47 | } 48 | 49 | //通配符 + 50 | if (content.startsWith(ADD)) { 51 | content = content.substring(content.startsWith("+.") ? 2 : 1); 52 | rule.setControls(Set.of(Rule.Control.OVERLAY)); 53 | } 54 | 55 | //判断是否是domain 56 | boolean haveAsterisk = content.contains(ASTERISK); 57 | String temp = haveAsterisk ? content.replace(ASTERISK, A) : content; 58 | if (PATTERN_DOMAIN.matcher(temp).matches()) { 59 | rule.setType(haveAsterisk ? Rule.Type.WILDCARD : Rule.Type.BASIC); 60 | } 61 | 62 | rule.setTarget(content); 63 | rule.setDest(UNKNOWN_IP); 64 | rule.setMode(Rule.Mode.DENY); 65 | rule.setScope(Rule.Scope.DOMAIN); 66 | if (rule.getType() == null) { 67 | rule.setType(Rule.Type.UNKNOWN); 68 | } 69 | return rule; 70 | } 71 | 72 | @Override 73 | public String format(Rule rule) { 74 | if (Rule.Type.UNKNOWN == rule.getType()) { 75 | if (RuleSet.CLASH == rule.getSource()) { 76 | return rule.getOrigin(); 77 | } 78 | return null; 79 | } else if (rule.getMode() == Rule.Mode.DENY && rule.getScope() == Rule.Scope.DOMAIN) { 80 | StringBuilder builder = new StringBuilder(); 81 | builder.append(WHITESPACE).append(WHITESPACE).append(DASH).append(WHITESPACE).append(QUOTE); 82 | 83 | Set controls = Optional.ofNullable(rule.getControls()).orElse(Set.of()); 84 | if (controls.contains(Rule.Control.OVERLAY)) { 85 | builder.append(ADD).append(DOT); 86 | } 87 | builder.append(rule.getTarget()); 88 | builder.append(QUOTE); 89 | return builder.toString(); 90 | } 91 | return null; 92 | } 93 | 94 | @Override 95 | public String headFormat() { 96 | return PAYLOAD + COLON; 97 | } 98 | 99 | @Override 100 | public boolean isComment(String line) { 101 | return line.startsWith(HASH); 102 | } 103 | 104 | @Override 105 | public String commented(String value) { 106 | return Util.splitIgnoreBlank(value, LF).stream() 107 | .map(e -> HASH + e.trim()) 108 | .collect(Collectors.joining(CRLF)); 109 | } 110 | 111 | @Override 112 | public void afterPropertiesSet() { 113 | this.register(RuleSet.CLASH, this); 114 | } 115 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/DnsmasqHandler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import org.fordes.adfs.enums.RuleSet; 4 | import org.fordes.adfs.model.Rule; 5 | import org.fordes.adfs.util.Util; 6 | import org.springframework.beans.factory.InitializingBean; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | import static org.fordes.adfs.constant.Constants.*; 13 | 14 | /** 15 | * @author fordes123 on 2024/5/27 16 | */ 17 | @Component 18 | public final class DnsmasqHandler extends Handler implements InitializingBean { 19 | 20 | @Override 21 | public Rule parse(String line) { 22 | 23 | String content = Util.subAfter(line, DNSMASQ_HEADER, true); 24 | List data = Util.splitIgnoreBlank(content, SLASH); 25 | if (data.size() == 1 || data.size() == 2) { 26 | String domain = data.getFirst(); 27 | String ip = data.size() > 1 ? data.get(1) : null; 28 | 29 | Rule rule = new Rule(); 30 | rule.setSource(RuleSet.DNSMASQ); 31 | rule.setOrigin(line); 32 | rule.setTarget(domain); 33 | rule.setDest(ip); 34 | rule.setScope(Rule.Scope.DOMAIN); 35 | rule.setType(Rule.Type.BASIC); 36 | rule.setMode((ip == null || LOCAL_IP.contains(ip)) ? Rule.Mode.DENY : Rule.Mode.REWRITE); 37 | return rule; 38 | } 39 | return Rule.EMPTY; 40 | } 41 | 42 | @Override 43 | public String format(Rule rule) { 44 | if (Rule.Scope.DOMAIN == rule.getScope() && 45 | Rule.Type.BASIC == rule.getType() && 46 | Rule.Mode.ALLOW != rule.getMode()) { 47 | 48 | StringBuilder builder = new StringBuilder(); 49 | builder.append(DNSMASQ_HEADER) 50 | .append(rule.getTarget()); 51 | if (Rule.Mode.REWRITE.equals(rule.getMode())) { 52 | builder.append(SLASH) 53 | .append(rule.getDest()); 54 | } 55 | builder.append(SLASH); 56 | return builder.toString(); 57 | } 58 | return null; 59 | } 60 | 61 | @Override 62 | public String commented(String value) { 63 | return Util.splitIgnoreBlank(value, LF).stream() 64 | .map(e -> HASH + WHITESPACE + e.trim()) 65 | .collect(Collectors.joining(CRLF)); 66 | } 67 | 68 | @Override 69 | public boolean isComment(String line) { 70 | return line.startsWith(HASH); 71 | } 72 | 73 | @Override 74 | public void afterPropertiesSet() { 75 | this.register(RuleSet.DNSMASQ, this); 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/EasylistHandler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.fordes.adfs.enums.RuleSet; 5 | import org.fordes.adfs.model.Rule; 6 | import org.fordes.adfs.util.Util; 7 | import org.springframework.beans.factory.InitializingBean; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Arrays; 11 | import java.util.Map; 12 | import java.util.Optional; 13 | import java.util.stream.Collectors; 14 | 15 | import static org.fordes.adfs.constant.Constants.*; 16 | 17 | /** 18 | * @author fordes123 on 2024/5/27 19 | */ 20 | @Slf4j 21 | @Component 22 | public final class EasylistHandler extends Handler implements InitializingBean { 23 | 24 | @Override 25 | public Rule parse(String line) { 26 | Rule rule = new Rule(); 27 | rule.setOrigin(line); 28 | rule.setSource(RuleSet.EASYLIST); 29 | rule.setMode(Rule.Mode.DENY); 30 | 31 | if (line.startsWith(DOUBLE_AT)) { 32 | rule.setMode(Rule.Mode.ALLOW); 33 | line = line.substring(2); 34 | } 35 | 36 | int _head = 0; 37 | if (line.startsWith(OR)) { 38 | _head = OR.length(); 39 | rule.getControls().add(Rule.Control.OVERLAY); 40 | } 41 | 42 | 43 | //修饰部分 44 | int _tail = line.indexOf(CARET); 45 | if (_tail > 0) { 46 | rule.getControls().add(Rule.Control.QUALIFIER); 47 | 48 | String modify = line.substring(_tail + 1); 49 | if (!modify.isEmpty()) { 50 | modify = modify.startsWith(DOLLAR) ? modify.substring(1) : modify; 51 | String[] array = modify.split(COMMA); 52 | if (Arrays.stream(array).allMatch(IMPORTANT::equals)) { 53 | rule.getControls().add(Rule.Control.IMPORTANT); 54 | } else { 55 | rule.setType(Rule.Type.UNKNOWN); 56 | return rule; 57 | } 58 | } 59 | } else if (line.endsWith(DOLLAR + IMPORTANT)) { 60 | rule.getControls().add(Rule.Control.IMPORTANT); 61 | _tail = line.length() - (DOLLAR.length() + IMPORTANT.length()); 62 | } 63 | 64 | 65 | //内容部分 66 | String content = line.substring(_head, _tail > 0 ? _tail : line.length()); 67 | 68 | if (content.startsWith(SLASH) && content.endsWith(SLASH)) { 69 | content = content.substring(1, content.length() - 1); 70 | rule.setType(Rule.Type.UNKNOWN); 71 | } 72 | 73 | //判断是否为基本或通配规则 74 | Util.isBaseRule(content, (origin, e) -> { 75 | if (rule.getType() == null) { 76 | rule.setType(e); 77 | } 78 | rule.setScope(Rule.Scope.DOMAIN); 79 | rule.setTarget(origin); 80 | if (Rule.Mode.DENY.equals(rule.getMode())) { 81 | rule.setDest(UNKNOWN_IP); 82 | } 83 | }, e -> { 84 | Map.Entry entry = Util.parseHosts(e); 85 | if (entry != null) { 86 | rule.setSource(RuleSet.HOSTS); 87 | rule.setTarget(entry.getValue()); 88 | rule.setMode(LOCAL_IP.contains(entry.getKey()) && !LOCAL_DOMAIN.contains(entry.getValue()) ? Rule.Mode.DENY : Rule.Mode.REWRITE); 89 | rule.setDest(Rule.Mode.DENY == rule.getMode() ? UNKNOWN_IP : entry.getKey()); 90 | rule.setScope(Rule.Scope.DOMAIN); 91 | rule.setType(Rule.Type.BASIC); 92 | } else { 93 | rule.setType(Rule.Type.UNKNOWN); 94 | } 95 | }); 96 | return rule; 97 | } 98 | 99 | @Override 100 | public String format(Rule rule) { 101 | if (Rule.Type.UNKNOWN != rule.getType() && Rule.Mode.REWRITE != rule.getMode()) { 102 | 103 | StringBuilder builder = new StringBuilder(); 104 | Optional.of(rule.getMode()) 105 | .filter(Rule.Mode.ALLOW::equals) 106 | .ifPresent(m -> builder.append(DOUBLE_AT)); 107 | 108 | Optional.of(rule.getControls()) 109 | .filter(e -> e.contains(Rule.Control.OVERLAY)) 110 | .ifPresent(c -> builder.append(OR)); 111 | 112 | builder.append(rule.getTarget()); 113 | 114 | Optional.of(rule.getControls()) 115 | .filter(e -> e.contains(Rule.Control.QUALIFIER)) 116 | .ifPresent(c -> builder.append(CARET)); 117 | 118 | Optional.of(rule.getControls()) 119 | .filter(e -> e.contains(Rule.Control.IMPORTANT)) 120 | .ifPresent(c -> builder.append(DOLLAR).append(IMPORTANT)); 121 | return builder.toString(); 122 | } 123 | 124 | //同源未知规则可直接写出 125 | if (Rule.Type.UNKNOWN == rule.getType() && RuleSet.EASYLIST == rule.getSource()) { 126 | return rule.getOrigin(); 127 | } 128 | return null; 129 | } 130 | 131 | @Override 132 | public String commented(String value) { 133 | return Util.splitIgnoreBlank(value, LF).stream() 134 | .map(e -> EXCLAMATION + WHITESPACE + e.trim()) 135 | .collect(Collectors.joining(CRLF)); 136 | } 137 | 138 | @Override 139 | public void afterPropertiesSet() { 140 | this.register(RuleSet.EASYLIST, this); 141 | } 142 | 143 | @Override 144 | public boolean isComment(String line) { 145 | return Util.startWithAny(line, HASH, EXCLAMATION) || Util.between(line, LEFT_BRACKETS, RIGHT_BRACKETS); 146 | } 147 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/Handler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import jakarta.annotation.Nonnull; 4 | import jakarta.annotation.Nullable; 5 | import org.fordes.adfs.constant.Constants; 6 | import org.fordes.adfs.enums.RuleSet; 7 | import org.fordes.adfs.model.Rule; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | public abstract sealed class Handler permits EasylistHandler, DnsmasqHandler, ClashHandler, 13 | SmartdnsHandler, HostsHandler { 14 | 15 | private static final Map handlerMap = new HashMap<>(RuleSet.values().length, 1); 16 | 17 | /** 18 | * 解析规则
19 | * 返回 null 即表示解析失败 20 | * 21 | * @param line 规则文本 22 | * @return {@link Rule} 23 | */ 24 | public abstract @Nonnull Rule parse(String line); 25 | 26 | /** 27 | * 转换规则
28 | * 29 | * @param rule {@link Rule} null 表示无法转换或失败 30 | * @return 规则文本 31 | */ 32 | public abstract @Nullable String format(Rule rule); 33 | 34 | /** 35 | * 生成注释 36 | * @param value 目标内容 37 | * @return 注释 38 | */ 39 | public abstract String commented(String value); 40 | 41 | /** 42 | * 某些规则格式拥有固定的头部内容，可实现此方法以返回 43 | */ 44 | public String headFormat() { 45 | return Constants.EMPTY; 46 | } 47 | 48 | /** 49 | * 某些规则格式拥有固定的尾部内容，可实现此方法以返回 50 | */ 51 | public String tailFormat() { 52 | return Constants.EMPTY; 53 | } 54 | 55 | /** 56 | * 验证规则文本是否为注释
57 | * 并不强制子类实现此方法，且不是注释不表示此规则有效 58 | * 59 | * @param line 规则文本 60 | * @return 默认 false 61 | */ 62 | public boolean isComment(String line) { 63 | return false; 64 | } 65 | 66 | /** 67 | * 根据 RuleSet 获取 Handler 68 | * 69 | * @param type {@link RuleSet} 70 | * @return {@link Handler} 71 | */ 72 | public static Handler getHandler(RuleSet type) { 73 | return handlerMap.get(type); 74 | } 75 | 76 | protected void register(RuleSet type, Handler handler) { 77 | handlerMap.put(type, handler); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/HostsHandler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.fordes.adfs.enums.RuleSet; 5 | import org.fordes.adfs.model.Rule; 6 | import org.fordes.adfs.util.Util; 7 | import org.springframework.beans.factory.InitializingBean; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Map; 11 | import java.util.Objects; 12 | import java.util.Optional; 13 | import java.util.stream.Collectors; 14 | 15 | import static org.fordes.adfs.constant.Constants.*; 16 | 17 | /** 18 | * @author fordes123 on 2024/5/27 19 | */ 20 | @Slf4j 21 | @Component 22 | public final class HostsHandler extends Handler implements InitializingBean { 23 | 24 | @Override 25 | public Rule parse(String line) { 26 | Map.Entry entry = Util.parseHosts(line); 27 | if (entry == null || Objects.equals(entry.getKey(), entry.getValue())) { 28 | return Rule.EMPTY; 29 | } 30 | 31 | Rule rule = new Rule(); 32 | rule.setSource(RuleSet.HOSTS); 33 | rule.setOrigin(line); 34 | rule.setTarget(entry.getValue()); 35 | rule.setMode(LOCAL_IP.contains(entry.getKey()) && !LOCAL_DOMAIN.contains(entry.getValue()) ? Rule.Mode.DENY : Rule.Mode.REWRITE); 36 | rule.setDest(Rule.Mode.DENY == rule.getMode() ? UNKNOWN_IP : entry.getKey()); 37 | rule.setScope(Rule.Scope.DOMAIN); 38 | rule.setType(Rule.Type.BASIC); 39 | return rule; 40 | } 41 | 42 | @Override 43 | public String format(Rule rule) { 44 | if (Rule.Scope.DOMAIN == rule.getScope() && 45 | Rule.Type.BASIC == rule.getType() && 46 | Rule.Mode.ALLOW != rule.getMode()) { 47 | return Optional.ofNullable(rule.getDest()).orElse(UNKNOWN_IP) + TAB + rule.getTarget(); 48 | } 49 | return null; 50 | } 51 | 52 | @Override 53 | public String commented(String value) { 54 | return Util.splitIgnoreBlank(value, LF).stream() 55 | .map(e -> HASH + WHITESPACE + e.trim()) 56 | .collect(Collectors.joining(CRLF)); 57 | } 58 | 59 | @Override 60 | public boolean isComment(String line) { 61 | return line.startsWith(HASH); 62 | } 63 | 64 | @Override 65 | public void afterPropertiesSet() { 66 | this.register(RuleSet.HOSTS, this); 67 | } 68 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/handler/rule/SmartdnsHandler.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.handler.rule; 2 | 3 | import org.fordes.adfs.enums.RuleSet; 4 | import org.fordes.adfs.model.Rule; 5 | import org.fordes.adfs.util.Util; 6 | import org.springframework.beans.factory.InitializingBean; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.List; 10 | import java.util.Set; 11 | import java.util.stream.Collectors; 12 | 13 | import static org.fordes.adfs.constant.Constants.*; 14 | import static org.fordes.adfs.model.Rule.Mode.ALLOW; 15 | 16 | /** 17 | * @author fordes123 on 2024/5/27 18 | */ 19 | @Component 20 | public final class SmartdnsHandler extends Handler implements InitializingBean { 21 | 22 | @Override 23 | public Rule parse(String line) { 24 | 25 | String content = Util.subAfter(line, SMARTDNS_HEADER, true); 26 | List data = Util.splitIgnoreBlank(content, SLASH); 27 | if (data.size() == 2) { 28 | String domain = data.getFirst(); 29 | String control = data.get(1); 30 | Rule rule = new Rule(); 31 | rule.setOrigin(line); 32 | rule.setSource(RuleSet.EASYLIST); 33 | 34 | switch (control) { 35 | case HASH -> rule.setMode(Rule.Mode.DENY); 36 | case DASH -> rule.setMode(ALLOW); 37 | default -> { 38 | //未知或不支持的控制符 如 #6 #4 39 | rule.setType(Rule.Type.UNKNOWN); 40 | return rule; 41 | } 42 | } 43 | 44 | //仅匹配主域名 45 | if (domain.startsWith(DASH)) { 46 | domain = domain.substring(0, line.length() - DASH.length()); 47 | } else { 48 | rule.setControls(Set.of(Rule.Control.OVERLAY)); 49 | } 50 | 51 | if (domain.startsWith(DOT)) { 52 | domain = domain.substring(0, line.length() - DOT.length()); 53 | } 54 | 55 | rule.setType(domain.startsWith(ASTERISK) ? Rule.Type.WILDCARD : Rule.Type.BASIC); 56 | rule.setTarget(domain); 57 | rule.setDest(UNKNOWN_IP); 58 | rule.setScope(Rule.Scope.DOMAIN); 59 | return rule; 60 | } 61 | return Rule.EMPTY; 62 | } 63 | 64 | @Override 65 | public String format(Rule rule) { 66 | if (Rule.Type.UNKNOWN == rule.getType()) { 67 | if (RuleSet.SMARTDNS == rule.getSource()) { 68 | return rule.getOrigin(); 69 | } 70 | return null; 71 | } else if (rule.getMode() != Rule.Mode.REWRITE && rule.getScope() == Rule.Scope.DOMAIN) { 72 | 73 | switch (rule.getType()) { 74 | case BASIC -> { 75 | return SMARTDNS_HEADER + 76 | (!rule.getControls().contains(Rule.Control.OVERLAY) ? 77 | (ASTERISK + DOT) : EMPTY) + 78 | rule.getTarget() + 79 | SLASH + 80 | (Rule.Mode.DENY.equals(rule.getMode()) ? HASH : DASH); 81 | } 82 | case WILDCARD -> { 83 | String domain = rule.getTarget(); 84 | if (domain.lastIndexOf(ASTERISK) == 0) { 85 | return SMARTDNS_HEADER + domain + SLASH + DASH; 86 | } 87 | } 88 | } 89 | } 90 | return null; 91 | } 92 | 93 | @Override 94 | public String commented(String value) { 95 | return Util.splitIgnoreBlank(value, LF).stream() 96 | .map(e -> HASH + WHITESPACE + e.trim()) 97 | .collect(Collectors.joining(CRLF)); 98 | } 99 | 100 | @Override 101 | public boolean isComment(String line) { 102 | return line.startsWith(HASH); 103 | } 104 | 105 | @Override 106 | public void afterPropertiesSet() { 107 | this.register(RuleSet.SMARTDNS, this); 108 | } 109 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/model/Rule.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.model; 2 | 3 | import lombok.Data; 4 | import org.fordes.adfs.enums.RuleSet; 5 | 6 | import java.util.HashSet; 7 | import java.util.Objects; 8 | import java.util.Set; 9 | 10 | /** 11 | * @author fordes123 on 2024/5/27 12 | */ 13 | @Data 14 | public class Rule { 15 | private RuleSet source; 16 | private String origin; 17 | 18 | private String target; 19 | private String dest; 20 | 21 | private Mode mode; 22 | private Scope scope; 23 | private Type type; 24 | private Set controls = new HashSet<>(Control.values().length, 1.0f); 25 | 26 | public static final Rule EMPTY = new Rule(); 27 | 28 | /** 29 | * 规则控制参数 30 | */ 31 | public enum Control { 32 | /** 33 | * 最高优先级 34 | */ 35 | IMPORTANT, 36 | 37 | /** 38 | * 覆盖子域名 39 | */ 40 | 41 | OVERLAY, 42 | 43 | /** 44 | * 限定符，通常是 ^ 45 | */ 46 | QUALIFIER, 47 | 48 | ; 49 | } 50 | 51 | /** 52 | * 规则模式 53 | */ 54 | public enum Mode { 55 | 56 | /** 57 | * 阻止 58 | */ 59 | DENY, 60 | 61 | /** 62 | * 解除阻止 63 | */ 64 | ALLOW, 65 | 66 | /** 67 | * 重写
68 | * 通常 hosts规则指向特定ip(非localhost)时即为重写 69 | */ 70 | REWRITE, 71 | 72 | ; 73 | } 74 | 75 | /** 76 | * 规则类型 77 | */ 78 | public enum Type { 79 | /** 80 | * 基本规则，不包含任何控制、匹配符号, 可以转换为 hosts 81 | */ 82 | BASIC, 83 | 84 | /** 85 | * 通配规则，仅使用通配符 86 | */ 87 | WILDCARD, 88 | 89 | /** 90 | * 其他规则，如使用了正则、高级修饰符号等，这表示目前无法支持 91 | */ 92 | UNKNOWN, 93 | 94 | ; 95 | } 96 | 97 | /** 98 | * 作用域 99 | */ 100 | public enum Scope { 101 | /** 102 | * ipv4或ipv6地址 103 | */ 104 | HOST, 105 | 106 | /** 107 | * 域名 108 | */ 109 | DOMAIN, 110 | 111 | /** 112 | * 路径、文件等 113 | */ 114 | PATH, 115 | 116 | ; 117 | } 118 | 119 | @Override 120 | public boolean equals(Object o) { 121 | if (this == o) return true; 122 | if (o instanceof Rule rule) { 123 | if (Type.UNKNOWN == this.type || Type.UNKNOWN == rule.getType()) { 124 | return Objects.equals(this.origin, rule.origin); 125 | } 126 | return Objects.equals(this.target, rule.target) && 127 | this.mode == rule.mode && 128 | this.scope == rule.scope && 129 | this.type == rule.type; 130 | } 131 | return false; 132 | } 133 | 134 | @Override 135 | public int hashCode() { 136 | if (Type.UNKNOWN == this.type) { 137 | return Objects.hash(this.origin); 138 | } 139 | return Objects.hash(getTarget(), getMode(), getScope(), getType()); 140 | } 141 | 142 | @Override 143 | public String toString() { 144 | if (Type.UNKNOWN == this.type) { 145 | return "Rule{" + 146 | "origin='" + origin + '\'' + 147 | '}'; 148 | } 149 | return "Rule{" + 150 | "target='" + target + '\'' + 151 | ", mode=" + mode + 152 | ", scope=" + scope + 153 | ", type=" + type + 154 | '}'; 155 | } 156 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/task/Endpoint.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.task; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.fordes.adfs.config.InputProperties; 6 | import org.fordes.adfs.config.OutputProperties; 7 | import org.fordes.adfs.handler.Parser; 8 | import org.fordes.adfs.handler.rule.Handler; 9 | import org.springframework.boot.ApplicationRunner; 10 | import org.springframework.boot.SpringApplication; 11 | import org.springframework.context.ApplicationContext; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.stereotype.Component; 14 | import reactor.core.publisher.Flux; 15 | import reactor.core.publisher.Mono; 16 | import reactor.core.scheduler.Schedulers; 17 | import reactor.util.function.Tuple2; 18 | import reactor.util.function.Tuples; 19 | 20 | import java.nio.charset.StandardCharsets; 21 | import java.nio.file.Files; 22 | import java.nio.file.Path; 23 | import java.nio.file.StandardOpenOption; 24 | import java.time.Duration; 25 | import java.util.List; 26 | 27 | /** 28 | * @author fordes on 2024/4/10 29 | */ 30 | @Slf4j 31 | @Component 32 | @RequiredArgsConstructor 33 | public class Endpoint { 34 | 35 | private final ApplicationContext context; 36 | private final InputProperties input; 37 | private final OutputProperties output; 38 | private final Parser parser; 39 | 40 | @Bean 41 | ApplicationRunner start() { 42 | return args -> { 43 | long start = System.currentTimeMillis(); 44 | this.initialize() 45 | .thenMany(Flux.fromStream(input.stream())) 46 | .flatMap(e -> parser.handle(e.getValue(), e.getKey()), 1) 47 | .flatMap(rule -> Flux.fromIterable(output.getFiles()) 48 | .filter(file -> file.filter().contains(rule.getType())) 49 | .flatMap(config -> { 50 | Handler handler = Handler.getHandler(config.type()); 51 | String content = handler.format(rule); 52 | 53 | return content == null 54 | ? Mono.empty() 55 | : Mono.just(Tuples.of(config, content)); 56 | })) 57 | .groupBy(Tuple2::getT1, Tuple2::getT2) 58 | .flatMap(group -> { 59 | Path path = Path.of(output.getPath(), group.key().name()); 60 | return group 61 | .bufferTimeout(5000, Duration.ofSeconds(1)) 62 | .concatMap(batch -> asyncBatchWrite(path, batch)) 63 | .subscribeOn(Schedulers.single()); 64 | }) 65 | .then() 66 | .doFinally(signal -> { 67 | log.info("all done, cost {}ms", System.currentTimeMillis() - start); 68 | this.exit(); 69 | }) 70 | .block(); 71 | }; 72 | } 73 | 74 | private Mono initialize() { 75 | long start = System.currentTimeMillis(); 76 | return Mono.just(output.getPath()) 77 | .flatMapMany(p -> { 78 | Path path = Path.of(p); 79 | return Mono.fromCallable(() -> { 80 | Files.createDirectories(path); 81 | return p; 82 | }).subscribeOn(Schedulers.boundedElastic()).flux(); 83 | }) 84 | .onErrorResume(ex -> { 85 | log.error("create output dir failed", ex); 86 | return Mono.empty(); 87 | }) 88 | .flatMap(dir -> 89 | Flux.fromIterable(output.getFiles()).map(config -> { 90 | String header = config.displayHeader(Handler.getHandler(config.type()), output.getFileHeader()); 91 | Path path = Path.of(dir, config.name()); 92 | return Tuples.of(header, path); 93 | }) 94 | ) 95 | .flatMap(t -> { 96 | String fileHeader = t.getT1(); 97 | Path path = t.getT2(); 98 | return Mono.fromCallable(() -> { 99 | 100 | Files.writeString( 101 | path, 102 | fileHeader, 103 | StandardOpenOption.CREATE, 104 | StandardOpenOption.TRUNCATE_EXISTING, 105 | StandardOpenOption.WRITE 106 | ); 107 | 108 | return true; 109 | }).subscribeOn(Schedulers.boundedElastic()).then(); 110 | }).doOnError(ex -> { 111 | log.error("initialize failed", ex); 112 | this.exit(); 113 | }) 114 | .doFinally(signal -> { 115 | log.info("initialize done in {}ms", System.currentTimeMillis() - start); 116 | }) 117 | .then(); 118 | } 119 | 120 | private Mono asyncBatchWrite(Path path, List batch) { 121 | return Mono.fromCallable(() -> { 122 | Files.write(path, batch, StandardCharsets.UTF_8, 123 | StandardOpenOption.CREATE, StandardOpenOption.APPEND); 124 | return (Void) null; 125 | }) 126 | .onErrorResume(e -> Mono.fromRunnable(() -> log.error("Write failed", e))) 127 | .subscribeOn(Schedulers.boundedElastic()); 128 | } 129 | 130 | private void exit() { 131 | int exit = SpringApplication.exit(this.context, () -> 0); 132 | System.exit(exit); 133 | } 134 | 135 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/util/BloomFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This program is free software: you can redistribute it and/or modify 3 | * it under the terms of the GNU Lesser General Public License as published by 4 | * the Free Software Foundation, either version 3 of the License, or 5 | * (at your option) any later version. 6 | * 7 | * This program is distributed in the hope that it will be useful, 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | * GNU Lesser General Public License for more details. 11 | * 12 | * You should have received a copy of the GNU Lesser General Public License 13 | * along with this program. If not, see . 14 | */ 15 | 16 | package org.fordes.adfs.util; 17 | 18 | import java.io.Serializable; 19 | import java.nio.charset.Charset; 20 | import java.security.MessageDigest; 21 | import java.security.NoSuchAlgorithmException; 22 | import java.util.BitSet; 23 | import java.util.Collection; 24 | 25 | /** 26 | * Implementation of a Bloom-filter, as described here: 27 | * http://en.wikipedia.org/wiki/Bloom_filter 28 | * 29 | * For updates and bugfixes, see http://github.com/magnuss/java-bloomfilter 30 | * 31 | * Inspired by the SimpleBloomFilter-class written by Ian Clarke. This 32 | * implementation provides a more evenly distributed Hash-function by 33 | * using a proper digest instead of the Java RNG. Many of the changes 34 | * were proposed in comments in his blog: 35 | * http://blog.locut.us/2008/01/12/a-decent-stand-alone-java-bloom-filter-implementation/ 36 | * 37 | * @param Object type that is to be inserted into the Bloom filter, e.g. String or Integer. 38 | * @author Magnus Skjegstad 39 | */ 40 | @SuppressWarnings("all") 41 | public class BloomFilter implements Serializable { 42 | private BitSet bitset; 43 | private int bitSetSize; 44 | private double bitsPerElement; 45 | private int expectedNumberOfFilterElements; // expected (maximum) number of elements to be added 46 | private int numberOfAddedElements; // number of elements actually added to the Bloom filter 47 | private int k; // number of hash functions 48 | 49 | static final Charset charset = Charset.forName("UTF-8"); // encoding used for storing hash values as strings 50 | 51 | static final String hashName = "MD5"; // MD5 gives good enough accuracy in most circumstances. Change to SHA1 if it's needed 52 | static final MessageDigest digestFunction; 53 | static { // The digest method is reused between instances 54 | MessageDigest tmp; 55 | try { 56 | tmp = java.security.MessageDigest.getInstance(hashName); 57 | } catch (NoSuchAlgorithmException e) { 58 | tmp = null; 59 | } 60 | digestFunction = tmp; 61 | } 62 | 63 | /** 64 | * Constructs an empty Bloom filter. The total length of the Bloom filter will be 65 | * c*n. 66 | * 67 | * @param c is the number of bits used per element. 68 | * @param n is the expected number of elements the filter will contain. 69 | * @param k is the number of hash functions used. 70 | */ 71 | public BloomFilter(double c, int n, int k) { 72 | this.expectedNumberOfFilterElements = n; 73 | this.k = k; 74 | this.bitsPerElement = c; 75 | this.bitSetSize = (int)Math.ceil(c * n); 76 | numberOfAddedElements = 0; 77 | this.bitset = new BitSet(bitSetSize); 78 | } 79 | 80 | /** 81 | * Constructs an empty Bloom filter. The optimal number of hash functions (k) is estimated from the total size of the Bloom 82 | * and the number of expected elements. 83 | * 84 | * @param bitSetSize defines how many bits should be used in total for the filter. 85 | * @param expectedNumberOElements defines the maximum number of elements the filter is expected to contain. 86 | */ 87 | public BloomFilter(int bitSetSize, int expectedNumberOElements) { 88 | this(bitSetSize / (double)expectedNumberOElements, 89 | expectedNumberOElements, 90 | (int) Math.round((bitSetSize / (double)expectedNumberOElements) * Math.log(2.0))); 91 | } 92 | 93 | /** 94 | * Constructs an empty Bloom filter with a given false positive probability. The number of bits per 95 | * element and the number of hash functions is estimated 96 | * to match the false positive probability. 97 | * 98 | * @param falsePositiveProbability is the desired false positive probability. 99 | * @param expectedNumberOfElements is the expected number of elements in the Bloom filter. 100 | */ 101 | public BloomFilter(double falsePositiveProbability, int expectedNumberOfElements) { 102 | this(Math.ceil(-(Math.log(falsePositiveProbability) / Math.log(2))) / Math.log(2), // c = k / ln(2) 103 | expectedNumberOfElements, 104 | (int)Math.ceil(-(Math.log(falsePositiveProbability) / Math.log(2)))); // k = ceil(-log_2(false prob.)) 105 | } 106 | 107 | /** 108 | * Construct a new Bloom filter based on existing Bloom filter data. 109 | * 110 | * @param bitSetSize defines how many bits should be used for the filter. 111 | * @param expectedNumberOfFilterElements defines the maximum number of elements the filter is expected to contain. 112 | * @param actualNumberOfFilterElements specifies how many elements have been inserted into the filterData BitSet. 113 | * @param filterData a BitSet representing an existing Bloom filter. 114 | */ 115 | public BloomFilter(int bitSetSize, int expectedNumberOfFilterElements, int actualNumberOfFilterElements, BitSet filterData) { 116 | this(bitSetSize, expectedNumberOfFilterElements); 117 | this.bitset = filterData; 118 | this.numberOfAddedElements = actualNumberOfFilterElements; 119 | } 120 | 121 | /** 122 | * Generates a digest based on the contents of a String. 123 | * 124 | * @param val specifies the input data. 125 | * @param charset specifies the encoding of the input data. 126 | * @return digest as long. 127 | */ 128 | public static int createHash(String val, Charset charset) { 129 | return createHash(val.getBytes(charset)); 130 | } 131 | 132 | /** 133 | * Generates a digest based on the contents of a String. 134 | * 135 | * @param val specifies the input data. The encoding is expected to be UTF-8. 136 | * @return digest as long. 137 | */ 138 | public static int createHash(String val) { 139 | return createHash(val, charset); 140 | } 141 | 142 | /** 143 | * Generates a digest based on the contents of an array of bytes. 144 | * 145 | * @param data specifies input data. 146 | * @return digest as long. 147 | */ 148 | public static int createHash(byte[] data) { 149 | return createHashes(data, 1)[0]; 150 | } 151 | 152 | /** 153 | * Generates digests based on the contents of an array of bytes and splits the result into 4-byte int's and store them in an array. The 154 | * digest function is called until the required number of int's are produced. For each call to digest a salt 155 | * is prepended to the data. The salt is increased by 1 for each call. 156 | * 157 | * @param data specifies input data. 158 | * @param hashes number of hashes/int's to produce. 159 | * @return array of int-sized hashes 160 | */ 161 | public static int[] createHashes(byte[] data, int hashes) { 162 | int[] result = new int[hashes]; 163 | 164 | int k = 0; 165 | byte salt = 0; 166 | while (k < hashes) { 167 | byte[] digest; 168 | synchronized (digestFunction) { 169 | digestFunction.update(salt); 170 | salt++; 171 | digest = digestFunction.digest(data); 172 | } 173 | 174 | for (int i = 0; i < digest.length/4 && k < hashes; i++) { 175 | int h = 0; 176 | for (int j = (i*4); j < (i*4)+4; j++) { 177 | h <<= 8; 178 | h |= ((int) digest[j]) & 0xFF; 179 | } 180 | result[k] = h; 181 | k++; 182 | } 183 | } 184 | return result; 185 | } 186 | 187 | /** 188 | * Compares the contents of two instances to see if they are equal. 189 | * 190 | * @param obj is the object to compare to. 191 | * @return True if the contents of the objects are equal. 192 | */ 193 | @Override 194 | public boolean equals(Object obj) { 195 | if (obj == null) { 196 | return false; 197 | } 198 | if (getClass() != obj.getClass()) { 199 | return false; 200 | } 201 | final BloomFilter other = (BloomFilter) obj; 202 | if (this.expectedNumberOfFilterElements != other.expectedNumberOfFilterElements) { 203 | return false; 204 | } 205 | if (this.k != other.k) { 206 | return false; 207 | } 208 | if (this.bitSetSize != other.bitSetSize) { 209 | return false; 210 | } 211 | if (this.bitset != other.bitset && (this.bitset == null || !this.bitset.equals(other.bitset))) { 212 | return false; 213 | } 214 | return true; 215 | } 216 | 217 | /** 218 | * Calculates a hash code for this class. 219 | * @return hash code representing the contents of an instance of this class. 220 | */ 221 | @Override 222 | public int hashCode() { 223 | int hash = 7; 224 | hash = 61 * hash + (this.bitset != null ? this.bitset.hashCode() : 0); 225 | hash = 61 * hash + this.expectedNumberOfFilterElements; 226 | hash = 61 * hash + this.bitSetSize; 227 | hash = 61 * hash + this.k; 228 | return hash; 229 | } 230 | 231 | 232 | /** 233 | * Calculates the expected probability of false positives based on 234 | * the number of expected filter elements and the size of the Bloom filter. 235 | *

236 | * The value returned by this method is the expected rate of false 237 | * positives, assuming the number of inserted elements equals the number of 238 | * expected elements. If the number of elements in the Bloom filter is less 239 | * than the expected value, the true probability of false positives will be lower. 240 | * 241 | * @return expected probability of false positives. 242 | */ 243 | public double expectedFalsePositiveProbability() { 244 | return getFalsePositiveProbability(expectedNumberOfFilterElements); 245 | } 246 | 247 | /** 248 | * Calculate the probability of a false positive given the specified 249 | * number of inserted elements. 250 | * 251 | * @param numberOfElements number of inserted elements. 252 | * @return probability of a false positive. 253 | */ 254 | public double getFalsePositiveProbability(double numberOfElements) { 255 | // (1 - e^(-k * n / m)) ^ k 256 | return Math.pow((1 - Math.exp(-k * (double) numberOfElements 257 | / (double) bitSetSize)), k); 258 | 259 | } 260 | 261 | /** 262 | * Get the current probability of a false positive. The probability is calculated from 263 | * the size of the Bloom filter and the current number of elements added to it. 264 | * 265 | * @return probability of false positives. 266 | */ 267 | public double getFalsePositiveProbability() { 268 | return getFalsePositiveProbability(numberOfAddedElements); 269 | } 270 | 271 | 272 | /** 273 | * Returns the value chosen for K.
274 | *
275 | * K is the optimal number of hash functions based on the size 276 | * of the Bloom filter and the expected number of inserted elements. 277 | * 278 | * @return optimal k. 279 | */ 280 | public int getK() { 281 | return k; 282 | } 283 | 284 | /** 285 | * Sets all bits to false in the Bloom filter. 286 | */ 287 | public void clear() { 288 | bitset.clear(); 289 | numberOfAddedElements = 0; 290 | } 291 | 292 | /** 293 | * Adds an object to the Bloom filter. The output from the object's 294 | * toString() method is used as input to the hash functions. 295 | * 296 | * @param element is an element to register in the Bloom filter. 297 | */ 298 | public void add(E element) { 299 | add(element.toString().getBytes(charset)); 300 | } 301 | 302 | /** 303 | * Adds an array of bytes to the Bloom filter. 304 | * 305 | * @param bytes array of bytes to add to the Bloom filter. 306 | */ 307 | public void add(byte[] bytes) { 308 | int[] hashes = createHashes(bytes, k); 309 | for (int hash : hashes) 310 | bitset.set(Math.abs(hash % bitSetSize), true); 311 | numberOfAddedElements ++; 312 | } 313 | 314 | /** 315 | * Adds all elements from a Collection to the Bloom filter. 316 | * @param c Collection of elements. 317 | */ 318 | public void addAll(Collection c) { 319 | for (E element : c) 320 | add(element); 321 | } 322 | 323 | /** 324 | * Returns true if the element could have been inserted into the Bloom filter. 325 | * Use getFalsePositiveProbability() to calculate the probability of this 326 | * being correct. 327 | * 328 | * @param element element to check. 329 | * @return true if the element could have been inserted into the Bloom filter. 330 | */ 331 | public boolean contains(E element) { 332 | return contains(element.toString().getBytes(charset)); 333 | } 334 | 335 | /** 336 | * Returns true if the array of bytes could have been inserted into the Bloom filter. 337 | * Use getFalsePositiveProbability() to calculate the probability of this 338 | * being correct. 339 | * 340 | * @param bytes array of bytes to check. 341 | * @return true if the array could have been inserted into the Bloom filter. 342 | */ 343 | public boolean contains(byte[] bytes) { 344 | int[] hashes = createHashes(bytes, k); 345 | for (int hash : hashes) { 346 | if (!bitset.get(Math.abs(hash % bitSetSize))) { 347 | return false; 348 | } 349 | } 350 | return true; 351 | } 352 | 353 | /** 354 | * Returns true if all the elements of a Collection could have been inserted 355 | * into the Bloom filter. Use getFalsePositiveProbability() to calculate the 356 | * probability of this being correct. 357 | * @param c elements to check. 358 | * @return true if all the elements in c could have been inserted into the Bloom filter. 359 | */ 360 | public boolean containsAll(Collection c) { 361 | for (E element : c) 362 | if (!contains(element)) 363 | return false; 364 | return true; 365 | } 366 | 367 | /** 368 | * Read a single bit from the Bloom filter. 369 | * @param bit the bit to read. 370 | * @return true if the bit is set, false if it is not. 371 | */ 372 | public boolean getBit(int bit) { 373 | return bitset.get(bit); 374 | } 375 | 376 | /** 377 | * Set a single bit in the Bloom filter. 378 | * @param bit is the bit to set. 379 | * @param value If true, the bit is set. If false, the bit is cleared. 380 | */ 381 | public void setBit(int bit, boolean value) { 382 | bitset.set(bit, value); 383 | } 384 | 385 | /** 386 | * Return the bit set used to store the Bloom filter. 387 | * @return bit set representing the Bloom filter. 388 | */ 389 | public BitSet getBitSet() { 390 | return bitset; 391 | } 392 | 393 | /** 394 | * Returns the number of bits in the Bloom filter. Use count() to retrieve 395 | * the number of inserted elements. 396 | * 397 | * @return the size of the bitset used by the Bloom filter. 398 | */ 399 | public int size() { 400 | return this.bitSetSize; 401 | } 402 | 403 | /** 404 | * Returns the number of elements added to the Bloom filter after it 405 | * was constructed or after clear() was called. 406 | * 407 | * @return number of elements added to the Bloom filter. 408 | */ 409 | public int count() { 410 | return this.numberOfAddedElements; 411 | } 412 | 413 | /** 414 | * Returns the expected number of elements to be inserted into the filter. 415 | * This value is the same value as the one passed to the constructor. 416 | * 417 | * @return expected number of elements. 418 | */ 419 | public int getExpectedNumberOfElements() { 420 | return expectedNumberOfFilterElements; 421 | } 422 | 423 | /** 424 | * Get expected number of bits per element when the Bloom filter is full. This value is set by the constructor 425 | * when the Bloom filter is created. See also getBitsPerElement(). 426 | * 427 | * @return expected number of bits per element. 428 | */ 429 | public double getExpectedBitsPerElement() { 430 | return this.bitsPerElement; 431 | } 432 | 433 | /** 434 | * Get actual number of bits per element based on the number of elements that have currently been inserted and the length 435 | * of the Bloom filter. See also getExpectedBitsPerElement(). 436 | * 437 | * @return number of bits per element. 438 | */ 439 | public double getBitsPerElement() { 440 | return this.bitSetSize / (double)numberOfAddedElements; 441 | } 442 | } -------------------------------------------------------------------------------- /src/main/java/org/fordes/adfs/util/Util.java: -------------------------------------------------------------------------------- 1 | package org.fordes.adfs.util; 2 | 3 | import jakarta.annotation.Nonnull; 4 | import jakarta.annotation.Nullable; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.fordes.adfs.model.Rule; 7 | import org.springframework.util.ObjectUtils; 8 | import org.springframework.util.StringUtils; 9 | 10 | import java.util.Arrays; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.function.BiConsumer; 14 | import java.util.function.Consumer; 15 | 16 | import static org.fordes.adfs.constant.Constants.*; 17 | import static org.fordes.adfs.constant.RegConstants.*; 18 | 19 | /** 20 | * @author fordes123 on 2022/9/19 21 | */ 22 | @Slf4j 23 | public class Util { 24 | 25 | /** 26 | * 给定字符串是否以特定前缀开始 27 | * 28 | * @param str 给定字符串 29 | * @param prefixes 前缀 30 | * @return 给定字符串是否以特定前缀开始 31 | */ 32 | public static boolean startWithAny(String str, String... prefixes) { 33 | if (!StringUtils.hasText(str) || ObjectUtils.isEmpty(prefixes)) { 34 | return false; 35 | } 36 | return Arrays.stream(prefixes).anyMatch(str::startsWith); 37 | } 38 | 39 | /** 40 | * 给定字符串是否以特定字符串开始和结束 41 | * 42 | * @param str 给定字符串 43 | * @param start 开始字符串 44 | * @param end 结束字符串 45 | * @return 给定字符串是否以特定字符串开始和结束 46 | */ 47 | public static boolean between(String str, String start, String end) { 48 | if (StringUtils.hasLength(str) && StringUtils.hasLength(start) && StringUtils.hasLength(end)) { 49 | return str.startsWith(start) && str.endsWith(end); 50 | } 51 | return false; 52 | } 53 | 54 | /** 55 | * 截取分隔字符串之前的字符串，不包括分隔字符串
56 | * 截取不到时返回空串 57 | * 58 | * @param str 被截取的字符串 59 | * @param flag 分隔字符串 60 | * @param isLast 是否是最后一个 61 | * @return 分隔字符串之前的字符串 62 | */ 63 | public static String subBefore(String str, String flag, boolean isLast) { 64 | if (StringUtils.hasLength(str) && StringUtils.hasLength(flag)) { 65 | int index = isLast ? str.lastIndexOf(flag) : str.indexOf(flag); 66 | if (index >= 0) { 67 | return str.substring(0, index); 68 | } 69 | } 70 | return EMPTY; 71 | } 72 | 73 | /** 74 | * 截取分隔字符串之后的字符串，不包括分隔字符串
75 | * 截取不到时返回空串 76 | * 77 | * @param content 被截取的字符串 78 | * @param flag 分隔字符串 79 | * @param isLast 是否是最后一个 80 | * @return 分隔字符串之后的字符串 81 | */ 82 | public static String subAfter(String content, String flag, boolean isLast) { 83 | if (StringUtils.hasLength(content) && StringUtils.hasLength(flag)) { 84 | int index = isLast ? content.lastIndexOf(flag) : content.indexOf(flag); 85 | if (index >= 0) { 86 | return content.substring(index + flag.length()); 87 | } 88 | } 89 | return EMPTY; 90 | } 91 | 92 | /** 93 | * 截取分隔字符串之间的字符串，不包括分隔字符串
94 | * 截取不到时返回空串 95 | * 96 | * @param content 被截取的字符串 97 | * @param start 开始分隔字符串 98 | * @param end 结束分隔字符串 99 | * @return 分隔字符串之间的字符串 100 | */ 101 | public static String subBetween(String content, String start, String end) { 102 | if (StringUtils.hasLength(content) && StringUtils.hasLength(start) && StringUtils.hasLength(end)) { 103 | int startIndex = content.indexOf(start); 104 | int endIndex = content.lastIndexOf(end); 105 | if (startIndex >= 0 && endIndex > 0 && startIndex < endIndex) { 106 | return content.substring(startIndex + start.length(), endIndex); 107 | } 108 | } 109 | return EMPTY; 110 | } 111 | 112 | /** 113 | * 切分字符串并移除空项 114 | * 115 | * @param str 待切分字符串 116 | * @param flag 分隔符 117 | * @return 切分后的字符串 118 | */ 119 | public static List splitIgnoreBlank(String str, String flag) { 120 | if (!StringUtils.hasLength(str) || !StringUtils.hasLength(flag)) { 121 | return List.of(); 122 | } 123 | return Arrays.stream(str.split(flag)) 124 | .filter(e -> !e.isBlank()) 125 | .toList(); 126 | } 127 | 128 | /** 129 | * 给定字符串是等于任一字符串 130 | * 131 | * @param str 给定字符串 132 | * @param values 任意字符串 133 | * @return 给定字符串是等于任一字符串 134 | */ 135 | public static boolean equalsAny(String str, String... values) { 136 | if (!StringUtils.hasLength(str) || ObjectUtils.isEmpty(values)) { 137 | return false; 138 | } 139 | return Arrays.asList(values).contains(str); 140 | } 141 | 142 | /** 143 | * 解析hosts规则，如不是则返回null 144 | * 145 | * @param content 待解析字符串 146 | * @return {@link Map.Entry} key:ip, value:域名 147 | */ 148 | public static @Nullable Map.Entry parseHosts(String content) { 149 | if (content.contains(TAB)) { 150 | content = content.replace(TAB, WHITESPACE); 151 | } 152 | List list = splitIgnoreBlank(content, WHITESPACE); 153 | if (list.size() == 2) { 154 | String ip = list.get(0).trim(); 155 | String domain = list.get(1).trim(); 156 | 157 | if (PATTERN_IP.matcher(ip).matches() && PATTERN_DOMAIN.matcher(domain).matches()) { 158 | return Map.entry(ip, domain); 159 | } 160 | } 161 | return null; 162 | } 163 | 164 | /** 165 | * 休眠线程，忽略中断异常 166 | * 167 | * @param millis 休眠时间，毫秒 168 | */ 169 | public static void sleep(long millis) { 170 | if (millis > 0L) { 171 | try { 172 | Thread.sleep(millis); 173 | } catch (InterruptedException ignored) { 174 | } 175 | } 176 | } 177 | 178 | /** 179 | * 转换相对路径为绝对路径 180 | * 181 | * @param path 路径 182 | * @return 规范化后的路径 183 | */ 184 | public static String normalizePath(@Nonnull String path) { 185 | 186 | boolean isAbsPath = '/' == path.charAt(0) || PATTERN_PATH_ABSOLUTE.matcher(path).matches(); 187 | 188 | if (!isAbsPath) { 189 | if (path.startsWith(DOT)) { 190 | path = path.substring(1); 191 | } 192 | if (path.startsWith(FILE_SEPARATOR)) { 193 | path = path.substring(FILE_SEPARATOR.length()); 194 | } 195 | path = ROOT_PATH + FILE_SEPARATOR + path; 196 | } 197 | return path; 198 | } 199 | 200 | 201 | public static void isBaseRule(String content, BiConsumer ifPresent, Consumer orElse) { 202 | String temp = content; 203 | if (temp.contains(ASTERISK)) { 204 | temp = content.replace(ASTERISK, A); 205 | } 206 | 207 | if (temp.startsWith(DOT)) { 208 | temp = temp.substring(1); 209 | } 210 | 211 | if (temp.endsWith(DOT)) { 212 | temp = temp.substring(0, temp.length() - 1); 213 | } 214 | 215 | if (PATTERN_DOMAIN.matcher(temp).matches()) { 216 | ifPresent.accept(content, content.equals(temp) ? Rule.Type.BASIC : Rule.Type.WILDCARD); 217 | } else if (DOMAIN_PART.matcher(temp).matches()) { 218 | ifPresent.accept(content, Rule.Type.WILDCARD); 219 | } else { 220 | orElse.accept(content); 221 | } 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /src/main/resources/application-dev.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | org.fordes.adfs: info 4 | 5 | application: 6 | config: 7 | domain-detect: 8 | enable: true 9 | concurrency: 64 10 | timeout: 2000 11 | provider: 12 | - 223.5.5.5 13 | rule: 14 | remote: 15 | - name: AdGuard 基础过滤器 16 | path: https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_Base/filter.txt 17 | type: easylist 18 | 19 | - name: Loyalsoldier/clash-rules 广告域名列表 20 | path: https://raw.githubusercontent.com/Loyalsoldier/clash-rules/release/reject.txt 21 | type: clash 22 | 23 | - name: 1Hosts lite 24 | path: https://o0.pages.dev/Lite/hosts.txt 25 | type: hosts 26 | 27 | - name: anti-ad smartdns 28 | path: https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-smartdns.conf 29 | type: smartdns 30 | 31 | - name: 32 | path: https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/adblock-for-dnsmasq.conf 33 | type: dnsmasq 34 | 35 | output: 36 | files: 37 | - name: easylist.txt 38 | type: easylist 39 | filter: 40 | - basic 41 | - wildcard 42 | - unknown 43 | 44 | - name: dns.txt 45 | type: easylist 46 | file-header: | 47 | Test 48 | filter: 49 | - basic 50 | - wildcard 51 | 52 | - name: modify.txt 53 | type: easylist 54 | filter: 55 | - unknown 56 | 57 | - name: dnsmasq.conf 58 | type: dnsmasq 59 | 60 | - name: clash.yaml 61 | type: clash 62 | 63 | - name: smartdns.conf 64 | type: smartdns 65 | 66 | - name: hosts 67 | type: hosts 68 | -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | application: 3 | name: ad-filters-subscriber 4 | profiles: 5 | active: dev #切换至 dev 即可输出调试信息 6 | 7 | logging: 8 | file: 9 | path: ./logs 10 | 11 | application: 12 | config: 13 | expected_quantity: 2000000 #预期规则数量 14 | fault_tolerance: 0.001 #容错率 15 | warn_limit: 6 #警告阈值, 原始规则长度小于该值时会输出警告日志 16 | 17 | # 域名检测，启用时将进行解析以验证域名有效性 18 | # 注意: 开启此功能可能导致处理时间大幅延长 19 | domain-detect: 20 | enable: false 21 | timeout: 10 22 | -------------------------------------------------------------------------------- /src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | 2 | ${AnsiColor.BRIGHT_GREEN} █████╗ ██████╗ ███████╗ ███████╗ 3 | ${AnsiColor.BRIGHT_GREEN} ██╔══██╗ ██╔══██╗ ██╔════╝ ██╔════╝ 4 | ${AnsiColor.BRIGHT_GREEN} ███████║ ██║ ██║ █████╗ ███████╗ 5 | ${AnsiColor.BRIGHT_GREEN} ██╔══██║ ██║ ██║ ██╔══╝ ╚════██║ 6 | ${AnsiColor.BRIGHT_GREEN} ██║ ██║ ██████╔╝ ██║ ███████║ 7 | ${AnsiColor.BRIGHT_GREEN} ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚══════╝ 8 | ${AnsiColor.BRIGHT_CYAN} :: Application :: ${AnsiColor.BRIGHT_RED}${spring.application.name} 9 | ${AnsiColor.BRIGHT_CYAN} :: Developers :: ${AnsiColor.BRIGHT_RED}fordes123 10 | ${AnsiColor.BRIGHT_CYAN} :: Github :: ${AnsiColor.BRIGHT_RED}https://github.com/fordes123/ad-filters-subscriber${AnsiColor.BRIGHT_WHITE} 11 | -------------------------------------------------------------------------------- /src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | %clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%10.10t]){faint} %clr(%-40.40logger{39}){cyan} %clr(%-6L){yellow} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx} 19 | UTF-8 20 | 21 | 22 | 23 | 24 | 25 | ${LOG_HOME}/${APP_NAME}.log 26 | 27 | 28 | ${LOG_HOME}/${APP_NAME}.%d{yyyy-MM-dd}.log 29 | 30 | 30 31 | 3GB 32 | 33 | true 34 | 35 | %d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx} 36 | UTF-8 37 | 38 | 39 | 40 | 41 | true 42 | 4096 43 | true 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | --------------------------------------------------------------------------------