├── .gitignore
├── .gitattributes
├── delayretest.sh
├── autoaddlist_smartdns.sh
├── debugip.sh
├── readme.md
├── autoaddlistdebug.sh
├── autoaddlist_dnsmasq.sh
└── testip.sh
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | *.bak
3 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
--------------------------------------------------------------------------------
/delayretest.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | echo $* | awk '{
3 | ip=$1
4 | domain=$2
5 | port=$3
6 | stop=0;
7 | while (stop==0){
8 | stop=1;
9 | system("sleep 120");
10 | while ("grep "ip" /proc/net/nf_conntrack"| getline ret > 0)
11 | {
12 | stop=0;
13 | break;
14 | }
15 | close("grep "ip" /proc/net/nf_conntrack");
16 | }
17 | "ipset test gfwlist "ip" 2>&1"| getline ipset;
18 | close("ipset test gfwlist "ip" 2>&1");
19 | if (index(ipset,"Warning")!=0){
20 | "ipset test china "ip" 2>&1"| getline ipset;
21 | close("ipset test china "ip" 2>&1");
22 | if (index(ipset,"Warning")!=0){
23 | print("china "ip" "domain" test again");
24 | system("ipset del gfwlist "$1);
25 | print "">"/tmp/run/"domain
26 | system("testip.sh "ip" "domain" "port" 1 1 &");
27 | }
28 | }
29 | }'
30 |
--------------------------------------------------------------------------------
/autoaddlist_smartdns.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | tail -F /var/log/smartdns-audit.log | awk -F "[, ]+" '{
3 | ip=$10;
4 | domain=$6;
5 | if ($8=="1" && (!(ip in a)))
6 | {
7 | "ipset test gfwlist "ip" 2>&1"| getline ipset;
8 | close("ipset test gfwlist "ip" 2>&1");
9 | if (index(ipset,"Warning")!=0){
10 | print("pass");
11 | next;
12 | }
13 |
14 | tryhttps=0;
15 | tryhttp=0;
16 | while ("grep "ip" /proc/net/nf_conntrack"|getline ret > 0)
17 | {
18 | split(ret, b);
19 | if (b[8]=="dst="ip)
20 | {
21 | if (b[10]=="dport=443"){
22 | tryhttps=1;
23 | break;
24 | }
25 | else if (b[10]=="dport=80"){
26 | tryhttp=1;
27 | }
28 | }
29 | }
30 | close("grep "ip" /proc/net/nf_conntrack");
31 | if (tryhttps==1)
32 | {
33 | print(ip" "domain" 443");
34 | a[ip]=domain;
35 | system("testip.sh "ip" "domain" 443 &");
36 | }
37 | else if (tryhttp==1)
38 | {
39 | print(ip" "domain" 80");
40 | a[ip]=domain;
41 | system("testip.sh "ip" "domain" 80 &");
42 | }
43 | }}'
--------------------------------------------------------------------------------
/debugip.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | dlchina=$1;
3 | ipset list gfwlist | awk -v dlchina="$dlchina" '{
4 | if (index($0,".")==0)
5 | {
6 | next;
7 | }
8 | "ipset test whitelist "$0" 2>&1"| getline ipset;
9 | close("ipset test whitelist "$0" 2>&1");
10 | if (index(ipset,"Warning")==0){
11 | white=0;
12 | }else{
13 | white=1;
14 | }
15 | "ipset test china "$0" 2>&1"| getline ipset;
16 | close("ipset test china "$0" 2>&1");
17 | if (index(ipset,"Warning")!=0){
18 | china=1;
19 | }
20 | else{
21 | china=0;
22 | }
23 | if (white==1)
24 | {
25 | if (china==0)
26 | {
27 | print("warning white ip not china"$0);
28 | ret=system("grep "$0" /tmp/nohup.out");
29 | if (ret!=0)
30 | {
31 | ret=system("grep "$0" /tmp/dnsmasq.log");
32 | }
33 | }
34 | }else if (china==1)
35 | {
36 | print("warning china ip not white"$0);
37 | ret=system("grep "$0" /tmp/nohup.out")
38 | if (ret!=0)
39 | {
40 | ret=system("grep "$0" /tmp/dnsmasq.log");
41 | }
42 | if (dlchina)
43 | {
44 | system("ipset del gfwlist "$0);
45 | }
46 | }
47 | }'
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | ## 原理
2 | 通过 dns日志来获得目标,通过nf_conntrack 80/443判断是否允许httping,允许的整个域名所有ip进行httping,如果超时或者rst,将结果加入ipset gfwlist,并且重试httping,如果不可用会取消加入ipset gfwlist
3 | ## 安装
4 | openwrt 免手动https://github.com/rufengsuixing/luci-app-autoipsetadder
5 | 依赖:httping,awk,ipset,curl,tail,stdbuf
6 |
7 | 安装httping:`opkg install httping`
8 | 安装stdbuf:`opkg install coreutils-stdbuf`
9 |
10 | - 二选一设置dns服务器日志记录:
11 | - smartdns:
12 | ```
13 | audit-enable yes
14 | audit-file /var/log/smartdns-audit.log
15 | audit-size 64K
16 | ```
17 | - dnsmasq:
18 | 以下开启dnsmasq的dns日志,并调整到需要的详细程度
19 | ```
20 | uci set dhcp.@dnsmasq[0].logfacility='/tmp/dnsmasq.log'
21 | uci delete dhcp.@dnsmasq[0].logqueries
22 | echo log-queries >> /etc/dnsmasq.conf
23 | uci commit dhcp
24 | ```
25 | - 对应你的dns服务程序复制autoaddlist.sh,testip.sh,delayretest.sh到/usr/bin/
26 | - 修改权限
27 | ```
28 | chmod 755 /usr/bin/autoaddlist.sh
29 | chmod 755 /usr/bin/testip.sh
30 | chmod 755 /usr/bin/delayretest.sh
31 | ```
32 | - 手动运行`/usr/bin/autoaddlist.sh &`
33 | 或者记录日志`nohup /usr/bin/autoaddlist.sh >>/tmp/nohup.out &`
34 | - crontab备用指令:
35 | 每小时删除日志
36 | ```
37 | 0 * * * * rm -f /tmp/log/smartdns*.gz
38 | 0 * * * * echo "" > /tmp/dnsmasq.log
39 | ```
40 | 停止指令备用:
41 | ```
42 | killall tail
43 | killall awk
44 | ```
45 | - debug用于寻找ipset gfwlist中的符合ipset china ip在/tmp/nohup.out中的日志
46 | `debugip.sh`
47 | ### 本程序输出日志:
48 |
49 | |输出|解释
50 | | -|-
51 | | `[ip] [domain] is in gfwlist pass"` | ip已经在ipset里
52 | | `[ip] [domain] [port]` | 记录检测到的可httping
53 | | `[浮点数值]`/`failed,` | httping得到的延迟结果,异步结果无参考价值
54 | | `can not connect autoaddip [ip] [domain]` | 直连无回应超时
55 | | `doname rst autoaddip [ip] [domain]` | 疑似直连rst
56 | | `proxy can not connect autodelip [ip] [domain]` | ipset后连接无回应超时
57 | | `doname proxy rst autodelip [ip] [domain]` | 疑似ipset后连接rst
58 | | `direct so slow autoaddip [ip] [domain]` | 直连有回应3s超时
59 | | `direct Connection refused autoaddip [ip] [domain]` | 直连拒绝连接
60 | | `change back to direct [ip] [domain]` | 尝试都失败或者都3s超时
61 | | `direct ssl so slow autoaddip [ip] [domain]` | httping超时无效bug被触发,ssl时间很久但成功了
62 | | `pass by packets=[number] [ip] [domain]` | 实验性质,在请求前看已经发送的包的数量>12放过
63 | | `[ip] [domain] pass by same domain ok` | 如果有一个可连接同域名ip放过
64 | | `warning china [ip] [domain] is in gfwlist` | 检测到china ipset与gfwlist重合
65 | | `ping packet loss autoaddip [ip] [domain]` | httping成功后,ping 5个包,返回收到1-3个包触发
66 |
67 | 注:同ip如果httping过不会重复探测,也不会有日志。
68 | [ ]httping在ssl上有问题,包括超时失效卡住和cloudflare的兼容不好,考虑之后用curl全部重写
69 |
--------------------------------------------------------------------------------
/autoaddlistdebug.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | awk -F "[, ]+" '/reply/{
3 | print($0);
4 | ip=$8;
5 | if (ip=="")
6 | {
7 | print("no ip");
8 | next;
9 | }
10 | if (index(ip,"")!=0)
11 | {
12 | print("cname");
13 | if (cname==1)
14 | {
15 | next;
16 | }
17 | cname=1;
18 | domain=$6;
19 | next;
20 | }
21 | if(lastdomain!=$6){
22 | for (ipindex in ipcache)
23 | {
24 | delete ipcache[ipindex];
25 | }
26 | ipcount=0;
27 | if (cname!=1)
28 | {
29 | domain=$6;
30 | testall=0;
31 | createpid=1;
32 | }}
33 |
34 | cname=0;
35 | lastdomain=$6
36 | if (index(ip,".")==0)
37 | {
38 | print(ip);
39 | print("noip");
40 | next;
41 | }
42 | ipcount+=1;
43 | if (!(ip in a))
44 | {
45 | "ipset test gfwlist "ip" 2>&1"| getline ipset;
46 | close("ipset test gfwlist "ip" 2>&1");
47 | if (index(ipset,"Warning")!=0){
48 | "ipset test china "ip" 2>&1"| getline ipset;
49 | close("ipset test china "ip" 2>&1");
50 | if (index(ipset,"Warning")!=0){
51 | print("warning china "ip" "domain" is in gfwlist")
52 | }else{
53 | print(ip" "domain" is in gfwlist pass");
54 | }
55 | next;
56 | }
57 | if (passdomain==domain)
58 | {
59 | print(ip" "domain" pass by same domain ok");
60 | a[ip]=domain;
61 | next;
62 | }
63 | ipcache[ipcount]=ip;
64 | if (testall==0){
65 | tryhttps=0;
66 | tryhttp=0;
67 | while ("grep "ip" /proc/net/nf_conntrack"| getline ret > 0)
68 | {
69 | split(ret, b," +");
70 | split(b[11], pagnum,"=");
71 | if (pagnum[2]>12)
72 | {
73 | print("pass by packets="pagnum[2]" "ip" "domain);
74 | for (ipindex in ipcache)
75 | {
76 | a[ipcache[ipindex]]=domain;
77 | delete ipcache[ipindex];
78 | }
79 | passdomain=domain;
80 | close("grep "ip" /proc/net/nf_conntrack");
81 | next;
82 | }
83 | if (b[8]=="dst="ip)
84 | {
85 | if (b[10]=="dport=443"){
86 | tryhttps=1;
87 | break;
88 | }
89 | else if (b[10]=="dport=80"){
90 | tryhttp=1;
91 | }
92 | }
93 | }
94 | close("grep "ip" /proc/net/nf_conntrack");
95 | }else{
96 | if (testall==443)
97 | {
98 | tryhttps=1
99 | }else{
100 | tryhttp=1
101 | }
102 | }
103 | if (tryhttps==1)
104 | { if (createpid==1)
105 | {
106 | print "">"/tmp/run/"domain
107 | close("/tmp/run/"domain);
108 | print("create"domain);
109 | print(ip" "domain" 443"ipcount-1);
110 | a[ip]=domain;
111 | system("testip.sh "ip" "domain" 443 "ipcount-1" &");
112 | delete ipcache[ipcount];
113 | createpid=0;
114 | }
115 | for (ipindex in ipcache){
116 | print(ipcache[ipindex]" "domain" 443 "ipindex-1);
117 | a[ipcache[ipindex]]=domain;
118 | system("testip.sh "ipcache[ipindex]" "domain" 443 "ipindex-1" &");
119 | delete ipcache[ipindex];
120 | }
121 |
122 | testall=443;
123 | }
124 | else if (tryhttp==1)
125 | {
126 | if (createpid==1)
127 | {
128 | print "">"/tmp/run/"domain
129 | close("/tmp/run/"domain);
130 | print("create"domain);
131 | print(ip" "domain" 80 "ipcount-1);
132 | a[ip]=domain;
133 | system("testip.sh "ip" "domain" 80 "ipcount-1" &");
134 | delete ipcache[ipcount];
135 | createpid=0;
136 | }
137 | for (ipindex in ipcache){
138 | print(ipcache[ipindex]" "domain" 80 "ipindex-1);
139 | a[ipcache[ipindex]]=domain;
140 | system("testip.sh "ipcache[ipindex]" "domain" 80 "ipindex-1" &");
141 | delete ipcache[ipindex];
142 | }
143 | testall=80;
144 | }else
145 | {print("not found")
146 | }}
147 | else{print("inlist");
148 | }}'
--------------------------------------------------------------------------------
/autoaddlist_dnsmasq.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | stdbuf -oL tail -F /tmp/dnsmasq.log | awk -F "[, ]+" '/reply/{
3 | ip=$8;
4 | if (ip=="")
5 | {
6 | next;
7 | }
8 | if (index(ip,"")!=0)
9 | {
10 | if (cname==1)
11 | {
12 | next;
13 | }
14 | cname=1;
15 | domain=$6;
16 | #第一次cname时锁定域名,防止解析cname对其改动
17 | next;
18 | }
19 | #以上获得上行是否为cname,本行不是cname执行以下内容
20 | #lastdomain记录上次非cname的域名,与本次域名对比
21 | if(lastdomain!=$6){
22 | for (ipindex in ipcache)
23 | {
24 | delete ipcache[ipindex];
25 | }
26 | ipcount=0;
27 | createpid=1;
28 | #上行非cname,并且不是同cname解析域名的多个ip,更新域名,清理同域名免试flag
29 | if (cname!=1)
30 | {
31 | domain=$6;
32 | testall=0;
33 | }}
34 | ipcount++;
35 | cname=0;
36 | lastdomain=$6
37 | #去除非ipv4
38 | if (index(ip,".")==0)
39 | {
40 | next;
41 | }
42 | #不重复探测ip
43 | if (!(ip in a))
44 | {
45 | #在gfwlist的chinaip警告,和忽略gfwlist中的ip
46 | "ipset test gfwlist "ip" 2>&1"| getline ipset;
47 | close("ipset test gfwlist "ip" 2>&1");
48 | if (index(ipset,"Warning")!=0){
49 | "ipset test china "ip" 2>&1"| getline ipset;
50 | close("ipset test china "ip" 2>&1");
51 | if (index(ipset,"Warning")!=0){
52 | print("warning china "ip" "domain" is in gfwlist")
53 | }else{
54 | print(ip" "domain" is in gfwlist pass");
55 | }
56 | next;
57 | }
58 | #包数>12的同域名放过
59 | if (passdomain==domain)
60 | {
61 | print(ip" "domain" pass by same domain ok");
62 | a[ip]=domain;
63 | next;
64 | }
65 | #ip压入缓存,用于未检测到443/80的缓存
66 | ipcache[ipcount]=ip;
67 | if (testall==0){
68 | tryhttps=0;
69 | tryhttp=0;
70 | #探测 nf_conntrack 的443/80
71 | while ("grep "ip" /proc/net/nf_conntrack"| getline ret > 0)
72 | {
73 | split(ret, b," +");
74 | split(b[11], pagnum,"=");
75 | #包数>12的放过
76 | if (pagnum[2]>12)
77 | {
78 | print("pass by packets="pagnum[2]" "ip" "domain);
79 | for (ipindex in ipcache)
80 | {
81 | a[ipcache[ipindex]]=domain;
82 | delete ipcache[ipindex];
83 | }
84 | passdomain=domain;
85 | close("grep "ip" /proc/net/nf_conntrack");
86 | ipcount--;
87 | next;
88 | }
89 | if (b[8]=="dst="ip)
90 | {
91 | if (b[10]=="dport=443"){
92 | tryhttps=1;
93 | break;
94 | }
95 | else if (b[10]=="dport=80"){
96 | tryhttp=1;
97 | }
98 | }
99 | }
100 | close("grep "ip" /proc/net/nf_conntrack");
101 | }else{
102 | if (testall==443)
103 | {
104 | tryhttps=1
105 | }else{
106 | tryhttp=1
107 | }
108 | }
109 | if (tryhttps==1)
110 | { if (createpid==1)
111 | {
112 | print "">"/tmp/run/"domain
113 | close("/tmp/run/"domain);
114 | print("create"domain);
115 | print(ip" "domain" 443"ipcount-1);
116 | a[ip]=domain;
117 | #正在使用的ip用最大延迟,最后探测,减少打断tcp的可能
118 | system("testip.sh "ip" "domain" 443 "ipcount-1" &");
119 | delete ipcache[ipcount];
120 | createpid=0;
121 | }
122 | #未检测到443/80同域名缓存的ip进行测试,ipindex-1为测试延迟时间
123 | for (ipindex in ipcache){
124 | print(ipcache[ipindex]" "domain" 443 "ipindex-1);
125 | a[ipcache[ipindex]]=domain;
126 | system("testip.sh "ipcache[ipindex]" "domain" 443 "ipindex-1" &");
127 | delete ipcache[ipindex];
128 | }
129 | #后续同域名ip免nf_conntrack测试
130 | testall=443;
131 | }
132 | else if (tryhttp==1)
133 | {
134 | if (createpid==1)
135 | {
136 | print "">"/tmp/run/"domain
137 | close("/tmp/run/"domain);
138 | print("create"domain);
139 | print(ip" "domain" 80 "ipcount-1);
140 | a[ip]=domain;
141 | system("testip.sh "ip" "domain" 80 "ipcount-1" &");
142 | delete ipcache[ipcount];
143 | createpid=0;
144 | }
145 | for (ipindex in ipcache){
146 | print(ipcache[ipindex]" "domain" 80 "ipindex-1);
147 | a[ipcache[ipindex]]=domain;
148 | system("testip.sh "ipcache[ipindex]" "domain" 80 "ipindex-1" &");
149 | delete ipcache[ipindex];
150 | }
151 | testall=80;
152 | }}
153 | }'
--------------------------------------------------------------------------------
/testip.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | echo $* | awk '{
3 | if ($4=="")
4 | {
5 | wait=0;
6 | }else
7 | {wait=$4;}
8 | system("sleep "wait);
9 | ERRNO="";
10 | getline drop< "/tmp/run/"$2;
11 | close("/tmp/run/"$2);
12 | if (ERRNO) {
13 | addlist=0;
14 | print("bypass"$1" "$2);
15 | next;}
16 | if ($3=="443")
17 | {
18 | cmd=("httping -c 1 -t 4 -l "$2" --divert-connect "$1);
19 | }
20 | else if ($3=="80")
21 | {
22 | cmd=("httping -c 1 -t 4 "$2" --divert-connect "$1);
23 | }
24 | addlist=0;
25 | slow=0;
26 | while ((cmd | getline ret) > 0)
27 | {
28 | if (addlist!=0)
29 | {
30 | continue;
31 | }
32 | else if (index(ret,"short read")!=0)
33 | {
34 | if (system("httping -q -c 1 -t 4 -l "$2" --divert-connect "$1)==0)
35 | {
36 | addlist=-1;
37 | break;
38 | }else{
39 | print("doname rst autoaddip "$1" "$2);
40 | addlist=1;
41 | }
42 | }
43 | else if (index(ret,"timeout")!=0)
44 | {
45 | while ((cmd | getline ret) > 0)
46 | {
47 | if (index(ret,"timeout")!=0)
48 | {
49 | print("direct so slow autoaddip "$1" "$2);
50 | addlist=1;
51 | slow=1;
52 | }
53 | }
54 | }else if (index(ret,"SSL handshake error: (null)")!=0)
55 | {
56 | if(system("curl -m 10 --resolve "$2":443:"$1" https://"$2" -o /dev/null 2>/dev/null")==0){
57 | addlist=-1;
58 | break;
59 | }
60 | }else if (index(ret,"Connection refused")!=0){
61 | print("direct Connection refused autoaddip"$1" "$2);
62 | addlist=1;
63 | }
64 | }
65 | close(cmd);
66 |
67 | if (addlist!=1)
68 | {
69 | if (addlist==0){
70 | split(ret, c,"[ /]+");
71 | print(c[6]);
72 | if (c[6]=="failed,"){
73 | print("can not connect autoaddip "$1" "$2);
74 | addlist=1;
75 | }
76 | else if (c[6]+0>10000)
77 | {
78 | print("direct ssl so slow autoaddip "$1" "$2);
79 | addlist=1;
80 | }else{
81 | addlist=-1;}
82 | }
83 | if (addlist==-1)
84 | {
85 | "ipset test china "$1" 2>&1"| getline ipset;
86 | close("ipset test china "$1" 2>&1");
87 | if (index(ipset,"Warning")==0){
88 | while (("ping -c 5 -q -A "$1 | getline ret) > 0)
89 | {
90 | if (index(ret,"packet loss")!=0)
91 | {
92 | split(ret, p,"[ ]+");
93 | if (p[4]>0 && p[4]<5)
94 | {
95 | print("ping packet loss autoaddip "$1" "$2);
96 | pingloss=1;
97 | addlist=1;
98 | }else{pingloss=0;}
99 | break;
100 | }
101 | }
102 | close("ping -c 5 -q "$1);
103 | }
104 | }
105 | }
106 | ERRNO="";
107 | if (pingloss!=1){
108 | getline drop< "/tmp/run/"$2;
109 | close("/tmp/run/"$2);
110 | }
111 | if (ERRNO) {addlist=0;next;}
112 | if (addlist==1){
113 | system("ipset add gfwlist "$1);
114 | while ((cmd | getline ret) > 0)
115 | {
116 | if (addlist==1)
117 | {
118 | if (index(ret,"short read")!=0)
119 | {
120 | system("ipset del gfwlist "$1);
121 | print("doname proxy rst autodelip "$1" "$2);
122 | addlist=-2;
123 | }
124 | else if (index(ret,"SSL handshake error: (null)")!=0)
125 | {
126 | if(system("curl -m 10 --resolve "$2":443:"$1" https://"$2" -o /dev/null 2>/dev/null")==0)
127 | {
128 | addlist=2;
129 | }
130 | }
131 | }
132 | }
133 | close(cmd);
134 | if (addlist==1){
135 | split(ret, c,"[ /]+");
136 | print(c[6]);
137 | if (c[6]=="failed,")
138 | {
139 | system("ipset del gfwlist "$1);
140 | print("proxy can not connect autodelip "$1" "$2);
141 | addlist=-2;
142 | }else{
143 | addlist=2;
144 | }
145 | }
146 | }
147 | }END{
148 | if (addlist==2)
149 | { if (pingloss==0){
150 | ERRNO="";
151 | getline drop< "/tmp/run/"$2;
152 | if (ERRNO) {
153 | system("ipset del gfwlist "$1);
154 | print("cancel add myself "$1" "$2" due to one ip success direct");
155 | }else{
156 | print $1"\n">>"/tmp/run/"$2;
157 | }
158 | close("/tmp/run/"$2);}
159 | }else if (addlist==-1)
160 | {
161 | print($1" "$2" direct success");
162 | while ((getline ret< "/tmp/run/"$2) > 0)
163 | {
164 | if (ret!=""){
165 | system("ipset del gfwlist "ret);
166 | print("cancel add someone "ret" "$2" due to me"$1" success direct");
167 | }
168 | }
169 | close("/tmp/run/"$2);
170 | system("rm /tmp/run/"$2" 2>/dev/null");
171 | print($1" del "$2);
172 | }else if (addlist==-2)
173 | {
174 | system("sleep 10");
175 | while ((getline ret< "/tmp/run/"$2) > 0)
176 | {
177 | if (ret!=""){
178 | system("ipset add gfwlist "$1);
179 | print("add "ret" "$2" due to one ip success proxy");
180 | break;}
181 | }
182 | close("/tmp/run/"$2);
183 | }}'
--------------------------------------------------------------------------------