├── .gitignore ├── findhost_test.go ├── client.go └── server.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | # Output of the go coverage tool, specifically when used with LiteIDE 27 | *.out 28 | -------------------------------------------------------------------------------- /findhost_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | ) 7 | 8 | func TestRegexp(t *testing.T) { 9 | hostPattern, err := regexp.Compile(`(?:[A-Za-z]+(?:\+[A-Za-z+]+)?://)?(?:[a-zA-Z0-9._-]+(?::[^@]*)?@)?\b([^/?#]+)\b`) 10 | if err != nil { 11 | t.Errorf("could not compile regexp:%s", err) 12 | } 13 | 14 | var url string 15 | var host [][]byte 16 | 17 | url = "http://www.baidu.com" 18 | host = hostPattern.FindSubmatch([]byte(url)) 19 | if string(host[1]) != "www.baidu.com" { 20 | t.Errorf("could not get correct host from %s. it returned [%s]", url, host) 21 | } 22 | 23 | url = "http://www.baidu.com/" 24 | host = hostPattern.FindSubmatch([]byte(url)) 25 | if string(host[1]) != "www.baidu.com" { 26 | t.Errorf("could not get correct host from %s. it returned [%s]", url, host) 27 | } 28 | 29 | url = "http://www.baidu.com?q=abcd" 30 | host = hostPattern.FindSubmatch([]byte(url)) 31 | if string(host[1]) != "www.baidu.com" { 32 | t.Errorf("could not get correct host from %s. it returned [%s]", url, host) 33 | } 34 | 35 | url = "http://www.baidu.com/?q=abcd" 36 | host = hostPattern.FindSubmatch([]byte(url)) 37 | if string(host[1]) != "www.baidu.com" { 38 | t.Errorf("could not get correct host from %s. it returned [%s]", url, host) 39 | } 40 | 41 | url = "www.baidu.com" 42 | host = hostPattern.FindSubmatch([]byte(url)) 43 | if string(host[1]) != "www.baidu.com" { 44 | t.Errorf("could not get correct host from %s. it returned [%s]", url, host) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "net" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/golang/glog" 14 | ) 15 | 16 | var options = &struct { 17 | listen string 18 | dnsServer string 19 | domain string 20 | readTimeout uint64 21 | dnsBatchSize int 22 | }{} 23 | 24 | var dnsServerAddr *net.UDPAddr 25 | var hostPattern *regexp.Regexp 26 | 27 | func init() { 28 | flag.StringVar(&options.listen, "listen", "127.0.0.1:8080", "default 127.0.0.1:8080") 29 | flag.StringVar(&options.dnsServer, "dns", "", "dns server, like 192.168.0.1:53") 30 | flag.StringVar(&options.domain, "domain", "", "domain required, like yourdomain.me") 31 | flag.Uint64Var(&options.readTimeout, "timeout", 10000, "timeout(ms) read from proxy server") 32 | flag.IntVar(&options.dnsBatchSize, "dns-batch-size", 500, "dns request maybe unsuccessful if larger than this") 33 | flag.Parse() 34 | 35 | if options.domain == "" { 36 | flag.Usage() 37 | glog.Fatalln("domain requrired") 38 | } 39 | 40 | if options.dnsServer == "" { 41 | flag.Usage() 42 | glog.Fatalln("dns server requrired") 43 | } 44 | } 45 | 46 | func init() { 47 | var err error 48 | hostPattern, err = regexp.Compile(`(?:[A-Za-z]+(?:\+[A-Za-z+]+)?://)?(?:[a-zA-Z0-9._-]+(?::[^@]*)?@)?\b([^/?#]+)\b`) 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | dnsServerAddr, err = net.ResolveUDPAddr("udp", options.dnsServer) 54 | if err != nil { 55 | panic(err) 56 | } 57 | } 58 | 59 | func sendBuffer(conn *net.TCPConn, buffer []byte) error { 60 | glog.V(10).Infof("content sent to %s:%s", conn.RemoteAddr(), buffer) 61 | for { 62 | n, err := conn.Write(buffer) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | if n == len(buffer) { 68 | return nil 69 | } 70 | buffer = buffer[n:] 71 | } 72 | } 73 | 74 | func passBetweenProxyServerAndRealClient(dnsConn *net.UDPConn, conn *net.TCPConn, closed *bool) { 75 | client := dnsConn.LocalAddr() 76 | streamPool := map[uint32][]byte{} 77 | nextStreamIdx := uint32(1) 78 | 79 | //timeout := time.Millisecond * time.Duration(options.readTimeout) 80 | 81 | buffer1 := make([]byte, 65535) 82 | for { 83 | //dnsConn.SetReadDeadline(time.Now().Add(timeout)) 84 | n, err := dnsConn.Read(buffer1) 85 | 86 | if err != nil { 87 | glog.Errorf("client[%s] read from proxy server error: %s", client, err) 88 | *closed = true 89 | return 90 | } 91 | if glog.V(9) { 92 | glog.Infof("client[%s] read %d bytes from proxy server", client, n) 93 | } 94 | 95 | buffer := make([]byte, n-4) 96 | copy(buffer, buffer1[4:]) 97 | 98 | streamIdx := binary.BigEndian.Uint32(buffer1[:4]) 99 | if glog.V(9) { 100 | glog.Infof("client[%s] streamIdx %d", client, streamIdx) 101 | } 102 | 103 | if streamIdx == 0 { 104 | *closed = true 105 | return 106 | } 107 | 108 | if streamIdx == nextStreamIdx { 109 | err := sendBuffer(conn, buffer) 110 | if err != nil { 111 | glog.Errorf("client[%s] write back to real client error:%s", client, err) 112 | } else { 113 | nextStreamIdx++ 114 | glog.V(9).Infof("client[%s] write %d response to real client", client, len(buffer)) 115 | } 116 | } else { 117 | streamPool[streamIdx] = buffer 118 | } 119 | 120 | for { 121 | if buffer, ok := streamPool[nextStreamIdx]; ok { 122 | err := sendBuffer(conn, buffer) 123 | if err != nil { 124 | glog.Errorf("client[%s] write back to real client error:%s", client, err) 125 | } else { 126 | glog.V(9).Infof("client[%s] write %d response to real client with %d streamIdx", client, len(buffer), streamIdx) 127 | delete(streamPool, nextStreamIdx) 128 | nextStreamIdx++ 129 | } 130 | } else { 131 | break 132 | } 133 | } 134 | } 135 | } 136 | 137 | // the startPositon|FragmentSize Protocol 138 | // host is nil if start is not 0 139 | func fakeDNSRequestEncode(buffer, host []byte, start int) []byte { 140 | rst := make([]byte, 65535) 141 | if glog.V(9) { 142 | glog.Infof("fakeDNSRequestEncode buffer size: %d", len(buffer)) 143 | glog.Infof("fakeDNSRequestEncode start postion: %d", start) 144 | } 145 | 146 | now := time.Now().UnixNano() 147 | domain := fmt.Sprintf("%d.%s", now, options.domain) 148 | 149 | binary.BigEndian.PutUint16(rst, uint16(now)) 150 | var b uint16 151 | b = 0 152 | //b |= (0 << 15) //QR 153 | //b |= (0 << 11) //OPcode 154 | //b |= (0 << 10) //AA 155 | //b |= (0 << 9) //TC 156 | b |= (1 << 8) //RD 157 | //b |= (0 << 7) //RA 158 | //b |= 0 //rcode 159 | binary.BigEndian.PutUint16(rst[2:], b) 160 | 161 | binary.BigEndian.PutUint16(rst[4:], 1) 162 | binary.BigEndian.PutUint16(rst[6:], 0) 163 | binary.BigEndian.PutUint16(rst[8:], 0) 164 | binary.BigEndian.PutUint16(rst[10:], 0) 165 | 166 | offset := 12 167 | 168 | for _, part := range strings.Split(domain, ".") { 169 | rst[offset] = uint8(len(part)) 170 | offset += 1 171 | copy(rst[offset:], []byte(part)) 172 | offset += len(part) 173 | } 174 | rst[offset] = 0 175 | offset += 1 176 | 177 | binary.BigEndian.PutUint16(rst[offset:], 1) 178 | offset += 2 179 | binary.BigEndian.PutUint16(rst[offset:], 1) 180 | offset += 2 181 | 182 | // fragment start position 183 | binary.BigEndian.PutUint32(rst[offset:], uint32(start)) 184 | offset += 4 185 | 186 | // put host length and host 187 | if start == 0 { 188 | binary.BigEndian.PutUint32(rst[offset:], uint32(len(host))) 189 | offset += 4 190 | copy(rst[offset:], host) 191 | offset += len(host) 192 | } 193 | 194 | copy(rst[offset:], buffer) 195 | 196 | return rst[:offset+len(buffer)] 197 | } 198 | 199 | /* 200 | set error if could not get correct host 201 | return nil,nil if first line is not covered yet 202 | */ 203 | func getHostFromFirstRequestBuffer(buffer []byte) ([]byte, error) { 204 | if glog.V(10) { 205 | glog.Infof("try to get host from %s", string(buffer)) 206 | } 207 | urlStart := 0 208 | for i := 0; i < len(buffer); i++ { 209 | if buffer[i] == ' ' { 210 | if urlStart == 0 { 211 | urlStart = i + 1 212 | } else { 213 | rst := hostPattern.FindSubmatch(buffer[urlStart:i]) 214 | if len(rst) != 2 { 215 | return nil, fmt.Errorf("could not get host from %s", string(buffer[:i])) 216 | } 217 | return rst[1], nil 218 | } 219 | } 220 | } 221 | return nil, nil 222 | } 223 | 224 | /* 225 | get stream from real client, and parse HOST from the first line. 226 | then send the stream to proxy server slice by slice 227 | */ 228 | func p(conn *net.TCPConn) { 229 | defer glog.Infof("connection to real client[%s] is closed after process goroutine returns", conn.RemoteAddr()) 230 | defer conn.Close() 231 | 232 | var err error 233 | 234 | dnsConn, err := net.DialUDP("udp", nil, dnsServerAddr) 235 | if err != nil { 236 | glog.Errorf("could not dial to %s", dnsServerAddr) 237 | return 238 | } 239 | 240 | var n, offset int 241 | var host []byte 242 | host = nil 243 | var previousBuffer, buffer []byte 244 | var closed bool 245 | offset = 0 246 | closed = false 247 | go passBetweenProxyServerAndRealClient(dnsConn, conn, &closed) 248 | 249 | buffer = make([]byte, options.dnsBatchSize) 250 | previousBuffer = make([]byte, 0) 251 | for closed == false { 252 | n, err = conn.Read(buffer) 253 | if err == io.EOF { 254 | //TODO curl could send EOF before server when keep-alive. we need pass the message to proxy server 255 | if glog.V(9) { 256 | glog.Infof("%s real client send EOF before server closes", conn.RemoteAddr()) 257 | } 258 | return 259 | } 260 | 261 | // get host from first line 262 | if host == nil { 263 | buffer = append(previousBuffer, buffer[:n]...) 264 | host, err = getHostFromFirstRequestBuffer(buffer) 265 | 266 | if err != nil { 267 | glog.Errorf("could not get host from %s: %s", buffer, err) 268 | return 269 | } 270 | 271 | // url is longer than options.dnsBatchSize 272 | if host == nil { 273 | previousBuffer = append(previousBuffer, buffer[:n]...) 274 | continue 275 | } 276 | previousBuffer = nil 277 | glog.V(2).Infof("get host[%s] from request", host) 278 | } 279 | 280 | dnsReqeust := fakeDNSRequestEncode(buffer, host, offset) 281 | offset += len(buffer) 282 | dnsConn.Write(dnsReqeust) 283 | } 284 | } 285 | 286 | func main() { 287 | tcpAddr, err := net.ResolveTCPAddr("tcp", options.listen) 288 | if err != nil { 289 | glog.Fatalf("could not resolve tcp address[%s]: %s", options.listen, err) 290 | } 291 | 292 | listener, err := net.ListenTCP("tcp", tcpAddr) 293 | if err != nil { 294 | glog.Fatalf("could not listen on %s: %s", options.listen, err) 295 | } 296 | glog.Infof("listen on %s", options.listen) 297 | 298 | for { 299 | conn, err := listener.AcceptTCP() 300 | if err != nil { 301 | glog.Errorf("Accept error: %s", err) 302 | } else { 303 | go p(conn) 304 | } 305 | } 306 | } 307 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "flag" 6 | "io" 7 | "net" 8 | "regexp" 9 | "sync" 10 | "time" 11 | 12 | "github.com/golang/glog" 13 | ) 14 | 15 | var hostRegexp = regexp.MustCompile(":\\d+$") 16 | var cblockmu sync.Mutex 17 | var DNSconn *net.UDPConn 18 | 19 | type clientBlock struct { 20 | lastUpdate time.Time 21 | addr *net.UDPAddr 22 | originalRequests [][]byte 23 | requestSlices map[uint32][]byte 24 | nextOffset uint32 25 | host []byte 26 | conn *net.TCPConn 27 | mu sync.Mutex 28 | co *sync.Cond 29 | } 30 | 31 | var clientBlocksMap map[string]*clientBlock = make(map[string]*clientBlock) 32 | 33 | var options = &struct { 34 | bind string 35 | cleanInterval int 36 | expire int 37 | udpBatchSize int 38 | }{} 39 | 40 | func init() { 41 | flag.StringVar(&options.bind, "bind", "0.0.0.0:53", "to which address faked dns server bind") 42 | flag.IntVar(&options.cleanInterval, "clean-interval", 60, "") 43 | flag.IntVar(&options.expire, "expire", 300, "expire time (s)") 44 | flag.IntVar(&options.udpBatchSize, "udp-batch-size", 4096, "udp package could not be too long") 45 | flag.Parse() 46 | } 47 | 48 | func cleanFragments() { 49 | for { 50 | if glog.V(5) { 51 | glog.Infof("clean expired fragments. requestFragmentsMap length : %d", len(clientBlocksMap)) 52 | } 53 | for client, fragments := range clientBlocksMap { 54 | if glog.V(5) { 55 | glog.Infof("client[%s] last update at %s", client, fragments.lastUpdate) 56 | } 57 | if fragments.lastUpdate.Add(time.Second * time.Duration(options.expire)).Before(time.Now()) { 58 | fragments.co.Signal() 59 | glog.Infof("delete client[%s] from map", client) 60 | cblockmu.Lock() 61 | delete(clientBlocksMap, client) 62 | cblockmu.Unlock() 63 | } 64 | } 65 | time.Sleep(time.Duration(options.cleanInterval) * time.Second) 66 | } 67 | } 68 | 69 | func abstractRealContentFromRawRequest(rawRequest []byte) (uint32, []byte) { 70 | questionNumber := int(binary.BigEndian.Uint16(rawRequest[4:])) 71 | offset := 12 72 | for i := 0; i < questionNumber; i++ { 73 | for { 74 | byteCount := int(rawRequest[offset]) 75 | if byteCount == 0 { 76 | offset++ 77 | break 78 | } 79 | offset += 1 + byteCount 80 | } 81 | } 82 | offset += 4 83 | start := binary.BigEndian.Uint32(rawRequest[offset:]) 84 | return start, append([]byte{}, rawRequest[offset+4:]...) 85 | } 86 | 87 | func connectHost(host []byte) *net.TCPConn { 88 | if !hostRegexp.Match(host) { 89 | host = append(host, ":80"...) 90 | } 91 | serverAddr, err := net.ResolveTCPAddr("tcp", string(host)) 92 | if err != nil { 93 | glog.Errorf("could not resove address[%s]", host) 94 | return nil 95 | } 96 | 97 | tcpConn, err := net.DialTCP("tcp", nil, serverAddr) 98 | if err != nil { 99 | glog.Errorf("could not dial to address[%s]", host) 100 | return nil 101 | } 102 | return tcpConn 103 | } 104 | 105 | func sendBuffer(conn *net.TCPConn, buffer []byte) error { 106 | glog.V(9).Infof("content sent to %s:%s", conn.RemoteAddr(), buffer) 107 | for { 108 | n, err := conn.Write(buffer) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | if n == len(buffer) { 114 | return nil 115 | } 116 | buffer = buffer[n:] 117 | } 118 | } 119 | 120 | func passBetweenRealServerAndProxyClient(conn *net.TCPConn, dnsRemoteAddr *net.UDPAddr) { 121 | buffer := make([]byte, options.udpBatchSize) 122 | streamIdx := 1 123 | for { 124 | n, err := conn.Read(buffer[4:]) 125 | if err == io.EOF { 126 | glog.V(2).Infof("client[%s] connection[%s] closed from server", dnsRemoteAddr, conn.RemoteAddr()) 127 | conn.Close() 128 | DNSconn.Write([]byte{0, 0, 0, 0}) 129 | return 130 | } 131 | if err != nil { 132 | glog.Errorf("read from real server error:%s", err) 133 | return 134 | } 135 | glog.V(9).Infof("client[%s] connection[%s] read %d bytes response", dnsRemoteAddr, conn.RemoteAddr(), n) 136 | binary.BigEndian.PutUint32(buffer, uint32(streamIdx)) 137 | _, err = DNSconn.WriteToUDP(buffer[:4+n], dnsRemoteAddr) 138 | if err != nil { 139 | glog.Errorf("client[%s] send to proxy client error:%s", dnsRemoteAddr, err) 140 | } else if glog.V(9) { 141 | glog.Infof("client[%s] send to proxy client with slice index %d", dnsRemoteAddr, streamIdx) 142 | } 143 | streamIdx++ 144 | } 145 | } 146 | 147 | /* 148 | parse host and build connection; 149 | loop requestSlices and send to real server 150 | */ 151 | func processClientRequest(client string) { 152 | glog.V(9).Infof("process client %s", client) 153 | clientBlock, _ := clientBlocksMap[client] 154 | glog.V(9).Infof("client[%s] there is %d original requests", client, len(clientBlock.originalRequests)) 155 | 156 | for { 157 | clientBlock.mu.Lock() 158 | if len(clientBlock.originalRequests) == 0 { 159 | clientBlock.mu.Unlock() 160 | //如果request slice在路上丢失, 可能会导致goroutine永远不醒来不return. 161 | //所以在clear expire的时候, 需要Signal 162 | glog.V(9).Infoln("no new original requests. wait for new ones.") 163 | clientBlock.co.L.Lock() 164 | clientBlock.co.Wait() 165 | glog.V(9).Infoln("i am awake, get new original requests.") 166 | clientBlock.co.L.Unlock() 167 | continue 168 | } 169 | for idx, originalRequest := range clientBlock.originalRequests { 170 | glog.V(9).Infof("client[%s] process the %d original request", client, idx) 171 | offset, content := abstractRealContentFromRawRequest(originalRequest) 172 | glog.V(9).Infof("client[%s] offset %d", client, offset) 173 | if offset == 0 { 174 | hostLength := binary.BigEndian.Uint32(content[:4]) 175 | clientBlock.host = append([]byte{}, content[4:hostLength+4]...) 176 | 177 | glog.Infof("client[%s] got host[%s]", client, clientBlock.host) 178 | 179 | clientBlock.requestSlices[offset] = content[4+hostLength:] 180 | 181 | } else { 182 | clientBlock.requestSlices[offset] = content 183 | } 184 | } 185 | 186 | clientBlock.originalRequests = make([][]byte, 0) 187 | clientBlock.mu.Unlock() 188 | 189 | if clientBlock.conn == nil { 190 | conn := connectHost(clientBlock.host) 191 | if conn != nil { 192 | glog.V(5).Infof("client[%s] connected to host[%s]", client, clientBlock.host) 193 | clientBlock.conn = conn 194 | go passBetweenRealServerAndProxyClient(conn, clientBlock.addr) 195 | } else { 196 | glog.Errorf("client[%s] could not connect to host[%s]", client, clientBlock.host) 197 | //cblockmu.Lock() 198 | //delete(clientBlocksMap, client) 199 | //cblockmu.Unlock() 200 | return 201 | } 202 | } 203 | 204 | for { 205 | if content, ok := clientBlock.requestSlices[clientBlock.nextOffset]; ok { 206 | err := sendBuffer(clientBlock.conn, content) 207 | if err != nil { 208 | //TODO if conn has been closed ? or need retry? 209 | glog.Errorf("could not send request to real server[%s]: %s", clientBlock.conn.RemoteAddr(), err) 210 | return 211 | } else { 212 | glog.V(9).Infof("client[%s] request slice sent to real server", client) 213 | //TODO delete in a loop?? 214 | //delete(clientBlock.requestSlices, clientBlock.nextOffset) 215 | clientBlock.nextOffset += uint32(len(content)) 216 | } 217 | } else { 218 | break 219 | } 220 | } 221 | } 222 | } 223 | 224 | func main() { 225 | udpAddr, err := net.ResolveUDPAddr("udp", options.bind) 226 | if err != nil { 227 | panic(err) 228 | } 229 | 230 | DNSconn, err = net.ListenUDP("udp", udpAddr) 231 | 232 | if err != nil { 233 | panic(err) 234 | } 235 | glog.Infof("listen on %s", options.bind) 236 | 237 | go cleanFragments() 238 | 239 | buffer := make([]byte, 65535) 240 | for { 241 | n, c, err := DNSconn.ReadFromUDP(buffer) 242 | if err != nil { 243 | glog.Errorf("read from %s error: %s", c, err) 244 | continue 245 | } 246 | 247 | if glog.V(9) { 248 | glog.Infof("read %d bytes from %s", n, c) 249 | } 250 | 251 | request := make([]byte, n) 252 | copy(request, buffer[:n]) 253 | 254 | cblockmu.Lock() 255 | // 琐应该放在最外面. 避免更新value的同时, 它也被删除 256 | if value, ok := clientBlocksMap[c.String()]; ok { 257 | if glog.V(5) { 258 | glog.Infof("client[%s] has been in map", c) 259 | } 260 | value.lastUpdate = time.Now() 261 | value.mu.Lock() 262 | value.originalRequests = append(value.originalRequests, request) 263 | value.mu.Unlock() 264 | value.co.Signal() 265 | } else { 266 | if glog.V(5) { 267 | glog.Infof("client[%s] is not in map", c) 268 | } 269 | originalRequests := [][]byte{request} 270 | clientBlocksMap[c.String()] = &clientBlock{ 271 | lastUpdate: time.Now(), 272 | originalRequests: originalRequests, 273 | requestSlices: map[uint32][]byte{}, 274 | nextOffset: 0, 275 | addr: c, 276 | host: nil, 277 | conn: nil, 278 | co: sync.NewCond(&sync.Mutex{}), 279 | } 280 | go processClientRequest(c.String()) 281 | } 282 | cblockmu.Unlock() 283 | } 284 | } 285 | --------------------------------------------------------------------------------