├── .gitignore ├── .travis.yml ├── Makefile ├── resource.go ├── iploc_test.go ├── LICENSE ├── ip.go ├── cmd ├── iploc-gen │ └── iploc-gen.go ├── iploc-fetch │ └── iploc-fetch.go └── iploc-conv │ └── iploc-conv.go ├── indexes.go ├── detail.go ├── README.md ├── iploc.go └── parser.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.9 4 | install: 5 | - go get -u github.com/spf13/pflag 6 | - go get -u golang.org/x/text/transform 7 | - go get -u golang.org/x/text/encoding/simplifiedchinese 8 | - go get -u github.com/google/btree 9 | - make install 10 | script: 11 | - make clean -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IPLOC_CONV = $(GOPATH)/bin/iploc-conv 2 | IPLOC_FETCH := $(GOPATH)/bin/iploc-fetch 3 | IPLOC_GEN = $(GOPATH)/bin/iploc-gen 4 | 5 | 6 | .PHONY: all test clean install prepare 7 | 8 | all: install prepare test 9 | 10 | clean: 11 | rm -f $(IPLOC_CONV) $(IPLOC_FETCH) $(IPLOC_GEN) 12 | rm -f qqwry.dat 13 | 14 | install: 15 | cd cmd/iploc-conv; go install 16 | cd cmd/iploc-fetch; go install 17 | cd cmd/iploc-gen; go install 18 | 19 | prepare: 20 | iploc-fetch ./qqwry.gbk.dat -q 21 | iploc-conv -s qqwry.gbk.dat -d qqwry.dat -n -q 22 | rm qqwry.gbk.dat 23 | 24 | test: 25 | go test -v 26 | 27 | bench: 28 | go test -bench=. -------------------------------------------------------------------------------- /resource.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | type resReadCloser interface { 9 | Read(b []byte) (n int, err error) 10 | ReadAt(b []byte, off int64) (n int, err error) 11 | Close() error 12 | } 13 | 14 | type resource struct { 15 | data []byte 16 | seek int64 17 | } 18 | 19 | func (res *resource) Read(b []byte) (n int, err error) { 20 | if len(b) == 0 { 21 | return 22 | } 23 | if max := len(res.data); len(b) > max { 24 | b = b[:max] 25 | } 26 | n, err = res.ReadAt(b, res.seek) 27 | res.seek += int64(n) 28 | return 29 | } 30 | 31 | func (res *resource) ReadAt(b []byte, off int64) (n int, err error) { 32 | if off < 0 { 33 | return 0, fmt.Errorf("negative offset: %d", off) 34 | } else if len(b) == 0 || off >= int64(len(res.data)) { 35 | return 0, nil 36 | } 37 | copy(b, res.data[off:]) 38 | if int64(len(b))+off > int64(len(res.data)) { 39 | return int(int64(len(res.data)) - off), io.EOF 40 | } 41 | return len(b), nil 42 | } 43 | 44 | func (res *resource) Close() error { 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /iploc_test.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | const DATPath = "qqwry.dat" 10 | 11 | func TestFind(t *testing.T) { 12 | loc, err := Open(DATPath) 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | 17 | detail1 := loc.Find("4.4") 18 | detail2, _ := Find(DATPath, "4.0.0.4") 19 | 20 | if detail1.Start.Compare(detail2.Start) != 0 || detail1.End.Compare(detail2.End) != 0 || detail1.String() != detail2.String() { 21 | t.Fatal("error") 22 | } 23 | } 24 | 25 | func BenchmarkFind(b *testing.B) { 26 | b.StopTimer() 27 | loc, err := Open(DATPath) 28 | if err != nil { 29 | b.Fatal(err) 30 | } 31 | rand.Seed(time.Now().UnixNano()) 32 | b.StartTimer() 33 | for i := 0; i < b.N; i++ { 34 | loc.FindUint(rand.Uint32()) 35 | } 36 | } 37 | 38 | func BenchmarkFindUnIndexed(b *testing.B) { 39 | b.StopTimer() 40 | loc, err := OpenWithoutIndexes(DATPath) 41 | if err != nil { 42 | b.Fatal(err) 43 | } 44 | rand.Seed(time.Now().UnixNano()) 45 | b.StartTimer() 46 | for i := 0; i < b.N; i++ { 47 | loc.FindUint(rand.Uint32()) 48 | } 49 | } 50 | 51 | func BenchmarkFindParallel(b *testing.B) { 52 | b.StopTimer() 53 | loc, err := Open(DATPath) 54 | if err != nil { 55 | b.Fatal(err) 56 | } 57 | rand.Seed(time.Now().UnixNano()) 58 | b.StartTimer() 59 | b.RunParallel(func(pb *testing.PB) { 60 | for pb.Next() { 61 | if loc.FindUint(rand.Uint32()) == nil { 62 | b.Fatal("error") 63 | } 64 | } 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Kayon 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /ip.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | const ( 12 | IPLen = 4 13 | ) 14 | 15 | type IP [IPLen]byte 16 | 17 | func (ip IP) Bytes() []byte { 18 | return ip[:] 19 | } 20 | 21 | func (ip IP) ReverseBytes() []byte { 22 | var b [4]byte 23 | copy(b[:], ip[:]) 24 | for i, j := 0, len(b)-1; i < len(b)/2; i, j = i+1, j-1 { 25 | b[i], b[j] = b[j], b[i] 26 | } 27 | return b[:] 28 | } 29 | 30 | func (ip IP) String() string { 31 | return fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]) 32 | } 33 | 34 | func (ip IP) Uint() uint32 { 35 | return binary.BigEndian.Uint32(ip[:]) 36 | } 37 | 38 | // Compare like bytes.Compare 39 | // The result will be 0 if == a, -1 if < a, and +1 if > a. 40 | func (ip IP) Compare(a IP) int { 41 | return bytes.Compare(ip[:], a[:]) 42 | } 43 | 44 | func ParseIP(s string) (ip IP, err error) { 45 | var b = make([]byte, 0, IPLen) 46 | var d uint64 47 | for i, v := range strings.Split(s, ".") { 48 | if d, err = strconv.ParseUint(v, 10, 8); err != nil { 49 | err = fmt.Errorf("invalid IP address %s", s) 50 | return 51 | } 52 | b = append(b, byte(d)) 53 | if i == 3 { 54 | break 55 | } 56 | } 57 | if len(b) == 0 { 58 | err = fmt.Errorf("invalid IP address %s", s) 59 | return 60 | } 61 | 62 | // filling，e.g. 127.1 -> 127.0.0.1 63 | // copy to array, right padding 64 | copy(ip[:], b) 65 | if padding := IPLen - len(b); padding > 0 { 66 | // left padding 67 | if lastIndex := len(b) - 1; b[lastIndex] > 0 { 68 | ip[lastIndex], ip[3] = 0, ip[lastIndex] 69 | } 70 | } 71 | return 72 | } 73 | 74 | func ParseBytesIP(b []byte) (ip IP) { 75 | copy(ip[:], b) 76 | return 77 | } 78 | 79 | func ParseUintIP(u uint32) (ip IP) { 80 | binary.BigEndian.PutUint32(ip[:], u) 81 | return 82 | } 83 | -------------------------------------------------------------------------------- /cmd/iploc-gen/iploc-gen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strconv" 10 | "text/template" 11 | 12 | "github.com/kayon/iploc" 13 | flag "github.com/spf13/pflag" 14 | ) 15 | 16 | const ( 17 | goFilename = "iploc-binary.go" 18 | goFileTPL = `// generated by iploc-gen 19 | package {{ .Package }} 20 | 21 | import ( 22 | "github.com/kayon/iploc" 23 | ) 24 | 25 | var IPLoc *iploc.Locator 26 | 27 | func init() { 28 | var err error 29 | if IPLoc, err = iploc.{{ .Method }}([]byte({{ .Dat }})); err != nil { 30 | panic(err) 31 | } 32 | }` 33 | ) 34 | 35 | var ( 36 | qqwrySrc string 37 | pkgName string 38 | noIndex bool 39 | help bool 40 | file = template.Must(template.New("file").Parse(goFileTPL)) 41 | ) 42 | 43 | func init() { 44 | flag.StringVarP(&pkgName, "pkg", "p", "main", "package name") 45 | flag.BoolVarP(&noIndex, "no-index", "n", false, "initialization with OpenWithoutIndexes") 46 | flag.BoolVarP(&help, "help", "h", false, "this help") 47 | flag.Usage = func() { 48 | fmt.Fprintf(os.Stderr, "iploc-gen: convert iploc and DAT into Go source code.\nUsage: iploc-gen [DAT path] [arguments]\nOptions:\n") 49 | flag.PrintDefaults() 50 | } 51 | flag.Parse() 52 | qqwrySrc = flag.Arg(0) 53 | 54 | if qqwrySrc == "" || help { 55 | flag.Usage() 56 | if help { 57 | os.Exit(0) 58 | } 59 | os.Exit(1) 60 | } 61 | } 62 | 63 | type Content struct { 64 | Dat string 65 | } 66 | 67 | func main() { 68 | p, err := iploc.NewParser(qqwrySrc, true) 69 | if err != nil { 70 | fmt.Fprintf(os.Stderr, err.Error()) 71 | os.Exit(1) 72 | } 73 | 74 | f, err := os.Create(goFilename) 75 | if err != nil { 76 | fmt.Fprintf(os.Stderr, err.Error()) 77 | os.Exit(1) 78 | } 79 | defer f.Close() 80 | 81 | buffer := bytes.NewBuffer(nil) 82 | cw, _ := zlib.NewWriterLevel(buffer, zlib.BestCompression) 83 | _, err = io.Copy(cw, p.Reader()) 84 | cw.Close() 85 | 86 | if err != nil { 87 | fmt.Fprintf(os.Stderr, err.Error()) 88 | os.Exit(1) 89 | } 90 | 91 | var method = "Load" 92 | if noIndex { 93 | method = "LoadWithoutIndexes" 94 | } 95 | 96 | file.Execute(f, map[string]string{"Package": pkgName, "Dat": strconv.QuoteToASCII(buffer.String()), "Method": method}) 97 | } 98 | -------------------------------------------------------------------------------- /indexes.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "encoding/gob" 5 | 6 | "github.com/google/btree" 7 | ) 8 | 9 | type dataItem = btree.Item 10 | 11 | // 索引时 12 | // 0: ip start 13 | // 1: ip end 14 | // 2: country position 15 | // 3: region position 16 | // 无索引时 17 | // 2: ip end position 18 | // 3: n/a 19 | type indexItem [4]uint32 20 | 21 | func (i indexItem) Less(than btree.Item) bool { 22 | // 默认降序 1:End < 0:Start 23 | if v, ok := than.(indexItem); ok { 24 | return i[1] < v[0] 25 | 26 | } 27 | return i[1] < than.(indexItemAscend)[0] 28 | } 29 | 30 | // indexItem 升序 31 | type indexItemAscend [4]uint32 32 | 33 | func (i indexItemAscend) Less(than btree.Item) bool { 34 | // 升序 0:End < 0:Start 35 | if v, ok := than.(indexItem); ok { 36 | return i[0] < v[0] 37 | 38 | } 39 | return i[0] < than.(indexItemAscend)[0] 40 | } 41 | 42 | type indexes struct { 43 | index *btree.BTree 44 | indexMid indexItem 45 | locations map[uint32][]byte 46 | } 47 | 48 | func (idx *indexes) indexOf(u uint32) (hit indexItem) { 49 | // 对比中间值决定高低顺序，提升查询速度 50 | if u > idx.indexMid[1] { 51 | idx.index.DescendLessOrEqual(indexItem{0, u}, func(i btree.Item) bool { 52 | hit = i.(indexItem) 53 | return false 54 | }) 55 | } else if u < idx.indexMid[0] { 56 | idx.index.AscendGreaterOrEqual(indexItemAscend{u}, func(i btree.Item) bool { 57 | hit = i.(indexItem) 58 | return false 59 | }) 60 | } else { 61 | hit = idx.indexMid 62 | } 63 | return 64 | } 65 | 66 | func (idx *indexes) getLocation(i, j uint32) Location { 67 | return parseLocation(idx.locations[i], idx.locations[j]) 68 | } 69 | 70 | func newIndexes(p *Parser) *indexes { 71 | idx := &indexes{ 72 | index: btree.New(10), 73 | } 74 | idx.locations = make(map[uint32][]byte) 75 | 76 | var ( 77 | item indexItem 78 | raw LocationRaw 79 | mid = int(p.Count()) >> 1 80 | has bool 81 | ) 82 | 83 | p.IndexRange(func(i int, start, end, pos uint32) bool { 84 | item = indexItem{start, end, pos} 85 | raw = p.ReadLocationRaw(int64(pos)) 86 | if raw.Text[0] != nil { 87 | if _, has = idx.locations[raw.Pos[0]]; !has { 88 | idx.locations[raw.Pos[0]] = raw.Text[0] 89 | } 90 | } 91 | if raw.Text[1] != nil { 92 | if _, has = idx.locations[raw.Pos[1]]; !has { 93 | idx.locations[raw.Pos[1]] = raw.Text[1] 94 | } 95 | } 96 | item[2] = raw.Pos[0] 97 | item[3] = raw.Pos[1] 98 | if i == mid { 99 | idx.indexMid = item 100 | } 101 | idx.index.ReplaceOrInsert(item) 102 | return true 103 | }) 104 | return idx 105 | } 106 | 107 | func init() { 108 | gob.Register([][4]uint32{}) 109 | gob.Register(map[uint32][]byte{}) 110 | } 111 | -------------------------------------------------------------------------------- /detail.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | var provincePrefix = map[int32]map[int32]int{'北': {'京': 0}, '天': {'津': 0}, '河': {'北': 0, '南': 0}, '山': {'东': 0, '西': 0}, '内': {'蒙': 1}, '辽': {'宁': 0}, '吉': {'宁': 0}, '黑': {'龙': 1}, '上': {'海': 0}, '江': {'苏': 0, '西': 0}, '浙': {'江': 0}, '安': {'徽': 0}, '福': {'福': 0}, '湖': {'南': 0, '北': 0}, '广': {'东': 0, '西': 0}, '海': {'南': 0}, '重': {'庆': 0}, '四': {'川': 0}, '贵': {'州': 0}, '云': {'南': 0}, '西': {'藏': 0}, '陕': {'西': 0}, '甘': {'肃': 0}, '青': {'海': 0}, '宁': {'夏': 0}, '新': {'疆': 0}, '香': {'港': 0}, '澳': {'门': 0}, '台': {'湾': 0}} 4 | var adEndSuffix = [2]map[int32]int{{'市': 1, '州': 1, '区': 1, '盟': 1}, {'县': 2, '市': 2, '旗': 2}} 5 | 6 | type Detail struct { 7 | IP IP 8 | Start IP 9 | End IP 10 | Location 11 | } 12 | 13 | func (detail Detail) String() string { 14 | return detail.Location.String() 15 | } 16 | 17 | func (detail Detail) Bytes() []byte { 18 | return detail.Location.Bytes() 19 | } 20 | 21 | func (detail Detail) InIP(ip IP) bool { 22 | return detail.Start.Compare(ip) < 1 && detail.End.Compare(ip) > -1 23 | } 24 | 25 | func (detail Detail) In(rawIP string) bool { 26 | ip, err := ParseIP(rawIP) 27 | if err != nil { 28 | return false 29 | } 30 | return detail.InIP(ip) 31 | } 32 | 33 | func (detail Detail) InUint(uintIP uint32) bool { 34 | return detail.InIP(ParseUintIP(uintIP)) 35 | } 36 | 37 | func (detail *Detail) fill() *Detail { 38 | if detail.Region == "N/A" { 39 | return detail 40 | } 41 | 42 | var ( 43 | rs = []rune(detail.Country) 44 | s [2][]rune 45 | p map[int32]int 46 | ok bool 47 | i, n int 48 | ) 49 | 50 | if p, ok = provincePrefix[rs[0]]; ok { 51 | i, ok = p[rs[1]] 52 | i += 2 53 | } 54 | if !ok { 55 | return detail 56 | } 57 | 58 | detail.Country = "中国" 59 | detail.Province = string(rs[:i]) 60 | 61 | if i >= len(rs) { 62 | if rs[0] == '北' || rs[0] == '天' || rs[0] == '上' || rs[0] == '重' { 63 | detail.City = string(rs[:i-1]) 64 | } 65 | return detail 66 | } 67 | 68 | if rs[i] == '市' { 69 | i++ 70 | detail.City = string(rs[:i]) 71 | } else if rs[i] == '省' { 72 | i++ 73 | } 74 | 75 | for ; i < len(rs); i++ { 76 | s[n] = append(s[n], rs[i]) 77 | if _, ok = adEndSuffix[n][rs[i]]; ok { 78 | if rs[i] != '市' && i + 1 < len(rs) && rs[i+1] == '市' { 79 | continue 80 | } 81 | n++ 82 | } 83 | if n > 1 { 84 | break 85 | } 86 | } 87 | 88 | if detail.City != "" { 89 | detail.County = string(s[0]) 90 | } else { 91 | detail.City = string(s[0]) 92 | detail.County = string(s[1]) 93 | } 94 | 95 | return detail 96 | } 97 | 98 | type Location struct { 99 | Country string 100 | Region string 101 | Province string 102 | City string 103 | County string 104 | raw string 105 | } 106 | 107 | func (location Location) String() string { 108 | return location.raw 109 | } 110 | 111 | func (location Location) Bytes() []byte { 112 | return []byte(location.raw) 113 | } 114 | 115 | func parseLocation(country, region []byte) Location { 116 | location := Location{ 117 | Country: string(country), 118 | Region: string(region), 119 | } 120 | location.raw = location.Country 121 | if region != nil { 122 | location.raw += " " + location.Region 123 | } 124 | return location 125 | } 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # iploc 2 | 3 | [![Build Status](https://travis-ci.org/kayon/iploc.svg?branch=master)](https://travis-ci.org/kayon/iploc) 4 | 5 | 使用纯真IP库 `qqwry.dat`，高性能，线程安全，并对国内数据格式化到省、市、县 6 | 7 | > 需要 go 1.9 或更高 8 | 9 | > 附带的 `qqwry.dat` 为 `UTF-8` 编码 `2018-05-15版本` 10 | 11 | > 不再提供 `qqwry.dat`, 新增命令行工具 `iploc-fetch`, 可在线获取官方最新版本的 `qqwry.dat` 12 | 13 | 14 | ## 安装 15 | 16 | ``` 17 | go get -u github.com/kayon/iploc/... 18 | ``` 19 | 20 | #### 无法安装 `golang.org/x/text` 包，没有梯子使用下面方法 21 | 22 | ``` 23 | $ mkdir -P $GOPATH/src/golang.org/x 24 | cd $GOPATH/src/golang.org/x 25 | git clone https://github.com/golang/text.git 26 | ``` 27 | 28 | ## 获取&更新 qqwry.dat 29 | 30 | #### 1. 下载 `qqwry.dat` 31 | 32 | ##### 方法一：使用命令行工具 [iploc-fetch](#iploc-fetch) 33 | 34 | 由于服务器限制国外IP，只能使用国内网络。 35 | 36 | 下载到当前目录，保存为 `qqwry.gbk.dat` 37 | 38 | ``` 39 | $ iploc-fetch qqwry.gbk.dat 40 | ``` 41 | 42 | ##### 方法二：手动下载 43 | 44 | 在[纯真官网下载](http://www.cz88.net/fox/ipdat.shtml)并安装，复制安装目录中的 `qqwry.dat` 45 | 46 | #### 2. 转换为 `UTF-8` 47 | 48 | 使用命令行工具 [iploc-conv](#iploc-conv) 将刚刚下载的 `qqwry.gbk.dat` 转换为 `UTF-8` 保存为 `qqwry.dat` 49 | 50 | ``` 51 | iploc-conv -s qqwry.gbk.dat -d qqwry.dat 52 | ``` 53 | 54 | 55 | ## Benchmarks 56 | 57 | ``` 58 | // 缓存索引 59 | BenchmarkFind-8 2000000 735 ns/op 136万/秒 60 | // 无索引 61 | BenchmarkFindUnIndexed-8 20000 78221 ns/op 1.2万/秒 62 | // 缓存索引，并发 63 | BenchmarkFindParallel-8 10000000 205 ns/op 487万/秒 64 | ``` 65 | 66 | ## 使用 67 | 68 | ``` 69 | func main() { 70 | loc, err := iploc.Open("qqwry.dat") 71 | if err != nil { 72 | panic(err) 73 | } 74 | detail := loc.Find("8.8.8") // 补全为8.8.0.8, 参考 ping 工具 75 | fmt.Printf("IP:%s; 网段:%s - %s; %s\n", detail.IP, detail.Start, detail.End, detail) 76 | 77 | detail2 := loc.Find("8.8.3.1") 78 | fmt.Printf("%t %t\n", detail.In(detail2.IP.String()), detail.String() == detail2.String()) 79 | 80 | // output 81 | // IP:8.8.0.8; 网段: 8.7.245.0 - 8.8.3.255; 美国 科罗拉多州布隆菲尔德市Level 3通信股份有限公司 82 | // true true 83 | 84 | detail = loc.Find("1.24.41.0") 85 | fmt.Println(detail.String()) 86 | fmt.Println(detail.Country, detail.Province, detail.City, detail.County) 87 | 88 | // output 89 | // 内蒙古锡林郭勒盟苏尼特右旗 联通 90 | // 中国 内蒙古 锡林郭勒盟 苏尼特右旗 91 | 92 | } 93 | ``` 94 | 95 | #### 快捷方法 96 | ##### Find(qqwrySrc, ip string) (*Detail, error) 97 | `iploc.Find` 使用 `OpenWithoutIndexes` 98 | 99 | #### 初始化 100 | ##### Open(qqwrySrc string) (*Locator, error) 101 | 102 | `iploc.Open` 缓存并索引，生成索引需要耗时500毫秒左右，但会带来更高的查询性能 103 | 104 | ##### OpenWithoutIndexes(qqwrySrc string) (*Locator, error) 105 | 106 | `iploc.OpenWithoutIndexes` 只读取文件头做简单检查，无索引 107 | 108 | #### 查询 109 | 110 | ``` 111 | (*Locator).Find(ip string) *Detail 112 | 113 | ``` 114 | > 如果IP不合法，返回 `nil` 115 | 116 | 117 | ## 命令行工具 118 | 119 | #### iploc-conv 120 | 121 | 将原版 `qqwry.dat` 由 `GBK` 转换为 `UTF-8` 122 | 123 | ``` 124 | $ iploc-conv -s src.gbk.dat -d dst.utf8.dat 125 | ``` 126 | 127 | > 新生成的DAT文件保留原数据结构，由于编码原因，文件会增大一点 128 | 129 | > 修正原 qqwry.dat 中几处错误的重定向 (qqwry.dat 2018-05-10)，并将 "CZ88.NET" 替换为 "N/A" 130 | 131 | #### iploc-fetch 132 | 133 | 从纯真官网下载最新 `qqwry.dat` 134 | 135 | 由于服务器限制国外IP，只能使用国内网络。 136 | 137 | ``` 138 | $ iploc-fetch qqwry.gbk.dat 139 | ``` 140 | 141 | > 下载后别忘了使用 `iploc-conv` 转换为 `UTF-8` 142 | 143 | #### iploc-gen 144 | 145 | 创建静态版本的 **iploc** 集成到你的项目中 146 | 147 | `iploc-gen` 会在当前目录创建 iploc-binary.go 文件，拷贝到你的项目中，通过变量名 *IPLoc* 直接可以使用 148 | 149 | ``` 150 | $ iploc-gen path/qqwry.dat 151 | ``` 152 | 153 | > `--pkg` 指定 package name, 默认 main 154 | 155 | > `-n` 使用 `OpenWithoutIndexes` 初始化，无索引 156 | 157 | 158 | ## 静态编译 iploc 和 qqwry.dat 并集成到你的项目中 159 | 160 | 编译后的二进制没有 `qqwry.dat` 依赖，不需要再带着 `qqwry.dat` 一起打包了 161 | 162 | ##### 示例 163 | 164 | 到项目目录 `$GOPATH/src/myproject/` 中 165 | 166 | ``` 167 | $ mkdir myloc && cd myloc 168 | $ iploc-gen path/qqwry.dat --pkg myloc 169 | ``` 170 | 171 | > $GOPATH/src/myproject/main.go 172 | 173 | ``` 174 | package main 175 | 176 | import ( 177 | "fmt" 178 | 179 | "myproject/myloc" 180 | ) 181 | 182 | func main() { 183 | fmt.Println(myloc.IPLoc.Find("8.8.8.8")) 184 | } 185 | ``` -------------------------------------------------------------------------------- /cmd/iploc-fetch/iploc-fetch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "os" 12 | 13 | flag "github.com/spf13/pflag" 14 | "golang.org/x/text/encoding/simplifiedchinese" 15 | "golang.org/x/text/transform" 16 | ) 17 | 18 | const ( 19 | userAgent = "Mozilla/3.0 (compatible; Indy Library)" 20 | urlCopywrite = "http://update.cz88.net/ip/copywrite.rar" 21 | urlQqwryDat = "http://update.cz88.net/ip/qqwry.rar" 22 | ) 23 | 24 | var ( 25 | outputFile string 26 | quiet, help bool 27 | 28 | fileSize uint64 29 | fileSizeW int 30 | ) 31 | 32 | type FetchWriter struct { 33 | n uint64 34 | } 35 | 36 | func (w *FetchWriter) Write(p []byte) (int, error) { 37 | n := len(p) 38 | w.n += uint64(n) 39 | w.progress() 40 | return n, nil 41 | } 42 | 43 | func (w *FetchWriter) progress() { 44 | printf("\rfetch: %6.2f%% %*d/%d bytes", float64(w.n)/float64(fileSize)*100, fileSizeW, w.n, fileSize) 45 | } 46 | 47 | func fetch(key uint32) error { 48 | resp, err := newRequest(urlQqwryDat) 49 | if err != nil { 50 | return err 51 | } 52 | defer resp.Body.Close() 53 | 54 | if uint64(resp.ContentLength) != fileSize { 55 | return fmt.Errorf("the file size of the agreement is different") 56 | } 57 | 58 | var b []byte 59 | buffer := bytes.NewBuffer(b[:]) 60 | 61 | _, err = io.Copy(buffer, io.TeeReader(resp.Body, &FetchWriter{})) 62 | printf("\n") 63 | if err != nil { 64 | return err 65 | } 66 | b = buffer.Bytes() 67 | 68 | // 解码前512字节 69 | for i := 0; i < 0x200; i++ { 70 | key *= 0x805 71 | key++ 72 | key &= 0xFF 73 | b[i] = b[i] ^ byte(key) 74 | } 75 | 76 | r, err := zlib.NewReader(bytes.NewBuffer(b)) 77 | if err != nil { 78 | return err 79 | } 80 | defer r.Close() 81 | 82 | f, err := os.Create(outputFile) 83 | if err != nil { 84 | return err 85 | } 86 | _, err = io.Copy(f, r) 87 | return err 88 | } 89 | 90 | func init() { 91 | flag.BoolVarP(&quiet, "quiet", "q", false, "only output error") 92 | flag.BoolVarP(&help, "help", "h", false, "this help") 93 | flag.CommandLine.SortFlags = false 94 | flag.Parse() 95 | 96 | flag.Usage = func() { 97 | fmt.Fprintf(os.Stderr, "iploc-fetch: fetch qqwry.dat.\nUsage: iploc-fetch [output filename] [arguments]\nOptions:\n") 98 | flag.PrintDefaults() 99 | } 100 | 101 | outputFile = flag.Arg(0) 102 | 103 | if outputFile == "" || help { 104 | flag.Usage() 105 | if help { 106 | os.Exit(0) 107 | } 108 | os.Exit(1) 109 | } 110 | } 111 | 112 | func main() { 113 | resp, err := newRequest(urlCopywrite) 114 | if err != nil { 115 | fmt.Fprintf(os.Stderr, err.Error()) 116 | os.Exit(1) 117 | } 118 | b, err := ioutil.ReadAll(resp.Body) 119 | resp.Body.Close() 120 | 121 | fatal(err) 122 | 123 | s := readVersion(b) 124 | if s == nil { 125 | fatal(fmt.Errorf("invalid file description")) 126 | } 127 | fileSize = uint64(binary.LittleEndian.Uint32(b[12:])) 128 | fileSizeW = len(fmt.Sprint(fileSize)) 129 | key := binary.LittleEndian.Uint32(b[20:]) 130 | 131 | printf("version: %s\n", toUTF8(s)) 132 | printf("fetch: ...") 133 | 134 | fatal(fetch(key)) 135 | } 136 | 137 | func newRequest(urls string) (*http.Response, error) { 138 | req, err := http.NewRequest("GET", urls, nil) 139 | if err != nil { 140 | return nil, err 141 | } 142 | req.Header.Set("User-Agent", userAgent) 143 | client := &http.Client{} 144 | return client.Do(req) 145 | } 146 | 147 | func readVersion(p []byte) []byte { 148 | var start = 24 149 | var end int 150 | 151 | if start >= len(p) { 152 | return nil 153 | } 154 | 155 | // 0x20 ASCII space 156 | for p[start] != 0x20 { 157 | start++ 158 | } 159 | start += 1 160 | end = start 161 | 162 | if end >= len(p) { 163 | return nil 164 | } 165 | 166 | for p[end] != 0x00 { 167 | end++ 168 | } 169 | return p[start:end] 170 | } 171 | 172 | func toUTF8(s []byte) (b []byte) { 173 | r := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder()) 174 | b, _ = ioutil.ReadAll(r) 175 | return 176 | } 177 | 178 | func fatal(err error) { 179 | if err != nil { 180 | fmt.Fprintf(os.Stderr, err.Error()+"\n") 181 | os.Exit(1) 182 | } 183 | } 184 | 185 | func printf(format string, args ...interface{}) { 186 | if quiet { 187 | return 188 | } 189 | fmt.Printf(format, args...) 190 | } 191 | -------------------------------------------------------------------------------- /iploc.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "fmt" 7 | "io/ioutil" 8 | "runtime" 9 | ) 10 | 11 | const Version = "1.0" 12 | 13 | type rangeIterator func(i int, start, end IP) bool 14 | 15 | // Find shorthand for iploc.Open.Find 16 | // not be preload file and without indexed 17 | func Find(qqwrySrc, rawIP string) (*Detail, error) { 18 | loc, err := OpenWithoutIndexes(qqwrySrc) 19 | if err != nil { 20 | return nil, err 21 | } 22 | defer loc.parser.Close() 23 | return loc.Find(rawIP), nil 24 | } 25 | 26 | // Open 生成索引，查询速度快 27 | func Open(qqwrySrc string) (loc *Locator, err error) { 28 | loc = &Locator{} 29 | var parser *Parser 30 | if parser, err = NewParser(qqwrySrc, true); err != nil { 31 | return nil, err 32 | } 33 | loc.indexes = newIndexes(parser) 34 | loc.count = int(parser.count) 35 | return 36 | } 37 | 38 | // OpenWithoutIndexes 无索引，不预载文件，打开速度快，但查询速度慢 39 | func OpenWithoutIndexes(qqwrySrc string) (loc *Locator, err error) { 40 | loc = &Locator{} 41 | if loc.parser, err = NewParser(qqwrySrc, false); err != nil { 42 | return nil, err 43 | } 44 | loc.count = int(loc.parser.count) 45 | return 46 | } 47 | 48 | func Load(b []byte) (loc *Locator, err error) { 49 | buf := bytes.NewReader(b) 50 | r, err := zlib.NewReader(buf) 51 | if err != nil { 52 | return nil, err 53 | } 54 | b, err = ioutil.ReadAll(r) 55 | if err != nil { 56 | return nil, err 57 | } 58 | loc = &Locator{} 59 | res := &resource{data: b} 60 | var parser *Parser 61 | if parser, err = NewParserRes(res, uint32(len(res.data))); err != nil { 62 | return nil, err 63 | } 64 | loc.indexes = newIndexes(parser) 65 | loc.count = int(parser.count) 66 | return 67 | } 68 | 69 | func LoadWithoutIndexes(b []byte) (loc *Locator, err error) { 70 | buf := bytes.NewReader(b) 71 | r, err := zlib.NewReader(buf) 72 | if err != nil { 73 | return nil, err 74 | } 75 | b, err = ioutil.ReadAll(r) 76 | if err != nil { 77 | return nil, err 78 | } 79 | loc = &Locator{} 80 | res := &resource{data:b} 81 | if loc.parser, err = NewParserRes(res, uint32(len(res.data))); err != nil { 82 | return nil, err 83 | } 84 | loc.count = int(loc.parser.count) 85 | return 86 | } 87 | 88 | type Locator struct { 89 | parser *Parser 90 | indexes *indexes 91 | count int 92 | } 93 | 94 | // Close close the file descriptor, if there is no preload 95 | func (loc *Locator) Close() error { 96 | if loc.parser != nil { 97 | return loc.parser.Close() 98 | } 99 | return nil 100 | } 101 | 102 | func (loc *Locator) Count() int { 103 | return loc.count 104 | } 105 | 106 | func (loc *Locator) FindIP(ip IP) *Detail { 107 | return loc.find(ip) 108 | } 109 | 110 | func (loc *Locator) FindUint(uintIP uint32) *Detail { 111 | return loc.find(ParseUintIP(uintIP)) 112 | } 113 | 114 | func (loc *Locator) Find(rawIP string) *Detail { 115 | ip, err := ParseIP(rawIP) 116 | if err != nil { 117 | return nil 118 | } 119 | return loc.find(ip) 120 | } 121 | 122 | func (loc *Locator) Range(iterator rangeIterator) { 123 | if loc.indexes != nil { 124 | var n int 125 | loc.indexes.index.AscendRange(nil, nil, func(i dataItem) bool { 126 | n++ 127 | return iterator(n, ParseUintIP(i.(indexItem)[0]), ParseUintIP(i.(indexItem)[1])) 128 | }) 129 | } else { 130 | loc.parser.IndexRange(func(i int, start, end, pos uint32) bool { 131 | return iterator(i, ParseUintIP(start), ParseUintIP(end)) 132 | }) 133 | } 134 | } 135 | 136 | func (loc *Locator) find(ip IP) *Detail { 137 | defer func() { 138 | if r := recover(); r != nil { 139 | buf := make([]byte, 1<<16) 140 | buf = buf[:runtime.Stack(buf, false)] 141 | fmt.Printf("iploc > panic: %v\n%s", r, buf) 142 | } 143 | }() 144 | 145 | hit := loc.seek(ip) 146 | detail := &Detail{ 147 | IP: ip, 148 | Start: ParseUintIP(hit[0]), 149 | End: ParseUintIP(hit[1]), 150 | Location: loc.getLocation(hit), 151 | } 152 | return detail.fill() 153 | } 154 | 155 | func (loc *Locator) seek(ip IP) (hit indexItem) { 156 | if loc.indexes != nil { 157 | hit = loc.indexes.indexOf(ip.Uint()) 158 | } else { 159 | var low, mid, high uint32 160 | var index int64 161 | var start, end IP 162 | high = loc.parser.Count() - 1 163 | for low <= high { 164 | mid = (low + high) >> 1 165 | index = int64(loc.parser.min + mid*indexBlockSize) 166 | start = ParseBytesIP(loc.parser.ReadBytes(index, ipByteSize)) 167 | if ip.Compare(start) < 0 { 168 | high = mid - 1 169 | } else { 170 | index = loc.parser.ReadPosition(index + ipByteSize) 171 | end = ParseBytesIP(loc.parser.ReadBytes(index, ipByteSize)) 172 | if ip.Compare(end) > 0 { 173 | low = mid + 1 174 | } else { 175 | hit = indexItem{start.Uint(), end.Uint(), uint32(index), 0} 176 | break 177 | } 178 | } 179 | } 180 | } 181 | return 182 | } 183 | 184 | func (loc *Locator) getLocation(item indexItem) Location { 185 | if loc.indexes != nil { 186 | return loc.indexes.getLocation(item[2], item[3]) 187 | } 188 | return loc.parser.digLocation(int64(item[2])) 189 | } 190 | -------------------------------------------------------------------------------- /cmd/iploc-conv/iploc-conv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "time" 12 | 13 | flag "github.com/spf13/pflag" 14 | "golang.org/x/text/encoding/simplifiedchinese" 15 | "golang.org/x/text/transform" 16 | 17 | "github.com/kayon/iploc" 18 | ) 19 | 20 | // 将原版 qqwry.dat 由 GBK 转换为 UTF-8 21 | // 修正原 qqwry.dat 中几处错误的重定向，并将 "CZ88.NET" 替换为 "N/A" 22 | 23 | var ( 24 | qqwrySrc, qqwryDst string 25 | quiet, noCheck, help bool 26 | gbkDecoder transform.Transformer 27 | ) 28 | 29 | // 修正 qqwry.dat 2018-05-10 30 | var fix = map[uint32][2]string{ 31 | 3413565439: {"广东省广州市", "有线宽带"}, // 203.118.223.255 32 | 3524192255: {"广东省广州市", "联通"}, // 210.14.231.255 33 | 3526640127: {"福建省厦门市", "联通"}, // 210.52.65.255 34 | 3526641407: {"广东省广州市", "联通"}, // 210.52.70.255 35 | 3526646783: {"广东省佛山市", "联通"}, // 210.52.91.255 36 | 3549601791: {"北京市", "广电网"}, // 211.146.159.255 37 | 3549605631: {"北京市", "广电网"}, // 211.146.174.255 38 | } 39 | 40 | func init() { 41 | flag.StringVarP(&qqwrySrc, "src", "s", "", "source DAT file (GBK)") 42 | flag.StringVarP(&qqwryDst, "dst", "d", "", "destination DAT file (UTF-8)") 43 | flag.BoolVarP(&quiet, "quiet", "q", false, "only output error") 44 | flag.BoolVarP(&noCheck, "nc", "n", false, "do not check the converted DAT") 45 | flag.BoolVarP(&help, "help", "h", false, "this help") 46 | flag.CommandLine.SortFlags = false 47 | flag.Parse() 48 | 49 | flag.Usage = func() { 50 | fmt.Fprintf(os.Stderr, "iploc-conv: converts DAT file from GBK into UTF-8.\nUsage: iploc-conv -s src.gbk.dat -d dst.utf8.dat [arguments]\nOptions:\n") 51 | flag.PrintDefaults() 52 | } 53 | 54 | if qqwrySrc == "" || qqwryDst == "" || help { 55 | flag.Usage() 56 | if help { 57 | os.Exit(0) 58 | } 59 | os.Exit(1) 60 | } else if qqwrySrc == qqwryDst { 61 | fmt.Fprintln(os.Stderr, "source and destination files is same") 62 | os.Exit(1) 63 | } 64 | 65 | gbkDecoder = simplifiedchinese.GBK.NewDecoder() 66 | } 67 | 68 | func main() { 69 | st := time.Now() 70 | parser, err := iploc.NewParser(qqwrySrc, true) 71 | if err != nil { 72 | fmt.Println(err) 73 | return 74 | } 75 | 76 | f, err := os.Create(qqwryDst) 77 | if err != nil { 78 | fmt.Println(err) 79 | return 80 | } 81 | 82 | var ( 83 | raw iploc.LocationRaw 84 | locOffset = make(map[uint32]uint32) 85 | locPos = make(map[string]uint32) 86 | indexes, s []byte 87 | head [8]byte 88 | offset, o uint32 89 | has bool 90 | count = parser.Count() 91 | l = len(fmt.Sprint(count)) 92 | w, n int 93 | wt = bufio.NewWriter(f) 94 | ) 95 | 96 | // write head 8 bytes 97 | wt.Write(head[:]) 98 | 99 | // head 8 bytes 100 | offset = 8 101 | parser.IndexRange(func(i int, start, end, pos uint32) bool { 102 | if !quiet { 103 | fmt.Printf("\rBuild %6.2f%% %*d/%d", float64(i)/float64(count)*100, l, i, count) 104 | } 105 | raw = parser.ReadLocationRaw(int64(pos)) 106 | // write end ip 4 bytes 107 | w, _ = wt.Write(iploc.ParseUintIP(end).ReverseBytes()) 108 | n += w 109 | // write start ip 4 bytes 110 | indexes = append(indexes, iploc.ParseUintIP(start).ReverseBytes()...) 111 | // write offset 3 bytes 112 | indexes = append(indexes, byte(offset), byte(offset>>8), byte(offset>>16)) 113 | // end ip 4 bytes 114 | offset += 4 115 | 116 | if _, has = fix[end]; has { 117 | if fix[end][0] != "" { 118 | if locPos[fix[end][0]] != 0 { 119 | raw.Text[0] = nil 120 | raw.Mode[0] = 0x02 121 | raw.Pos[0] = locPos[fix[end][0]] 122 | } else { 123 | raw.Text[0] = []byte(fix[end][0]) 124 | raw.Mode[0] = 0x00 125 | raw.Pos[0] = offset 126 | } 127 | } 128 | if fix[end][1] != "" { 129 | if locPos[fix[end][1]] != 0 { 130 | raw.Text[1] = nil 131 | raw.Mode[1] = 0x02 132 | raw.Pos[1] = locPos[fix[end][1]] 133 | } else { 134 | raw.Text[1] = []byte(fix[end][0]) 135 | raw.Mode[1] = 0x00 136 | raw.Pos[1] = uint32(len(raw.Text[0])) + offset 137 | } 138 | } 139 | } 140 | 141 | if raw.Text[0] != nil { 142 | if raw.Mode[0] != 0x00 { 143 | w, _ = wt.Write([]byte{raw.Mode[0]}) 144 | n += w 145 | offset += 1 146 | } 147 | locOffset[raw.Pos[0]] = offset 148 | s = toUTF8(raw.Text[0]) 149 | locPos[string(s)] = raw.Pos[0] 150 | w, _ = wt.Write(append(s, 0x00)) 151 | n += w 152 | offset += uint32(len(s)) + 1 153 | } else { 154 | o = locOffset[raw.Pos[0]] 155 | w, _ = wt.Write([]byte{raw.Mode[0], byte(o), byte(o>>8), byte(o>>16)}) 156 | n += w 157 | offset += 4 // 1+3 158 | } 159 | 160 | // redirected redundant data (mode 0x02) 161 | if raw.Text[1] != nil && locOffset[raw.Pos[1]] == 0 { 162 | locOffset[raw.Pos[1]] = offset 163 | s = toUTF8(raw.Text[1]) 164 | // CZ88.NET 165 | if bytes.Compare(s, []byte{32, 67, 90, 56, 56, 46, 78, 69, 84}) == 0 { 166 | s = []byte{78, 47, 65} // N/A 167 | } 168 | locPos[string(s)] = raw.Pos[1] 169 | w, _ = wt.Write(append(s, 0x00)) 170 | n += w 171 | offset += uint32(len(s)) + 1 172 | } else { 173 | o = locOffset[raw.Pos[1]] 174 | w, _ = wt.Write([]byte{0x02, byte(o), byte(o>>8), byte(o>>16)}) 175 | n += w 176 | offset += 4 177 | } 178 | return true 179 | }) 180 | 181 | if !quiet { 182 | fmt.Printf("\rBuild %6.2f%% %*d/%d %s\n", 100.0, l, count, count, time.Since(st)) 183 | } 184 | 185 | min := n + 8 186 | max := min + len(indexes) - 7 187 | binary.LittleEndian.PutUint32(head[:4], uint32(min)) 188 | binary.LittleEndian.PutUint32(head[4:], uint32(max)) 189 | wt.Write(indexes) 190 | wt.Flush() 191 | f.WriteAt(head[:], io.SeekStart) 192 | f.Close() 193 | 194 | if !noCheck { 195 | check() 196 | } 197 | } 198 | 199 | func check() { 200 | if !quiet { 201 | fmt.Print("Check ...") 202 | } 203 | var ( 204 | locGBK, locUTF8 *iploc.Locator 205 | err error 206 | logger io.WriteCloser 207 | ) 208 | 209 | if locGBK, err = iploc.Open(qqwrySrc); err != nil { 210 | fmt.Fprintf(os.Stderr, err.Error()) 211 | os.Exit(1) 212 | } 213 | 214 | if locUTF8, err = iploc.Open(qqwryDst); err != nil { 215 | fmt.Fprintf(os.Stderr, err.Error()) 216 | os.Exit(1) 217 | } 218 | 219 | if locGBK.Count() != locUTF8.Count() { 220 | fmt.Fprintf(os.Stderr, "it's not the same version") 221 | os.Exit(1) 222 | } 223 | if logger, err = os.Create("iploc-conv.log"); err != nil { 224 | fmt.Fprintf(os.Stderr, err.Error()) 225 | os.Exit(1) 226 | } 227 | var ( 228 | detail1, detail2 *iploc.Detail 229 | count = locGBK.Count() 230 | l = len(fmt.Sprint(count)) 231 | warning int 232 | st = time.Now() 233 | ) 234 | locGBK.Range(func(i int, start, end iploc.IP) bool { 235 | detail1 = locGBK.FindIP(start) 236 | detail2 = locUTF8.FindIP(start) 237 | if !compare(detail1, detail2, logger) { 238 | warning ++ 239 | } 240 | if !quiet { 241 | fmt.Printf("\rCheck %6.2f%% %*d/%d", float64(i)/float64(count)*100, l, i, count) 242 | } 243 | return true 244 | }) 245 | logger.Close() 246 | 247 | if !quiet { 248 | fmt.Printf("\rCheck %6.2f%% %*d/%d %s\n", 100.0, l, count, count, time.Since(st)) 249 | } 250 | if warning > 0 { 251 | fmt.Printf("%d warnings. please see iploc-conv.log for more details\n", warning) 252 | } else { 253 | os.Remove("iploc-conv.log") 254 | } 255 | } 256 | 257 | func compare(a, b *iploc.Detail, logger io.WriteCloser) (ok bool) { 258 | var s string 259 | defer func() { 260 | if !ok { 261 | if a.Region == " CZ88.NET" && b.Region == "N/A" { 262 | ok = true 263 | return 264 | } 265 | fmt.Fprintf(logger, " GBK: %s,%s %d,%d\n", a.Start, a.End, a.Start.Uint(), a.End.Uint()) 266 | fmt.Fprintf(logger, "UTF8: %s,%s %d,%d\n", b.Start, b.End, b.Start.Uint(), b.End.Uint()) 267 | if s != "" { 268 | fmt.Fprintf(logger, " GBK: %s %v\n", s, []byte(s)) 269 | fmt.Fprintf(logger, "UTF8: %s %v\n", b, []byte(b.String())) 270 | fmt.Fprint(logger, "\n") 271 | } else { 272 | fmt.Fprint(logger, "\n") 273 | } 274 | } 275 | }() 276 | if a.Start.Compare(b.Start) != 0 || a.End.Compare(b.End) != 0 { 277 | ok = false 278 | return 279 | } 280 | if _, ok = fix[a.End.Uint()]; ok { 281 | return 282 | } 283 | s = string(toUTF8(a.Bytes())) 284 | ok = s == b.String() 285 | if s == "" { 286 | s = "" 287 | } 288 | return 289 | } 290 | 291 | func toUTF8(s []byte) (b []byte) { 292 | r := transform.NewReader(bytes.NewReader(s), gbkDecoder) 293 | b, _ = ioutil.ReadAll(r) 294 | return 295 | } 296 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package iploc 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | ) 10 | 11 | const ( 12 | indexBlockSize = 7 13 | ipByteSize = 4 14 | positionByteSize = 3 15 | 16 | terminatorFlag = 0x00 17 | redirectAll = 0x01 18 | redirectPart = 0x02 19 | ) 20 | 21 | /* 22 | 23 | 原始下载地址 http://www.cz88.net/fox/ipdat.shtml 24 | 原版qqwry.dat, 字节序: LittleEndian 编码字符集: GBK 25 | 使用转换工具 iploc-conv 转换为UTF-8 26 | 文件内容结构, 所有偏移位置都是3字节的绝对偏移 27 | <文件头 8字节|数据区|索引区 7字节的倍数> 28 | [文件头] 索引区开始位置(4字节)|索引区结束位置(4字节) 29 | [数据区] 结束IP(4字节)|国家地区数据 30 | *国家地区数据 31 | 通常是 国家数据(0x00结尾)|地区数据(0x00结尾) 32 | 当国家地区数据的首字节是 0x01 或 0x02时为重定向模式，后3字节为偏移位置 33 | 34 | 0x01 国家和地区都重定向, 偏移位置(3字节) 35 | 0x02 国家重定向, 偏移位置(3字节)|地区数据 36 | 无重定向 国家数据(0x00结尾)|地区数据 37 | 38 | 地区数据可能还有一次重定向(0x02开头) 39 | 40 | [索引区] 起始IP(4字节)|偏移位置(3字节) = 每条索引 7字节 41 | */ 42 | 43 | type Parser struct { 44 | res resReadCloser 45 | min uint32 46 | max uint32 47 | total uint32 48 | count uint32 49 | size uint32 50 | } 51 | 52 | type indexIterator func(i int, start, end, pos uint32) bool 53 | 54 | type LocationRaw struct { 55 | Text [2][]byte 56 | Pos [2]uint32 57 | Mode [2]byte 58 | } 59 | 60 | func NewParser(qqwrySrc string, preload bool) (*Parser, error) { 61 | var ( 62 | err error 63 | size uint32 64 | fd *os.File 65 | b []byte 66 | res resReadCloser 67 | ) 68 | 69 | if preload { 70 | if b, err = ioutil.ReadFile(qqwrySrc); err != nil { 71 | return nil, err 72 | } 73 | size = uint32(len(b)) 74 | res = &resource{data: b} 75 | } else { 76 | if fd, err = os.OpenFile(qqwrySrc, os.O_RDONLY, 0400); err != nil { 77 | return nil, err 78 | } 79 | fi, err := fd.Stat() 80 | if err != nil { 81 | return nil, err 82 | } 83 | size = uint32(fi.Size()) 84 | res = fd 85 | } 86 | return NewParserRes(res, size) 87 | } 88 | 89 | func NewParserRes(res resReadCloser, size uint32) (*Parser, error) { 90 | if res == nil { 91 | return nil, fmt.Errorf("nil resource") 92 | } 93 | var ( 94 | p = &Parser{res: res} 95 | b []byte 96 | n int 97 | err error 98 | errInvalidDat = fmt.Errorf("invalid IP dat file") 99 | ) 100 | b = make([]byte, ipByteSize*2) 101 | if n, err = p.res.ReadAt(b, 0); err != nil || n != ipByteSize*2 { 102 | return nil, errInvalidDat 103 | } 104 | 105 | p.min = binary.LittleEndian.Uint32(b[:ipByteSize]) 106 | p.max = binary.LittleEndian.Uint32(b[ipByteSize:]) 107 | if (p.max-p.min)%indexBlockSize != 0 || size != p.max+indexBlockSize { 108 | return nil, errInvalidDat 109 | } 110 | p.total = (p.max - p.min) / indexBlockSize 111 | p.count = (p.max-p.min)/indexBlockSize + 1 112 | p.size = size 113 | return p, nil 114 | } 115 | 116 | func (p *Parser) Close() error { 117 | return p.res.Close() 118 | } 119 | 120 | func (p *Parser) Count() uint32 { 121 | return p.count 122 | } 123 | 124 | func (p *Parser) Size() uint32 { 125 | return p.size 126 | } 127 | 128 | func (p *Parser) Reader() io.Reader { 129 | return p.res 130 | } 131 | 132 | // (*Parser) ReadByte 读取1字节，用来识别重定向模式 133 | func (p *Parser) ReadByte(pos int64) byte { 134 | b := make([]byte, 1) 135 | n, err := p.res.ReadAt(b, pos) 136 | if err != nil || n != 1 { 137 | panic("ReadByte damaged DAT files, position: " + fmt.Sprint(pos)) 138 | } 139 | return b[0] 140 | } 141 | 142 | // (*Parser) ReadBytes 读取n字节并翻转 143 | func (p *Parser) ReadBytes(pos, n int64) (b []byte) { 144 | b = make([]byte, n) 145 | i, err := p.res.ReadAt(b, pos) 146 | if err != nil || int64(i) != n { 147 | panic("ReadBytes damaged DAT files, position: " + fmt.Sprint(pos)) 148 | } 149 | // reverse bytes 150 | for i, j := 0, len(b)-1; i < len(b)/2; i, j = i+1, j-1 { 151 | b[i], b[j] = b[j], b[i] 152 | } 153 | return 154 | } 155 | 156 | // (*Parser) ReadPosition 读取3字节的偏移位置 157 | func (p *Parser) ReadPosition(offset int64) int64 { 158 | b := p.ReadBytes(offset, positionByteSize) 159 | // left padding to the 32 bits 160 | if i := 4 - len(b); i > 0 { 161 | b = append(make([]byte, i), b...) 162 | } 163 | // bytes has been reversed, so it won't use binary.LittleEndian 164 | return int64(binary.BigEndian.Uint32(b)) 165 | } 166 | 167 | // (*Parser) ReadText 读取国家地区数据，以0x00结尾 168 | func (p *Parser) ReadText(offset int64) ([]byte, int) { 169 | if uint32(offset) >= p.min { 170 | return nil, 0 171 | } 172 | var s []byte 173 | var b byte 174 | for { 175 | b = p.ReadByte(offset) 176 | if b != terminatorFlag { 177 | s = append(s, b) 178 | } else if len(s) > 0 { 179 | break 180 | } 181 | offset++ 182 | } 183 | return s, len(s) 184 | } 185 | 186 | func (p *Parser) ReadString(offset int64) (string, int) { 187 | s, n := p.ReadText(offset) 188 | return string(s), n 189 | } 190 | 191 | // (*Parser) ReadRegion 读取地区数据，处理可能的重定向 192 | func (p *Parser) ReadRegion(offset int64) (s []byte) { 193 | switch p.ReadByte(offset) { 194 | case redirectPart: 195 | s, _ = p.ReadText(p.ReadPosition(offset + 1)) 196 | default: 197 | s, _ = p.ReadText(offset) 198 | } 199 | return 200 | } 201 | 202 | func (p *Parser) ReadRegionString(offset int64) string { 203 | return string(p.ReadRegion(offset)) 204 | } 205 | 206 | func (p *Parser) digLocation(offset int64) (location Location) { 207 | var n int 208 | switch p.ReadByte(offset + ipByteSize) { 209 | case redirectAll: 210 | offset = p.ReadPosition(offset + ipByteSize + 1) 211 | switch p.ReadByte(offset) { 212 | case redirectPart: 213 | location.Country, _ = p.ReadString(p.ReadPosition(offset + 1)) 214 | location.Region = p.ReadRegionString(offset + 1 + positionByteSize) 215 | default: 216 | location.Country, n = p.ReadString(offset) 217 | // +1, skip 1 bytes 0x00 218 | location.Region = p.ReadRegionString(offset + 1 + int64(n)) 219 | } 220 | case redirectPart: 221 | location.Country, _ = p.ReadString(p.ReadPosition(offset + ipByteSize + 1)) 222 | location.Region = p.ReadRegionString(offset + ipByteSize + 1 + positionByteSize) 223 | default: 224 | location.Country, n = p.ReadString(offset + ipByteSize) 225 | // +1, skip 1 bytes 0x00 226 | location.Region = p.ReadRegionString(offset + ipByteSize + 1 + int64(n)) 227 | } 228 | location.raw = location.Country 229 | if location.Region != "" { 230 | location.raw += " " + location.Region 231 | } 232 | return 233 | } 234 | 235 | func (p *Parser) readRegionRaw(offset int64) (s []byte, pos uint32, mode byte) { 236 | switch p.ReadByte(offset) { 237 | case redirectPart: 238 | pos = uint32(p.ReadPosition(offset + 1)) 239 | mode = redirectPart 240 | default: 241 | s, _ = p.ReadText(offset) 242 | } 243 | return 244 | } 245 | 246 | // ReadLocationRaw 用于导出或索引 247 | func (p *Parser) ReadLocationRaw(offset int64) (raw LocationRaw) { 248 | var n int 249 | raw.Mode[0] = p.ReadByte(offset + ipByteSize) 250 | switch raw.Mode[0] { 251 | case redirectAll: 252 | offset = p.ReadPosition(offset + ipByteSize + 1) 253 | switch p.ReadByte(offset) { 254 | case redirectPart: 255 | raw.Mode[0] = redirectPart 256 | raw.Pos[0] = uint32(p.ReadPosition(offset + 1)) 257 | raw.Text[1], raw.Pos[1], raw.Mode[1] = p.readRegionRaw(offset + 1 + positionByteSize) 258 | if raw.Text[1] != nil { 259 | raw.Pos[1] = uint32(offset + 1 + positionByteSize) 260 | } 261 | default: 262 | raw.Pos[0] = uint32(offset) 263 | _, n = p.ReadText(offset) 264 | _, raw.Pos[1], _ = p.readRegionRaw(offset + 1 + int64(n)) 265 | if raw.Pos[1] == 0 { 266 | raw.Pos[1] = uint32(offset + 1 + int64(n)) 267 | } 268 | } 269 | case redirectPart: 270 | raw.Pos[0] = uint32(p.ReadPosition(offset + ipByteSize + 1)) 271 | raw.Text[1], raw.Pos[1], raw.Mode[1] = p.readRegionRaw(offset + ipByteSize + 1 + positionByteSize) 272 | if raw.Text[1] != nil { 273 | raw.Pos[1] = uint32(offset + ipByteSize + 1 + positionByteSize) 274 | } 275 | default: 276 | raw.Pos[0] = uint32(offset + ipByteSize) 277 | raw.Mode[0] = 0x00 278 | raw.Text[0], n = p.ReadText(offset + ipByteSize) 279 | raw.Text[1], raw.Pos[1], raw.Mode[1] = p.readRegionRaw(offset + ipByteSize + 1 + int64(n)) 280 | if raw.Text[1] != nil { 281 | raw.Pos[1] = uint32(offset + ipByteSize + 1 + int64(n)) 282 | } 283 | } 284 | return 285 | } 286 | 287 | // (*Parser) IndexRange 288 | // calls the iterator for every index within the range (i, start, end, Pos) 289 | // until iterator returns false. 290 | func (p *Parser) IndexRange(iterator indexIterator) { 291 | var ( 292 | count = int(p.count) 293 | index, pos int64 294 | ) 295 | for i := 0; i < count; i++ { 296 | index = int64(p.min) + indexBlockSize*int64(i) 297 | pos = p.ReadPosition(index + 4) 298 | if !iterator( 299 | i+1, 300 | ParseBytesIP(p.ReadBytes(index, ipByteSize)).Uint(), 301 | ParseBytesIP(p.ReadBytes(pos, ipByteSize)).Uint(), 302 | uint32(pos), 303 | ) { 304 | break 305 | } 306 | } 307 | } 308 | --------------------------------------------------------------------------------