├── .gitignore ├── .github ├── FUNDING.yml └── workflows │ ├── golangci-lint.yml │ ├── go.yml │ └── codeql-analysis.yml ├── version.go ├── go.mod ├── config.yaml.example ├── common_types.go ├── t ├── nginx_geo_ipv6.t └── nginx_geo.t ├── LICENSE ├── common_geobase.go ├── go.sum ├── ipgeobase.go ├── tor.go ├── utils.go ├── ip2proxy.go ├── ipgeobase_rus_regions.go ├── README.md ├── maxmind.go └── app.go /.gitignore: -------------------------------------------------------------------------------- 1 | ip2geo 2 | ip2geo-mac 3 | output 4 | *.exe 5 | .vscode 6 | output/ 7 | .DS_Store 8 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [m-messiah] 4 | patreon: m_messiah 5 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // VERSION for `-version` flag output. Equals to git tag in releases. 4 | const VERSION = "unversioned" 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/m-messiah/ip2geo 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/fatih/color v1.18.0 7 | github.com/jinzhu/configor v1.2.2 8 | golang.org/x/net v0.39.0 9 | ) 10 | 11 | require ( 12 | github.com/BurntSushi/toml v1.5.0 // indirect 13 | github.com/mattn/go-colorable v0.1.14 // indirect 14 | github.com/mattn/go-isatty v0.0.20 // indirect 15 | golang.org/x/sys v0.32.0 // indirect 16 | golang.org/x/text v0.24.0 // indirect 17 | gopkg.in/yaml.v3 v3.0.1 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /config.yaml.example: -------------------------------------------------------------------------------- 1 | --- 2 | loglevel: 0 3 | outputdir: "output" 4 | tor: 5 | enabled: true 6 | ip2proxy: 7 | printtype: true 8 | lite: 9 | enabled: false 10 | filename: PX4LITE.ZIP 11 | pro: 12 | enabled: false 13 | token: "1234567890" 14 | maxmind: 15 | enabled: true 16 | licensekey: "ABCDEFGH" 17 | lang: en 18 | ipver: 6 19 | nobase64: true 20 | tznames: false 21 | nocountry: false 22 | include: RU,EN,JP 23 | exclude: US 24 | ipgeobase: 25 | enabled: true 26 | 27 | -------------------------------------------------------------------------------- /common_types.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "net" 6 | "strings" 7 | ) 8 | 9 | // IPList sortable by network 10 | type IPList []string 11 | 12 | func (d IPList) Less(i, j int) bool { 13 | ipI := net.ParseIP(strings.Split(d[i], "-")[0]) 14 | ipJ := net.ParseIP(strings.Split(d[j], "-")[0]) 15 | return bytes.Compare(ipI, ipJ) < 0 16 | } 17 | 18 | func (d IPList) Len() int { 19 | return len(d) 20 | } 21 | 22 | func (d IPList) Swap(i, j int) { 23 | d[i], d[j] = d[j], d[i] 24 | } 25 | 26 | // Error of GeoBase handle 27 | type Error struct { 28 | err error 29 | Module string 30 | Action string 31 | } 32 | -------------------------------------------------------------------------------- /t/nginx_geo_ipv6.t: -------------------------------------------------------------------------------- 1 | use Test::Nginx::Socket 'no_plan'; 2 | run_tests(); 3 | 4 | __DATA__ 5 | 6 | === TEST 1: ip2geo -ipv6 7 | --- main_config 8 | error_log $TEST_NGINX_IP2GEO_DIR/error.log; 9 | --- http_config 10 | 11 | # City 12 | geo $city { 13 | include $TEST_NGINX_IP2GEO_DIR/output/mm_city.txt; 14 | } 15 | 16 | # Country 17 | geo $country { 18 | include $TEST_NGINX_IP2GEO_DIR/output/mm_country.txt; 19 | } 20 | # CountryCode 21 | geo $country_code { 22 | include $TEST_NGINX_IP2GEO_DIR/output/mm_country_code.txt; 23 | } 24 | # TZ 25 | geo $tz { 26 | include $TEST_NGINX_IP2GEO_DIR/output/mm_tz.txt; 27 | } 28 | 29 | --- config 30 | location /t { 31 | default_type text/plain; 32 | return 200 "Ok"; 33 | } 34 | --- request 35 | GET /t 36 | --- error_code: 200 37 | --- no_error_log 38 | [error] 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2018 Maxim Muzafarov (m_messiah) 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /t/nginx_geo.t: -------------------------------------------------------------------------------- 1 | use Test::Nginx::Socket 'no_plan'; 2 | run_tests(); 3 | 4 | __DATA__ 5 | 6 | === TEST 1: ip2geo 7 | --- main_config 8 | error_log $TEST_NGINX_IP2GEO_DIR/error.log; 9 | --- http_config 10 | 11 | # City 12 | geo $city { 13 | ranges; 14 | include $TEST_NGINX_IP2GEO_DIR/output/mm_city.txt; 15 | } 16 | 17 | # Country 18 | geo $country { 19 | ranges; 20 | include $TEST_NGINX_IP2GEO_DIR/output/mm_country.txt; 21 | } 22 | # CountryCode 23 | geo $country_code { 24 | ranges; 25 | include $TEST_NGINX_IP2GEO_DIR/output/mm_country_code.txt; 26 | } 27 | # TZ 28 | geo $tz { 29 | ranges; 30 | include $TEST_NGINX_IP2GEO_DIR/output/mm_tz.txt; 31 | } 32 | # Tor 33 | geo $is_tor { 34 | ranges; 35 | default 0; 36 | include $TEST_NGINX_IP2GEO_DIR/output/tor.txt; 37 | } 38 | 39 | --- config 40 | location /t { 41 | default_type text/plain; 42 | return 200 "Ok"; 43 | } 44 | --- request 45 | GET /t 46 | --- error_code: 200 47 | --- no_error_log 48 | [error] 49 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | jobs: 10 | golangci: 11 | name: lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: golangci-lint 16 | uses: golangci/golangci-lint-action@v2 17 | with: 18 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 19 | version: latest 20 | 21 | # Optional: golangci-lint command line arguments. 22 | # args: --issues-exit-code=0 23 | 24 | # Optional: show only new issues if it's a pull request. The default value is `false`. 25 | only-new-issues: true 26 | 27 | # Optional: if set to true then the action will use pre-installed Go. 28 | # skip-go-installation: true 29 | 30 | # Optional: if set to true then the action don't cache or restore ~/go/pkg. 31 | # skip-pkg-cache: true 32 | 33 | # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. 34 | # skip-build-cache: true 35 | -------------------------------------------------------------------------------- /common_geobase.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type geoItem struct { 4 | Name string 5 | RegID int 6 | ID string 7 | City string 8 | Network string 9 | TZ string 10 | Country string 11 | CountryCode string 12 | } 13 | 14 | // GeoBase interface for downloadable and convertible geo database 15 | type GeoBase interface { 16 | download() ([]byte, error) 17 | unpack([]byte) error 18 | citiesDB() (map[string]geoItem, error) 19 | writeMap(map[string]geoItem) error 20 | name() string 21 | addError(Error) 22 | } 23 | 24 | // Generate GeoBase (download, unpack, parse and write in nginx map format) 25 | func Generate(geobase GeoBase) { 26 | answer, err := geobase.download() 27 | if err != nil { 28 | geobase.addError(Error{err, geobase.name(), "Download"}) 29 | return 30 | } 31 | printMessage(geobase.name(), "Download", "OK") 32 | err = geobase.unpack(answer) 33 | if err != nil { 34 | geobase.addError(Error{err, geobase.name(), "Unpack"}) 35 | return 36 | } 37 | printMessage(geobase.name(), "Unpack", "OK") 38 | cities, err := geobase.citiesDB() 39 | if err != nil { 40 | geobase.addError(Error{err, geobase.name(), "Generate Cities"}) 41 | return 42 | } 43 | printMessage(geobase.name(), "Generate cities", "OK") 44 | if err := geobase.writeMap(cities); err != nil { 45 | geobase.addError(Error{err, geobase.name(), "Write map"}) 46 | return 47 | } 48 | printMessage(geobase.name(), "Write nginx maps", "OK") 49 | geobase.addError(Error{err: nil}) 50 | } 51 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 2 | github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= 3 | github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 4 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 5 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 6 | github.com/jinzhu/configor v1.2.2 h1:sLgh6KMzpCmaQB4e+9Fu/29VErtBUqsS2t8C9BNIVsA= 7 | github.com/jinzhu/configor v1.2.2/go.mod h1:iFFSfOBKP3kC2Dku0ZGB3t3aulfQgTGJknodhFavsU8= 8 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 9 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 10 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 11 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 12 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 13 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 14 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 15 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 16 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 17 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 18 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 21 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 22 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 23 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | tags: [ 'v*' ] 7 | pull_request: 8 | branches: [ main ] 9 | 10 | jobs: 11 | 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: 1.24 21 | 22 | - name: Set up environment 23 | run: | 24 | sudo apt-get install -y nginx tzdata cpanminus 25 | sudo cpanm --quiet local::lib 26 | eval $(perl -Mlocal::lib) 27 | sudo cpanm --quiet --notest Test::Nginx::Socket 28 | 29 | - name: Add version info 30 | run: sed -i 's/const VERSION = .*/const VERSION = "'${GITHUB_REF##*/}'"/' version.go 31 | 32 | - name: Build 33 | run: go build -o ip2geo -v ./... 34 | 35 | - name: Test version 36 | run: ./ip2geo -version 37 | 38 | - name: Download MaxMind 39 | env: 40 | MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }} 41 | run: wget -O maxmind.zip 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City-CSV&suffix=zip&license_key='$MAXMIND_LICENSE_KEY 42 | 43 | - name: Test maxmind 44 | run: | 45 | ./ip2geo -maxmind-filename maxmind.zip 46 | TEST_NGINX_IP2GEO_DIR=$PWD prove t/nginx_geo.t 47 | 48 | - name: Test nobase64 49 | run: | 50 | ./ip2geo -lang en -nobase64 -maxmind-filename maxmind.zip 51 | TEST_NGINX_IP2GEO_DIR=$PWD prove t/nginx_geo.t 52 | - name: Test ipv6 53 | run: | 54 | ./ip2geo -ipver 6 -maxmind-filename maxmind.zip -maxmind 55 | TEST_NGINX_IP2GEO_DIR=$PWD prove t/nginx_geo_ipv6.t 56 | - name: Test ipv6 for en 57 | run: | 58 | ./ip2geo -ipver 6 -lang en -nobase64 -maxmind-filename maxmind.zip -maxmind 59 | TEST_NGINX_IP2GEO_DIR=$PWD prove t/nginx_geo_ipv6.t 60 | 61 | - name: Build release files 62 | if: ${{ github.ref == 'refs/heads/main' }} || startsWith(github.ref, 'refs/tags/') 63 | run: | 64 | for goos in darwin linux freebsd openbsd; do 65 | for arch in 386 arm amd64 arm64; do 66 | go build -o ip2geo-$goos-$arch 67 | done 68 | done 69 | - name: Release 70 | uses: softprops/action-gh-release@v1 71 | if: startsWith(github.ref, 'refs/tags/') 72 | with: 73 | files: ip2geo-* 74 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | jobs: 21 | analyze: 22 | name: Analyze 23 | runs-on: ubuntu-latest 24 | permissions: 25 | actions: read 26 | contents: read 27 | security-events: write 28 | 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | language: [ 'go' ] 33 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 34 | # Learn more: 35 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 36 | 37 | steps: 38 | - name: Checkout repository 39 | uses: actions/checkout@v2 40 | 41 | # Initializes the CodeQL tools for scanning. 42 | - name: Initialize CodeQL 43 | uses: github/codeql-action/init@v1 44 | with: 45 | languages: ${{ matrix.language }} 46 | # If you wish to specify custom queries, you can do so here or in a config file. 47 | # By default, queries listed here will override any specified in a config file. 48 | # Prefix the list here with "+" to use these queries and those in the config file. 49 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 50 | 51 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 52 | # If this step fails, then you should remove it and run the build manually (see below) 53 | - name: Autobuild 54 | uses: github/codeql-action/autobuild@v1 55 | 56 | # ℹ️ Command-line programs to run using the OS shell. 57 | # 📚 https://git.io/JvXDl 58 | 59 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 60 | # and modify them (or add more) to build your code if your project 61 | # uses a compiled language 62 | 63 | #- run: | 64 | # make bootstrap 65 | # make release 66 | 67 | - name: Perform CodeQL Analysis 68 | uses: github/codeql-action/analyze@v1 69 | -------------------------------------------------------------------------------- /ipgeobase.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | ) 11 | 12 | // IPGeobase - GeoBase compatible generator for ipgeobase.ru 13 | type IPGeobase struct { 14 | OutputDir string 15 | ErrorsChan chan Error 16 | archive []*zip.File 17 | } 18 | 19 | func (ipgeobase *IPGeobase) name() string { 20 | return "IPGeobase" 21 | } 22 | 23 | func (ipgeobase *IPGeobase) addError(err Error) { 24 | ipgeobase.ErrorsChan <- err 25 | } 26 | 27 | func (ipgeobase *IPGeobase) download() ([]byte, error) { 28 | resp, err := http.Get("http://ipgeobase.ru/files/db/Main/geo_files.zip") 29 | if err != nil { 30 | printMessage("IPGeobase", "Download no answer", "FAIL") 31 | return nil, err 32 | } 33 | defer resp.Body.Close() 34 | answer, err := io.ReadAll(resp.Body) 35 | if err != nil { 36 | printMessage("IPGeobase", "Download bad answer", "FAIL") 37 | return nil, err 38 | } 39 | return answer, nil 40 | } 41 | 42 | func (ipgeobase *IPGeobase) unpack(response []byte) error { 43 | file, err := Unpack(response) 44 | if err == nil { 45 | ipgeobase.archive = file 46 | } 47 | return err 48 | } 49 | 50 | func (ipgeobase *IPGeobase) citiesDB() (map[string]geoItem, error) { 51 | cities := make(map[string]geoItem) 52 | for record := range readCSVDatabase(ipgeobase.archive, "cities.txt", "IPGeobase", '\t', true) { 53 | if len(record) < 3 { 54 | printMessage("IPGeobase", fmt.Sprintf("cities.txt too short line: %s", record), "FAIL") 55 | continue 56 | } 57 | // Format is: \t\t\t\t\t 58 | cid, city, regionName := record[0], record[1], record[2] 59 | if region, ok := REGIONS[regionName]; ok { 60 | if cid == "1199" { 61 | region = REGIONS["Москва"] 62 | } 63 | cities[cid] = geoItem{ 64 | City: city, 65 | RegID: region.ID, 66 | TZ: region.TZ, 67 | } 68 | } 69 | } 70 | if len(cities) < 1 { 71 | return nil, errors.New("Cities db is empty") 72 | } 73 | return cities, nil 74 | } 75 | 76 | func (ipgeobase *IPGeobase) parseNetwork(cities map[string]geoItem) <-chan geoItem { 77 | database := make(chan geoItem) 78 | go func() { 79 | for record := range readCSVDatabase(ipgeobase.archive, "cidr_optim.txt", "IPGeobase", '\t', true) { 80 | if len(record) < 5 { 81 | printMessage("IPGeobase", fmt.Sprintf("cidr_optim.txt too short line: %s", record), "FAIL") 82 | continue 83 | } 84 | // Format is: \t\t\t\tcity_id 85 | ipRange, country, cid := removeSpace(record[2]), record[3], record[4] 86 | if info, ok := cities[cid]; country == "RU" && ok { 87 | info.Network = ipRange 88 | database <- info 89 | } 90 | } 91 | close(database) 92 | }() 93 | return database 94 | } 95 | 96 | func (ipgeobase *IPGeobase) writeMap(cities map[string]geoItem) error { 97 | reg, err := openMapFile(ipgeobase.OutputDir, "region.txt") 98 | if err != nil { 99 | return err 100 | } 101 | city, err := openMapFile(ipgeobase.OutputDir, "city.txt") 102 | if err != nil { 103 | return err 104 | } 105 | tz, err := openMapFile(ipgeobase.OutputDir, "tz.txt") 106 | if err != nil { 107 | return err 108 | } 109 | defer reg.Close() 110 | defer city.Close() 111 | defer tz.Close() 112 | 113 | for info := range ipgeobase.parseNetwork(cities) { 114 | fmt.Fprintf(city, "%s %s;\n", info.Network, base64.StdEncoding.EncodeToString([]byte(info.City))) 115 | fmt.Fprintf(reg, "%s %02d;\n", info.Network, info.RegID) 116 | fmt.Fprintf(tz, "%s %s;\n", info.Network, info.TZ) 117 | } 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /tor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "sort" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const torListLength = 1 15 | 16 | // Tor network lists DB 17 | type Tor struct { 18 | OutputDir string 19 | ErrorsChan chan Error 20 | list IPList 21 | tempLists chan map[string]bool 22 | } 23 | 24 | // Generate Tor maps for nginx (download, parse, merge, write) 25 | func (tor *Tor) Generate() { 26 | tor.tempLists = make(chan map[string]bool, torListLength) 27 | // go tor.blutmagieDownload() 28 | go tor.torProjectDownload() 29 | if err := tor.merge(); err != nil { 30 | tor.ErrorsChan <- Error{err, "TOR", "Merge"} 31 | return 32 | } 33 | printMessage("TOR", "Merge", "OK") 34 | if err := tor.writeMap(); err != nil { 35 | tor.ErrorsChan <- Error{err, "TOR", "nginx"} 36 | return 37 | } 38 | printMessage("TOR", "Write nginx maps", "OK") 39 | tor.ErrorsChan <- Error{err: nil} 40 | } 41 | 42 | // func (tor *Tor) blutmagieDownload() { 43 | // resp, err := http.Get("https://torstatus.blutmagie.de/ip_list_exit.php/Tor_ip_list_EXIT.csv") 44 | // if err != nil { 45 | // printMessage("TOR", "Blutmagie Download", "FAIL") 46 | // tor.tempLists <- nil 47 | // return 48 | // } 49 | // defer resp.Body.Close() 50 | // torlist := make(map[string]bool) 51 | // reader := bufio.NewReader(resp.Body) 52 | // for { 53 | // line, err := reader.ReadString('\n') 54 | // // Stop at EOF. 55 | // if err == io.EOF { 56 | // break 57 | // } 58 | // if err != nil { 59 | // printMessage("TOR", "can't read line from blutmagie", "WARN") 60 | // continue 61 | // } 62 | // if len(line) < 1 { 63 | // continue 64 | // } 65 | // torlist[strings.TrimSpace(line)] = true 66 | // } 67 | // printMessage("TOR", "Blutmagie Download", "OK") 68 | // tor.tempLists <- torlist 69 | // } 70 | 71 | func (tor *Tor) torProjectDownload() { 72 | client := &http.Client{Timeout: time.Second * 30} 73 | resp, err := client.Get("https://check.torproject.org/exit-addresses") 74 | if err != nil { 75 | printMessage("TOR", "Torproject Download", "FAIL") 76 | tor.tempLists <- nil 77 | return 78 | } 79 | defer resp.Body.Close() 80 | torproject := make(map[string]bool) 81 | reader := bufio.NewReader(resp.Body) 82 | for { 83 | line, err := reader.ReadString('\n') 84 | if err == io.EOF { 85 | break 86 | } 87 | if err != nil { 88 | printMessage("TOR", "Can't read line from torproject", "WARN") 89 | continue 90 | } 91 | if len(line) < 1 { 92 | continue 93 | } 94 | if !strings.Contains(line, "ExitAddress") { 95 | continue 96 | } 97 | fields := strings.Fields(line) 98 | torproject[fields[1]] = true 99 | } 100 | printMessage("TOR", "Torproject Download", "OK") 101 | tor.tempLists <- torproject 102 | } 103 | 104 | func (tor *Tor) merge() error { 105 | result := make(map[string]bool) 106 | for i := 0; i < torListLength; i++ { 107 | m := <-tor.tempLists 108 | if m == nil { 109 | continue 110 | } 111 | for k, v := range m { 112 | result[k] = v 113 | } 114 | } 115 | ipList := make(IPList, len(result)) 116 | i := 0 117 | for ip := range result { 118 | ipList[i] = ip 119 | i++ 120 | } 121 | sort.Sort(ipList) 122 | tor.list = ipList 123 | if len(tor.list) > 0 { 124 | return nil 125 | } 126 | return errors.New("torlist empty") 127 | } 128 | 129 | func (tor *Tor) writeMap() error { 130 | torFile, err := openMapFile(tor.OutputDir, "tor.txt") 131 | if err != nil { 132 | return err 133 | } 134 | defer torFile.Close() 135 | for _, ip := range tor.list { 136 | fmt.Fprintf(torFile, "%s-%s 1;\n", ip, ip) 137 | } 138 | return nil 139 | 140 | } 141 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "encoding/csv" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net" 11 | "os" 12 | "path" 13 | "strings" 14 | "time" 15 | "unicode" 16 | 17 | "github.com/fatih/color" 18 | "golang.org/x/net/html/charset" 19 | ) 20 | 21 | func removeSpace(s string) string { 22 | return strings.Map(func(r rune) rune { 23 | if unicode.IsSpace(r) { 24 | return -1 25 | } 26 | return r 27 | }, s) 28 | } 29 | 30 | func getFileFromZip(archive []*zip.File, filename string) (*zip.File, error) { 31 | for _, file := range archive { 32 | if strings.Contains(file.Name, filename) { 33 | return file, nil 34 | } 35 | } 36 | return nil, errors.New("file not found") 37 | } 38 | 39 | func openMapFile(outputDir, filename string) (*os.File, error) { 40 | filepath := path.Join(outputDir, filename) 41 | return os.Create(filepath) 42 | } 43 | 44 | func printMessage(module, message, status string) { 45 | var statusMesage string 46 | switch status { 47 | case "OK": 48 | if Config.LogLevel > 0 { 49 | return 50 | } 51 | statusMesage = color.GreenString(status) 52 | case "WARN": 53 | if Config.LogLevel > 1 { 54 | return 55 | } 56 | statusMesage = color.YellowString(status) 57 | case "FAIL": 58 | statusMesage = color.RedString(status) 59 | default: 60 | if Config.LogLevel > 1 { 61 | return 62 | } 63 | statusMesage = color.BlueString(status) 64 | } 65 | fmt.Printf("%-15s | %-60s [%s]\n", module, message, statusMesage) 66 | } 67 | 68 | func getIPRange(ipver int, network string) string { 69 | if ipver == 4 { 70 | _, ipnet, err := net.ParseCIDR(network) 71 | if err != nil { 72 | return "" 73 | } 74 | ipb := make(net.IP, net.IPv4len) 75 | copy(ipb, ipnet.IP) 76 | for i, v := range ipb { 77 | ipb[i] = v | ^ipnet.Mask[i] 78 | } 79 | return fmt.Sprintf("%s-%s", ipnet.IP, ipb) 80 | } 81 | if strings.Contains(network, ":") && strings.Contains(network, "/") { 82 | return network 83 | } 84 | return "" 85 | } 86 | 87 | // Convert uint to net.IP 88 | func int2ip(ipnr int64) net.IP { 89 | var bytes [4]byte 90 | bytes[0] = byte(ipnr & 0xFF) 91 | bytes[1] = byte((ipnr >> 8) & 0xFF) 92 | bytes[2] = byte((ipnr >> 16) & 0xFF) 93 | bytes[3] = byte((ipnr >> 24) & 0xFF) 94 | 95 | return net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0]) 96 | } 97 | 98 | func readCSVDatabase(archive []*zip.File, filename string, dbType string, comma rune, windowsEncoding bool) chan []string { 99 | yield := make(chan []string) 100 | go func() { 101 | defer close(yield) 102 | file, err := getFileFromZip(archive, filename) 103 | if err != nil { 104 | printMessage(dbType, fmt.Sprintf("%s %s", filename, err.Error()), "FAIL") 105 | return 106 | } 107 | fp, err := file.Open() 108 | if err != nil { 109 | printMessage(dbType, fmt.Sprintf("Can't open %s", filename), "FAIL") 110 | yield <- nil 111 | } 112 | defer fp.Close() 113 | var r *csv.Reader 114 | if windowsEncoding { 115 | utf8, err := charset.NewReader(fp, "text/csv; charset=windows-1251") 116 | if err != nil { 117 | printMessage(dbType, fmt.Sprintf("%s not in cp1251", filename), "FAIL") 118 | yield <- nil 119 | } 120 | r = csv.NewReader(utf8) 121 | } else { 122 | r = csv.NewReader(fp) 123 | } 124 | r.Comma, r.LazyQuotes = comma, true 125 | for { 126 | record, err := r.Read() 127 | // Stop at EOF. 128 | if err == io.EOF { 129 | break 130 | } 131 | if err != nil { 132 | printMessage(dbType, fmt.Sprintf("Can't read line from %s", filename), "WARN") 133 | continue 134 | } 135 | yield <- record 136 | } 137 | 138 | }() 139 | return yield 140 | } 141 | 142 | func convertTZToOffset(t time.Time, tz string) string { 143 | location, err := time.LoadLocation(tz) 144 | if err != nil { 145 | return "" 146 | } 147 | _, offset := t.In(location).Zone() 148 | return fmt.Sprintf("UTC%+d", offset/3600) 149 | } 150 | 151 | // Unpack zip file and return slice of zip.File`s 152 | func Unpack(response []byte) ([]*zip.File, error) { 153 | zipReader, err := zip.NewReader(bytes.NewReader(response), int64(len(response))) 154 | var file []*zip.File 155 | if err == nil { 156 | file = zipReader.File 157 | } 158 | return file, err 159 | } 160 | -------------------------------------------------------------------------------- /ip2proxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net" 9 | "net/http" 10 | "os" 11 | "path" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | type ip2proxyItem struct { 17 | IPFrom net.IP 18 | IPTo net.IP 19 | ProxyType string 20 | CountryCode string 21 | Country string 22 | Region string 23 | City string 24 | ISP string 25 | } 26 | 27 | type ip2proxy struct { 28 | archive []*zip.File 29 | OutputDir string 30 | ErrorsChan chan Error 31 | Token string 32 | Filename string 33 | Name string 34 | csvFilename string 35 | zipFilename string 36 | PrintType bool 37 | } 38 | 39 | func (o *ip2proxy) checkErr(err error, message string) bool { 40 | if err != nil { 41 | o.ErrorsChan <- Error{err, o.Name, message} 42 | return true 43 | } 44 | printMessage(o.Name, message, "OK") 45 | return false 46 | } 47 | 48 | func (o *ip2proxy) Get() { 49 | if o.Name == "ip2proxyPro" { 50 | o.csvFilename = "IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP.CSV" 51 | o.zipFilename = "PX4" 52 | } else if o.Name == "ip2proxyLite" { 53 | o.csvFilename = "IP2PROXY-LITE-PX4.CSV" 54 | o.zipFilename = "PX4LITE" 55 | } else { 56 | o.ErrorsChan <- Error{errors.New("Unknown ip2proxy type requested"), o.Name, "bad init"} 57 | return 58 | } 59 | fileData, err := o.getZip() 60 | if o.checkErr(err, "Get ZIP") { 61 | return 62 | } 63 | err = o.unpack(fileData) 64 | if o.checkErr(err, "Unpack") { 65 | return 66 | } 67 | err = o.Write() 68 | if o.checkErr(err, "Write Nginx Map") { 69 | return 70 | } 71 | o.ErrorsChan <- Error{err: nil} 72 | } 73 | 74 | func (o *ip2proxy) getZip() ([]byte, error) { 75 | if len(o.Token) > 0 { 76 | return o.download() 77 | } else if len(o.Filename) > 0 { 78 | return os.ReadFile(o.Filename) 79 | } else { 80 | return nil, errors.New("Token or Filename must be passed") 81 | } 82 | } 83 | 84 | func (o *ip2proxy) download() ([]byte, error) { 85 | client := &http.Client{} 86 | req, err := http.NewRequest("GET", "https://www.ip2location.com/download", nil) 87 | if err != nil { 88 | return nil, err 89 | } 90 | q := req.URL.Query() 91 | q.Add("file", o.zipFilename) 92 | q.Add("token", o.Token) 93 | req.URL.RawQuery = q.Encode() 94 | resp, err := client.Do(req) 95 | if err != nil { 96 | return nil, err 97 | } 98 | defer resp.Body.Close() 99 | if resp.StatusCode != 200 { 100 | return nil, fmt.Errorf("failed with code %d", resp.StatusCode) 101 | } 102 | answer, err := io.ReadAll(resp.Body) 103 | if err != nil { 104 | return nil, err 105 | } 106 | return answer, nil 107 | } 108 | 109 | func (o *ip2proxy) unpack(response []byte) error { 110 | file, err := Unpack(response) 111 | if err == nil { 112 | o.archive = file 113 | } 114 | return err 115 | } 116 | 117 | func (o *ip2proxy) Parse(filename string) <-chan *ip2proxyItem { 118 | database := make(chan *ip2proxyItem) 119 | go func() { 120 | for record := range readCSVDatabase(o.archive, filename, o.Name, ',', false) { 121 | item, err := o.lineToItem(record) 122 | if err != nil { 123 | printMessage(o.Name, fmt.Sprintf("Can't parse line from %s with %v", filename, err), "WARN") 124 | continue 125 | } 126 | database <- item 127 | } 128 | close(database) 129 | }() 130 | return database 131 | } 132 | 133 | func (o *ip2proxy) lineToItem(line []string) (*ip2proxyItem, error) { 134 | if len(line) != 8 { 135 | return nil, fmt.Errorf("Number of field is not 8") 136 | } 137 | var ( 138 | ipFromInt, ipToInt int64 139 | err error 140 | ) 141 | if ipFromInt, err = strconv.ParseInt(line[0], 10, 64); err != nil { 142 | return nil, fmt.Errorf("Can't parse FromIP with: %v", err) 143 | } 144 | if ipToInt, err = strconv.ParseInt(line[1], 10, 64); err != nil { 145 | return nil, fmt.Errorf("Can't parse ToIP with: %v", err) 146 | } 147 | return &ip2proxyItem{ 148 | IPFrom: int2ip(ipFromInt), 149 | IPTo: int2ip(ipToInt), 150 | ProxyType: line[2], 151 | CountryCode: line[3], 152 | Country: line[4], 153 | Region: line[5], 154 | City: line[6], 155 | ISP: line[7], 156 | }, nil 157 | } 158 | 159 | func (o *ip2proxy) Write() error { 160 | netFile, err := os.Create(path.Join(o.OutputDir, o.Name+"_net.txt")) 161 | if err != nil { 162 | return err 163 | } 164 | defer netFile.Close() 165 | ispFile, err := os.Create(path.Join(o.OutputDir, o.Name+"_isp.txt")) 166 | if err != nil { 167 | return err 168 | } 169 | defer ispFile.Close() 170 | var mapValue string 171 | for item := range o.Parse(o.csvFilename) { 172 | if o.PrintType { 173 | mapValue = item.ProxyType 174 | } else { 175 | mapValue = "1" 176 | } 177 | fmt.Fprintf(netFile, "%s-%s \"%s\";\n", item.IPFrom, item.IPTo, mapValue) 178 | fmt.Fprintf(ispFile, "%s-%s \"%s\";\n", item.IPFrom, item.IPTo, strings.Replace(item.ISP, "\"", "\\\"", -1)) 179 | } 180 | return nil 181 | } 182 | -------------------------------------------------------------------------------- /ipgeobase_rus_regions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // RegInfo - Code and TZ for region 4 | type RegInfo struct { 5 | ID int 6 | TZ string 7 | } 8 | 9 | // REGIONS - list of russian regions 10 | var REGIONS = map[string]RegInfo{ 11 | "Республика Адыгея": {1, "UTC+3"}, 12 | "Республика Башкортостан": {2, "UTC+5"}, 13 | "Республика Бурятия": {3, "UTC+8"}, 14 | "Республика Алтай": {4, "UTC+6"}, 15 | "Республика Дагестан": {5, "UTC+3"}, 16 | "Республика Ингушетия": {6, "UTC+3"}, 17 | "Республика Кабардино-Балкария": {7, "UTC+3"}, 18 | "Республика Калмыкия": {8, "UTC+3"}, 19 | "Республика Карачаево-Черкессия": {9, "UTC+3"}, 20 | "Республика Карелия": {10, "UTC+3"}, 21 | "Республика Коми": {11, "UTC+3"}, 22 | "Республика Марий Эл": {12, "UTC+3"}, 23 | "Республика Мордовия": {13, "UTC+3"}, 24 | "Республика Саха (Якутия)": {14, "UTC+11"}, 25 | "Республика Северная Осетия (Алания)": {15, "UTC+3"}, 26 | "Республика Татарстан": {16, "UTC+3"}, 27 | "Республика Тыва (Тува)": {17, "UTC+7"}, 28 | "Республика Удмуртия": {18, "UTC+4"}, 29 | "Республика Хакасия": {19, "UTC+7"}, 30 | "Республика Чечня": {20, "UTC+3"}, 31 | "Республика Чувашия": {21, "UTC+3"}, 32 | "Алтайский край": {22, "UTC+6"}, 33 | "Краснодарский край": {23, "UTC+3"}, 34 | "Красноярский край": {24, "UTC+7"}, 35 | "Приморский край": {25, "UTC+10"}, 36 | "Ставропольский край": {26, "UTC+3"}, 37 | "Хабаровский край": {27, "UTC+10"}, 38 | "Амурская область": {28, "UTC+9"}, 39 | "Архангельская область": {29, "UTC+3"}, 40 | "Астраханская область": {30, "UTC+3"}, 41 | "Белгородская область": {31, "UTC+3"}, 42 | "Брянская область": {32, "UTC+3"}, 43 | "Владимирская область": {33, "UTC+3"}, 44 | "Волгоградская область": {34, "UTC+3"}, 45 | "Вологодская область": {35, "UTC+3"}, 46 | "Воронежская область": {36, "UTC+3"}, 47 | "Ивановская область": {37, "UTC+3"}, 48 | "Иркутская область": {38, "UTC+8"}, 49 | "Калининградская область": {39, "UTC+2"}, 50 | "Калужская область": {40, "UTC+3"}, 51 | "Камчатский край": {41, "UTC+12"}, 52 | "Кемеровская область": {42, "UTC+7"}, 53 | "Кировская область": {43, "UTC+3"}, 54 | "Костромская область": {44, "UTC+3"}, 55 | "Курганская область": {45, "UTC+5"}, 56 | "Курская область": {46, "UTC+3"}, 57 | "Ленинградская область": {47, "UTC+3"}, 58 | "Липецкая область": {48, "UTC+3"}, 59 | "Магаданская область": {49, "UTC+10"}, 60 | "Московская область": {50, "UTC+3"}, 61 | "Мурманская область": {51, "UTC+3"}, 62 | "Нижегородская область": {52, "UTC+3"}, 63 | "Новгородская область": {53, "UTC+3"}, 64 | "Новосибирская область": {54, "UTC+6"}, 65 | "Омская область": {55, "UTC+6"}, 66 | "Оренбургская область": {56, "UTC+5"}, 67 | "Орловская область": {57, "UTC+3"}, 68 | "Пензенская область": {58, "UTC+3"}, 69 | "Пермский край": {59, "UTC+5"}, 70 | "Псковская область": {60, "UTC+3"}, 71 | "Ростовская область": {61, "UTC+3"}, 72 | "Рязанская область": {62, "UTC+3"}, 73 | "Самарская область": {63, "UTC+4"}, 74 | "Саратовская область": {64, "UTC+3"}, 75 | "Сахалинская область": {65, "UTC+11"}, 76 | "Свердловская область": {66, "UTC+5"}, 77 | "Смоленская область": {67, "UTC+3"}, 78 | "Тамбовская область": {68, "UTC+3"}, 79 | "Тверская область": {69, "UTC+3"}, 80 | "Томская область": {70, "UTC+6"}, 81 | "Тульская область": {71, "UTC+3"}, 82 | "Тюменская область": {72, "UTC+5"}, 83 | "Ульяновская область": {73, "UTC+3"}, 84 | "Челябинская область": {74, "UTC+5"}, 85 | "Забайкальский край": {75, "UTC+8"}, 86 | "Ярославская область": {76, "UTC+3"}, 87 | "Москва": {77, "UTC+3"}, 88 | "Санкт-Петербург": {78, "UTC+3"}, 89 | "Еврейская автономная область": {79, "UTC+10"}, 90 | "Ненецкий автономный округ": {83, "UTC+3"}, 91 | "Ханты-Мансийский автономный округ": {86, "UTC+5"}, 92 | "Чукотский автономный округ": {87, "UTC+12"}, 93 | "Ямало-Ненецкий автономный округ": {89, "UTC+5"}, 94 | "Республика Крым": {91, "UTC+3"}, 95 | } 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ip2Geo importer 2 | 3 | [![Test](https://github.com/m-messiah/ip2geo/actions/workflows/go.yml/badge.svg)](https://github.com/m-messiah/ip2geo/actions/workflows/go.yml)[![golangci-lint](https://github.com/m-messiah/ip2geo/actions/workflows/golangci-lint.yml/badge.svg)](https://github.com/m-messiah/ip2geo/actions/workflows/golangci-lint.yml)[![CodeQL](https://github.com/m-messiah/ip2geo/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/m-messiah/ip2geo/actions/workflows/codeql-analysis.yml)[![GitHub release](https://img.shields.io/github/release/m-messiah/ip2geo.svg?style=for-the-badge)](https://github.com/m-messiah/ip2geo/releases/latest)[![Github Releases](https://img.shields.io/github/downloads/m-messiah/ip2geo/total.svg?style=for-the-badge)](https://github.com/m-messiah/ip2geo/releases)[![Maintainability](https://api.codeclimate.com/v1/badges/8687e760d260b8499393/maintainability)](https://codeclimate.com/github/m-messiah/ip2geo/maintainability) 4 | 5 | Импортер ipgeo-данных в файлы, понятные для [nginx geo module](http://nginx.org/ru/docs/http/ngx_http_geo_module.html), с поддержкой кодов регионов РФ. 6 | 7 | Поддерживает Ipgeobase.ru, TOR-списки, MaxMind GeoLite (для городов и стран), базы IP2Proxy PX4 и PX4LITE. 8 | 9 | ## Установка 10 | 11 | 1. Скачать соответствующий архитектуре бинарник с github куда-нибудь в $PATH 12 | 2. Сделать его исполняемым 13 | 3. Пользоваться 14 | 15 | (также, при наличии Go окружения можно собрать самостоятельно через go get + go build) 16 | 17 | ## Запуск 18 | 19 | По умолчанию, ip2geo генерирует все возможные map-файлы, но все настраиваемо с помощью ключей: 20 | 21 | -c string 22 | Путь до конфигурационного файла (см. config.yaml.example) 23 | -output string 24 | Директория для записи map-файлов (по умолчанию: "output") 25 | -q Be quiet - skip [OK] 26 | -qq Be very quiet - show only errors 27 | -version Вывести текущую версию программы и выйти. 28 | -ipgeobase 29 | Генерация IPgeobase баз (название города, код региона, часовой пояс) 30 | -tor 31 | Генерация списков TOR нод. 32 | -ip2proxy 33 | Генерация ip2proxy PX4LITE сетей 34 | -ip2proxy-token string 35 | Токен для скачивания ip2proxy PX4LITE баз https://lite.ip2location.com/file-download 36 | -ip2proxy-lite-filename string 37 | Путь до уже скачанного zip файла PX4LITE 38 | -ip2proxy-pro 39 | Генерация ip2proxy PX4 сетей 40 | -ip2proxy-pro-token string 41 | Токен для скачивания ip2proxy PX4 42 | -ip2proxy-pro-filename string 43 | Путь до уже скачанного zip файла PX4 44 | -ip2proxy-print-type 45 | Вместо 1 указывать тип прокси из ip2proxy (PUB/DCH/e.t.c.) 46 | -maxmind 47 | Генерация баз MaxMind (название города, часовой пояс) 48 | Дальше параметры для MaxMind: 49 | -maxmind-license-key string 50 | Лицензионный ключ для MaxMind 51 | -maxmind-filename string 52 | Путь до уже скачанного файла MaxMind GeoLite2-City-CSV.csv 53 | -lang string 54 | Язык MaxMind баз (по умолчанию ru) 55 | -ipver int 56 | MaxMind версия IP (4 or 6) (default 4) 57 | -include string 58 | MaxMind фильтр: использовать только перечисленные страны 59 | Принимает список ISO-кодов стран, разделенных пробелами ("RU FR EN") 60 | -exclude string 61 | MaxMind фильтр: исключает из вывода перечисленные страны. (см формат выше) 62 | -nobase64 63 | Не перекодирует MaxMind города в base64, записывая их в map-файл как есть. Не используйте, если не уверены в кодировке MaxMind. 64 | -nocountry 65 | Не создавать map-файлы с названиями стран и iso-кодами стран из MaxMind 66 | 67 | 68 | ### Формат geomap-файлов 69 | 70 | geomap-файлы предназначены для использования в nginx в виде: 71 | 72 | ```nginx 73 | # Region 74 | geo $region { 75 | ranges; 76 | include geo/region.txt; 77 | } 78 | # City 79 | geo $city_geo { 80 | ranges; 81 | include geo/city.txt; 82 | } 83 | 84 | geo $city_mm { 85 | ranges; 86 | include geo/mm_city.txt; 87 | } 88 | 89 | map $city_geo $city { 90 | "" $city_mm; 91 | default $city_geo; 92 | } 93 | # Country 94 | geo $country { 95 | ranges; 96 | include geo/mm_country.txt; 97 | } 98 | # Country Code 99 | geo $country_code { 100 | ranges; 101 | include geo/mm_country_code.txt; 102 | } 103 | # TZ 104 | geo $tz_geo { 105 | ranges; 106 | include geo/tz.txt; 107 | } 108 | 109 | geo $tz_mm { 110 | ranges; 111 | include geo/mm_tz.txt; 112 | } 113 | 114 | map $tz_geo $tz { 115 | "" $tz_mm; 116 | default $tz_geo; 117 | } 118 | # Tor 119 | geo $is_tor { 120 | ranges; 121 | default 0; 122 | include geo/tor.txt; 123 | } 124 | # Proxy 125 | geo $is_proxy { 126 | ranges; 127 | default 0; 128 | include geo/ip2proxy_net.txt; 129 | } 130 | ``` 131 | 132 | Таким образом, IP адреса в файлах записаны в виде диапазона (range) и отсортированы по возрастанию IP. Карты сделаны каскадно, чтобы решить проблему пересечений диапазонов. IPGeobase используется в первую очередь, и если адрес там не найден, то MaxMind. 133 | 134 | Для того чтобы название города всегда отдавалось корректно - оно кодируется в base64 от utf8 (если не указан флаг `-nobase64`). 135 | -------------------------------------------------------------------------------- /maxmind.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | // MaxMind - GeoBase compatible generator for geolite.maxmind.com 17 | type MaxMind struct { 18 | archive []*zip.File 19 | OutputDir string 20 | ErrorsChan chan Error 21 | maxMindConfig 22 | } 23 | 24 | func (maxmind *MaxMind) name() string { 25 | return "MaxMind" 26 | } 27 | 28 | func (maxmind *MaxMind) addError(err Error) { 29 | maxmind.ErrorsChan <- err 30 | } 31 | 32 | func (maxmind *MaxMind) download() ([]byte, error) { 33 | // If used filename, no download 34 | if len(maxmind.Filename) > 0 { 35 | return os.ReadFile(maxmind.Filename) 36 | } 37 | client := &http.Client{} 38 | req, err := http.NewRequest("GET", "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City-CSV&suffix=zip", nil) 39 | if err != nil { 40 | return nil, err 41 | } 42 | q := req.URL.Query() 43 | q.Add("license_key", maxmind.LicenseKey) 44 | req.URL.RawQuery = q.Encode() 45 | resp, err := client.Do(req) 46 | if err != nil { 47 | return nil, err 48 | } 49 | defer resp.Body.Close() 50 | answer, err := io.ReadAll(resp.Body) 51 | if err != nil { 52 | return nil, err 53 | } 54 | return answer, nil 55 | } 56 | 57 | func (maxmind *MaxMind) unpack(response []byte) error { 58 | file, err := Unpack(response) 59 | if err == nil { 60 | maxmind.archive = file 61 | } 62 | return err 63 | } 64 | 65 | func (maxmind *MaxMind) lineToItem(record []string, currentTime time.Time) (*string, *geoItem, string, error) { 66 | if len(record) < 13 { 67 | return nil, nil, "FAIL", errors.New("too short line") 68 | } 69 | countryCode := record[4] 70 | if len(countryCode) < 1 || len(record[5]) < 1 { 71 | return nil, nil, "", errors.New("too short country") 72 | } 73 | if len(maxmind.Include) > 1 && !strings.Contains(maxmind.Include, countryCode) { 74 | return nil, nil, "", errors.New("country skipped") 75 | } 76 | if strings.Contains(maxmind.Exclude, countryCode) { 77 | return nil, nil, "", errors.New("country Excluded") 78 | } 79 | tz := record[12] 80 | if !maxmind.TZNames { 81 | tz = convertTZToOffset(currentTime, record[12]) 82 | } 83 | if len(record[10]) < 1 { 84 | return nil, nil, "", errors.New("too short city name") 85 | } 86 | return &record[0], &geoItem{ 87 | ID: record[0], 88 | City: record[10], 89 | TZ: tz, 90 | CountryCode: record[4], 91 | Country: record[5], 92 | }, "", nil 93 | } 94 | 95 | func (maxmind *MaxMind) citiesDB() (map[string]geoItem, error) { 96 | locations := make(map[string]geoItem) 97 | currentTime := time.Now() 98 | filename := "GeoLite2-City-Locations-" + maxmind.Lang + ".csv" 99 | for record := range readCSVDatabase(maxmind.archive, filename, "MaxMind", ',', false) { 100 | key, location, severity, err := maxmind.lineToItem(record, currentTime) 101 | if err != nil { 102 | if len(severity) > 0 { 103 | printMessage("MaxMind", fmt.Sprintf(filename+" %v", err), severity) 104 | } 105 | continue 106 | } 107 | locations[*key] = *location 108 | } 109 | if len(locations) < 1 { 110 | return nil, errors.New("Locations db is empty") 111 | } 112 | return locations, nil 113 | } 114 | 115 | func (maxmind *MaxMind) parseNetwork(locations map[string]geoItem) <-chan geoItem { 116 | database := make(chan geoItem) 117 | go func() { 118 | var ipRange string 119 | var geoID string 120 | filename := "GeoLite2-City-Blocks-IPv" + strconv.Itoa(maxmind.IPVer) + ".csv" 121 | for record := range readCSVDatabase(maxmind.archive, filename, "MaxMind", ',', false) { 122 | if len(record) < 2 { 123 | printMessage("MaxMind", fmt.Sprintf(filename+" too short line: %s", record), "FAIL") 124 | continue 125 | } 126 | ipRange = getIPRange(maxmind.IPVer, record[0]) 127 | if ipRange == "" { 128 | continue 129 | } 130 | geoID = record[1] 131 | if location, ok := locations[geoID]; ok { 132 | location.Network = ipRange 133 | database <- location 134 | } 135 | } 136 | close(database) 137 | }() 138 | return database 139 | } 140 | 141 | func (maxmind *MaxMind) writeMap(locations map[string]geoItem) error { 142 | city, err := openMapFile(maxmind.OutputDir, "mm_city.txt") 143 | if err != nil { 144 | return err 145 | } 146 | tz, err := openMapFile(maxmind.OutputDir, "mm_tz.txt") 147 | if err != nil { 148 | return err 149 | } 150 | var country *os.File 151 | var countryCode *os.File 152 | if !maxmind.NoCountry { 153 | country, err = openMapFile(maxmind.OutputDir, "mm_country.txt") 154 | if err != nil { 155 | return err 156 | } 157 | countryCode, err = openMapFile(maxmind.OutputDir, "mm_country_code.txt") 158 | if err != nil { 159 | return err 160 | } 161 | defer country.Close() 162 | defer countryCode.Close() 163 | } 164 | defer city.Close() 165 | defer tz.Close() 166 | 167 | for location := range maxmind.parseNetwork(locations) { 168 | var cityName string 169 | var countryName string 170 | if maxmind.NoBase64 { 171 | cityName = "\"" + strings.Replace(location.City, "\"", "\\\"", -1) + "\"" 172 | countryName = "\"" + strings.Replace(location.Country, "\"", "\\\"", -1) + "\"" 173 | } else { 174 | cityName = base64.StdEncoding.EncodeToString([]byte(location.City)) 175 | countryName = base64.StdEncoding.EncodeToString([]byte(location.Country)) 176 | } 177 | 178 | fmt.Fprintf(city, "%s %s;\n", location.Network, cityName) 179 | fmt.Fprintf(tz, "%s %s;\n", location.Network, location.TZ) 180 | if !maxmind.NoCountry { 181 | fmt.Fprintf(country, "%s %s;\n", location.Network, countryName) 182 | fmt.Fprintf(countryCode, "%s %s;\n", location.Network, location.CountryCode) 183 | } 184 | } 185 | return nil 186 | } 187 | -------------------------------------------------------------------------------- /app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/jinzhu/configor" 9 | ) 10 | 11 | type ip2ProxyConfig struct { 12 | Enabled bool `default:"false"` 13 | Token string 14 | Filename string 15 | } 16 | 17 | type maxMindConfig struct { 18 | Enabled bool `default:"false"` 19 | LicenseKey string 20 | Filename string 21 | IPVer int `default:"4"` 22 | Lang string `default:"ru"` 23 | TZNames bool `default:"false"` 24 | Include string 25 | Exclude string 26 | NoBase64 bool `default:"false"` 27 | NoCountry bool `default:"false"` 28 | } 29 | 30 | // Config - all configuration for tool defined here 31 | var Config = struct { 32 | LogLevel int `default:"0"` 33 | OutputDir string `default:"output"` 34 | TOR struct { 35 | Enabled bool `default:"false"` 36 | } 37 | IP2Proxy struct { 38 | Lite ip2ProxyConfig 39 | Pro ip2ProxyConfig 40 | PrintType bool `default:"false"` 41 | } 42 | MaxMind maxMindConfig 43 | IPGeobase struct { 44 | Enabled bool `default:"false"` 45 | } 46 | }{} 47 | 48 | func configLoad() { 49 | configFile := flag.String("c", "", "Read config from file") 50 | quiet := flag.Bool("q", false, "Be quiet - skip [OK]") 51 | veryQuiet := flag.Bool("qq", false, "Be very quiet - show only errors") 52 | version := flag.Bool("version", false, "Print version information and exit") 53 | 54 | flag.StringVar(&Config.OutputDir, "output", "output", "output directory for files") 55 | flag.BoolVar(&Config.IPGeobase.Enabled, "ipgeobase", false, "enable ipgeobase generation") 56 | flag.BoolVar(&Config.TOR.Enabled, "tor", false, "enable tor generation") 57 | flag.BoolVar(&Config.IP2Proxy.Lite.Enabled, "ip2proxy", false, "enable ip2proxy PX4-LITE generation") 58 | flag.BoolVar(&Config.IP2Proxy.Pro.Enabled, "ip2proxy-pro", false, "enable ip2proxy PX4 generation") 59 | flag.StringVar(&Config.IP2Proxy.Lite.Token, "ip2proxy-token", "", "Get token here https://lite.ip2location.com/file-download") 60 | flag.StringVar(&Config.IP2Proxy.Pro.Token, "ip2proxy-pro-token", "", "ip2proxy download token") 61 | flag.StringVar(&Config.IP2Proxy.Lite.Filename, "ip2proxy-lite-filename", "", "Filename of already downloaded ip2proxy-lite db") 62 | flag.StringVar(&Config.IP2Proxy.Pro.Filename, "ip2proxy-pro-filename", "", "Filename of already downloaded ip2proxy db") 63 | flag.BoolVar(&Config.IP2Proxy.PrintType, "ip2proxy-print-type", false, "Print proxy type in map, instead of `1`") 64 | flag.BoolVar(&Config.MaxMind.Enabled, "maxmind", false, "enable MaxMind generation") 65 | flag.StringVar(&Config.MaxMind.LicenseKey, "maxmind-license-key", "", "MaxMind license key for download") 66 | flag.StringVar(&Config.MaxMind.Filename, "maxmind-filename", "", "Filename of already downloaded MaxMind db") 67 | flag.IntVar(&Config.MaxMind.IPVer, "ipver", 4, "MaxMind ip version (4 or 6)") 68 | flag.StringVar(&Config.MaxMind.Lang, "lang", "ru", "MaxMind city name language") 69 | flag.BoolVar(&Config.MaxMind.TZNames, "tznames", false, "MaxMind TZ in names format (for example `Europe/Moscow`)") 70 | flag.StringVar(&Config.MaxMind.Include, "include", "", "MaxMind output filter: only these countries") 71 | flag.StringVar(&Config.MaxMind.Exclude, "exclude", "", "MaxMind output filter: except these countries") 72 | flag.BoolVar(&Config.MaxMind.NoBase64, "nobase64", false, "MaxMind Cities as-is (without base64 encode). DO NOT USE IT IF YOU NOT SURE ABOUT MaxMind encoding") 73 | flag.BoolVar(&Config.MaxMind.NoCountry, "nocountry", false, "do not add MaxMind country maps") 74 | flag.Parse() 75 | if *version { 76 | printMessage("ip2geo", "version "+VERSION, "OK") 77 | os.Exit(0) 78 | } 79 | err := configor.New(&configor.Config{Silent: true}).Load(&Config, *configFile) 80 | if err != nil { 81 | printMessage("ip2geo", fmt.Sprintf("configor failed: %s", err), "FAIL") 82 | os.Exit(1) 83 | } 84 | if *quiet { 85 | Config.LogLevel = 1 86 | } 87 | if *veryQuiet { 88 | Config.LogLevel = 2 89 | } 90 | 91 | if !(Config.IPGeobase.Enabled || Config.TOR.Enabled || Config.MaxMind.Enabled || Config.IP2Proxy.Lite.Enabled || Config.IP2Proxy.Pro.Enabled) { 92 | // By default, generate all maps except IPGeobase 93 | Config.IPGeobase.Enabled = false 94 | Config.TOR.Enabled = true 95 | Config.MaxMind.Enabled = Config.MaxMind.LicenseKey != "" || Config.MaxMind.Filename != "" 96 | Config.IP2Proxy.Lite.Enabled = Config.IP2Proxy.Lite.Token != "" || Config.IP2Proxy.Lite.Filename != "" 97 | Config.IP2Proxy.Pro.Enabled = Config.IP2Proxy.Pro.Token != "" || Config.IP2Proxy.Pro.Filename != "" 98 | } 99 | } 100 | 101 | func main() { 102 | configLoad() 103 | 104 | _ = os.MkdirAll(Config.OutputDir, 0755) 105 | if Config.LogLevel < 2 { 106 | printMessage(" ", "Use output directory", Config.OutputDir) 107 | } 108 | goroutinesCount := 0 109 | errorChannel := make(chan Error) 110 | if Config.IPGeobase.Enabled { 111 | goroutinesCount++ 112 | i := IPGeobase{ 113 | OutputDir: Config.OutputDir, 114 | ErrorsChan: errorChannel, 115 | } 116 | go Generate(&i) 117 | } 118 | 119 | if Config.TOR.Enabled { 120 | goroutinesCount++ 121 | t := Tor{ 122 | OutputDir: Config.OutputDir, 123 | ErrorsChan: errorChannel, 124 | } 125 | go t.Generate() 126 | } 127 | 128 | if Config.MaxMind.Enabled { 129 | goroutinesCount++ 130 | m := MaxMind{ 131 | OutputDir: Config.OutputDir, 132 | ErrorsChan: errorChannel, 133 | maxMindConfig: Config.MaxMind, 134 | } 135 | go Generate(&m) 136 | } 137 | 138 | for t, ip2ProxyType := range [2]ip2ProxyConfig{Config.IP2Proxy.Lite, Config.IP2Proxy.Pro} { 139 | if ip2ProxyType.Enabled { 140 | goroutinesCount++ 141 | var name string 142 | if t == 0 { 143 | name = "ip2proxyLite" 144 | } else { 145 | name = "ip2proxyPro" 146 | 147 | } 148 | o := ip2proxy{ 149 | Name: name, 150 | Token: ip2ProxyType.Token, 151 | Filename: ip2ProxyType.Filename, 152 | ErrorsChan: errorChannel, 153 | OutputDir: Config.OutputDir, 154 | PrintType: Config.IP2Proxy.PrintType, 155 | } 156 | go o.Get() 157 | } 158 | } 159 | 160 | for i := 0; i < goroutinesCount; i++ { 161 | err := <-errorChannel 162 | if err.err != nil { 163 | printMessage(err.Module, err.Action+": "+err.err.Error(), "FAIL") 164 | os.Exit(1) 165 | } 166 | } 167 | if Config.LogLevel < 1 { 168 | printMessage(" ", "Generation done", "OK") 169 | } 170 | } 171 | --------------------------------------------------------------------------------