├── .github ├── release-branch.sh ├── release-rule-set.sh ├── renovate.json ├── update_dependencies.sh └── workflows │ ├── build.yaml │ └── release.yaml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── go.mod ├── go.sum └── main.go /.github/release-branch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -o pipefail 4 | 5 | mkdir -p release 6 | cd release 7 | git init 8 | git config --local user.email "github-action@users.noreply.github.com" 9 | git config --local user.name "GitHub Action" 10 | git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geosite.git 11 | git branch -M release 12 | cp ../*.db ../*.sha256sum . 13 | git add . 14 | git commit -m "Update release" 15 | git push -f origin release 16 | -------------------------------------------------------------------------------- /.github/release-rule-set.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -o pipefail 4 | 5 | function releaseRuleSet() { 6 | dirName=$1 7 | pushd $dirName 8 | git init 9 | git config --local user.email "github-action@users.noreply.github.com" 10 | git config --local user.name "GitHub Action" 11 | git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geosite.git 12 | git branch -M $dirName 13 | git add . 14 | git commit -m "Update rule-set" 15 | git push -f origin $dirName 16 | popd 17 | } 18 | 19 | releaseRuleSet rule-set 20 | releaseRuleSet rule-set-unstable 21 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "commitMessagePrefix": "[dependencies]", 4 | "extends": [ 5 | "config:base", 6 | ":disableRateLimiting" 7 | ], 8 | "baseBranches": [ 9 | "main" 10 | ], 11 | "golang": { 12 | "enabled": false 13 | }, 14 | "packageRules": [ 15 | { 16 | "matchManagers": [ 17 | "github-actions" 18 | ], 19 | "groupName": "github-actions" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /.github/update_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROJECTS=$(dirname "$0")/../.. 4 | 5 | go get -x github.com/sagernet/sing-box@$(git -C $PROJECTS/sing-box rev-parse HEAD) 6 | go mod tidy 7 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | build: 8 | name: Build 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v3 13 | with: 14 | fetch-depth: 0 15 | - name: Setup Go 16 | uses: actions/setup-go@v5 17 | with: 18 | go-version: ^1.22 19 | - name: Build geosite 20 | id: build 21 | env: 22 | NO_SKIP: true 23 | run: | 24 | go run -v . -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: "0 0 * * *" 6 | jobs: 7 | build: 8 | name: Build 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v3 13 | with: 14 | fetch-depth: 0 15 | - name: Setup Go 16 | uses: actions/setup-go@v5 17 | with: 18 | go-version: ^1.22 19 | - name: Build geosite 20 | id: build 21 | run: | 22 | go run -v . 23 | - name: Release rule sets 24 | if: steps.build.outputs.skip != 'true' 25 | run: .github/release-rule-set.sh 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | - name: Generate sha256 hash 29 | if: steps.build.outputs.skip != 'true' 30 | run: | 31 | sha256sum geosite.db > geosite.db.sha256sum 32 | sha256sum geosite-cn.db > geosite-cn.db.sha256sum 33 | - name: Release release branch 34 | if: steps.build.outputs.skip != 'true' 35 | run: .github/release-branch.sh 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | - uses: dev-drprasad/delete-older-releases@v0.3.2 39 | if: steps.build.outputs.skip != 'true' 40 | with: 41 | keep_latest: 10 42 | env: 43 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 44 | - name: Release geosite 45 | if: steps.build.outputs.skip != 'true' 46 | uses: softprops/action-gh-release@v1 47 | with: 48 | tag_name: ${{ steps.build.outputs.tag }} 49 | files: | 50 | geosite.db 51 | geosite.db.sha256sum 52 | geosite-cn.db 53 | geosite-cn.db.sha256sum 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /vendor/ 3 | **.db 4 | /rule-set/ 5 | /rule-set-unstable/ 6 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | - gofumpt 5 | - govet 6 | - gci 7 | - staticcheck 8 | 9 | linters-settings: 10 | gci: 11 | custom-order: true 12 | sections: 13 | - standard 14 | - prefix(github.com/sagernet/) 15 | - default 16 | staticcheck: 17 | go: '1.20' 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2022 by nekohasekai 2 | 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, either version 3 of the License, or 6 | (at your option) any later version. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU General Public License for more details. 12 | 13 | You should have received a copy of the GNU General Public License 14 | along with this program. If not, see . -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | fmt: 2 | @gofumpt -l -w . 3 | @gofmt -s -w . 4 | @gci write --custom-order -s standard -s "prefix(github.com/sagernet/)" -s "default" . 5 | 6 | fmt_install: 7 | go install -v mvdan.cc/gofumpt@latest 8 | go install -v github.com/daixiang0/gci@latest 9 | 10 | lint: 11 | golangci-lint run ./... 12 | 13 | lint_install: 14 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 15 | 16 | test: 17 | go test -v ./... -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sagernet/sing-geosite 2 | 3 | go 1.21 4 | 5 | toolchain go1.22.3 6 | 7 | require ( 8 | github.com/google/go-github/v45 v45.2.0 9 | github.com/sagernet/sing v0.5.0-beta.1 10 | github.com/sagernet/sing-box v1.9.5-0.20240912063220-bed673aa630c 11 | github.com/v2fly/v2ray-core/v5 v5.18.0 12 | google.golang.org/protobuf v1.34.2 13 | ) 14 | 15 | require ( 16 | github.com/adrg/xdg v0.5.0 // indirect 17 | github.com/golang/protobuf v1.5.4 // indirect 18 | github.com/google/go-querystring v1.1.0 // indirect 19 | github.com/logrusorgru/aurora v2.0.3+incompatible // indirect 20 | github.com/miekg/dns v1.1.62 // indirect 21 | github.com/sagernet/sing-dns v0.3.0-beta.14 // indirect 22 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect 23 | golang.org/x/crypto v0.27.0 // indirect 24 | golang.org/x/mod v0.19.0 // indirect 25 | golang.org/x/net v0.29.0 // indirect 26 | golang.org/x/sync v0.8.0 // indirect 27 | golang.org/x/sys v0.25.0 // indirect 28 | golang.org/x/tools v0.23.0 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= 2 | github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 6 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= 7 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 8 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 9 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 11 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 12 | github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= 13 | github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= 14 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 15 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 16 | github.com/google/pprof v0.0.0-20240320155624-b11c3daa6f07 h1:57oOH2Mu5Nw16KnZAVLdlUjmPH/TSYCKTJgG0OVfX0Y= 17 | github.com/google/pprof v0.0.0-20240320155624-b11c3daa6f07/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= 18 | github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= 19 | github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 20 | github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= 21 | github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= 22 | github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= 23 | github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= 24 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 26 | github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= 27 | github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= 28 | github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs= 29 | github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= 30 | github.com/sagernet/quic-go v0.47.0-beta.2 h1:1tCGWFOSaXIeuQaHrwOMJIYvlupjTcaVInGQw5ArULU= 31 | github.com/sagernet/quic-go v0.47.0-beta.2/go.mod h1:bLVKvElSEMNv7pu7SZHscW02TYigzQ5lQu3Nh4wNh8Q= 32 | github.com/sagernet/sing v0.5.0-beta.1 h1:THZMZgJcDQxutE++6Ckih1HlvMtXple94RBGa6GSg2I= 33 | github.com/sagernet/sing v0.5.0-beta.1/go.mod h1:ARkL0gM13/Iv5VCZmci/NuoOlePoIsW0m7BWfln/Hak= 34 | github.com/sagernet/sing-box v1.9.5-0.20240912063220-bed673aa630c h1:iLUOSU5rx96BJAgTbqySE9zloGOvIk18MDHznU5+ass= 35 | github.com/sagernet/sing-box v1.9.5-0.20240912063220-bed673aa630c/go.mod h1:pcO6hmTsvx1LS3DkTz0P1zjQmEFCStINed4ar5NDQzg= 36 | github.com/sagernet/sing-dns v0.3.0-beta.14 h1:/s+fJzYKsvLaNDt/2rjpsrDcN8wmCO2JbX6OFrl8Nww= 37 | github.com/sagernet/sing-dns v0.3.0-beta.14/go.mod h1:rscgSr5ixOPk8XM9ZMLuMXCyldEQ1nLvdl0nfv+lp00= 38 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 39 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 40 | github.com/v2fly/v2ray-core/v5 v5.18.0 h1:KSw6Q7YPuV1ZAQzdjwxQqCKBIkZnp0DewjjPDEYidAg= 41 | github.com/v2fly/v2ray-core/v5 v5.18.0/go.mod h1:qC7xF/dQh/Dy+kFxn/4/KN3OXeuliG8IJM4AmG5dTO0= 42 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= 43 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= 44 | golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= 45 | golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= 46 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= 47 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= 48 | golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= 49 | golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 50 | golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= 51 | golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= 52 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 53 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 54 | golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= 55 | golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 56 | golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= 57 | golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= 58 | golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= 59 | golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= 60 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 61 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 62 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 63 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 64 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 65 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "crypto/sha256" 7 | "encoding/hex" 8 | "io" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "sort" 13 | "strings" 14 | 15 | "github.com/sagernet/sing-box/common/geosite" 16 | "github.com/sagernet/sing-box/common/srs" 17 | C "github.com/sagernet/sing-box/constant" 18 | "github.com/sagernet/sing-box/log" 19 | "github.com/sagernet/sing-box/option" 20 | "github.com/sagernet/sing/common" 21 | E "github.com/sagernet/sing/common/exceptions" 22 | 23 | "github.com/google/go-github/v45/github" 24 | "github.com/v2fly/v2ray-core/v5/app/router/routercommon" 25 | "google.golang.org/protobuf/proto" 26 | ) 27 | 28 | var githubClient *github.Client 29 | 30 | func init() { 31 | accessToken, loaded := os.LookupEnv("ACCESS_TOKEN") 32 | if !loaded { 33 | githubClient = github.NewClient(nil) 34 | return 35 | } 36 | transport := &github.BasicAuthTransport{ 37 | Username: accessToken, 38 | } 39 | githubClient = github.NewClient(transport.Client()) 40 | } 41 | 42 | func fetch(from string) (*github.RepositoryRelease, error) { 43 | names := strings.SplitN(from, "/", 2) 44 | latestRelease, _, err := githubClient.Repositories.GetLatestRelease(context.Background(), names[0], names[1]) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return latestRelease, err 49 | } 50 | 51 | func get(downloadURL *string) ([]byte, error) { 52 | log.Info("download ", *downloadURL) 53 | response, err := http.Get(*downloadURL) 54 | if err != nil { 55 | return nil, err 56 | } 57 | defer response.Body.Close() 58 | return io.ReadAll(response.Body) 59 | } 60 | 61 | func download(release *github.RepositoryRelease) ([]byte, error) { 62 | geositeAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool { 63 | return *it.Name == "dlc.dat" 64 | }) 65 | geositeChecksumAsset := common.Find(release.Assets, func(it *github.ReleaseAsset) bool { 66 | return *it.Name == "dlc.dat.sha256sum" 67 | }) 68 | if geositeAsset == nil { 69 | return nil, E.New("geosite asset not found in upstream release ", release.Name) 70 | } 71 | if geositeChecksumAsset == nil { 72 | return nil, E.New("geosite asset not found in upstream release ", release.Name) 73 | } 74 | data, err := get(geositeAsset.BrowserDownloadURL) 75 | if err != nil { 76 | return nil, err 77 | } 78 | remoteChecksum, err := get(geositeChecksumAsset.BrowserDownloadURL) 79 | if err != nil { 80 | return nil, err 81 | } 82 | checksum := sha256.Sum256(data) 83 | if hex.EncodeToString(checksum[:]) != string(remoteChecksum[:64]) { 84 | return nil, E.New("checksum mismatch") 85 | } 86 | return data, nil 87 | } 88 | 89 | func parse(vGeositeData []byte) (map[string][]geosite.Item, error) { 90 | vGeositeList := routercommon.GeoSiteList{} 91 | err := proto.Unmarshal(vGeositeData, &vGeositeList) 92 | if err != nil { 93 | return nil, err 94 | } 95 | domainMap := make(map[string][]geosite.Item) 96 | for _, vGeositeEntry := range vGeositeList.Entry { 97 | code := strings.ToLower(vGeositeEntry.CountryCode) 98 | domains := make([]geosite.Item, 0, len(vGeositeEntry.Domain)*2) 99 | attributes := make(map[string][]*routercommon.Domain) 100 | for _, domain := range vGeositeEntry.Domain { 101 | if len(domain.Attribute) > 0 { 102 | for _, attribute := range domain.Attribute { 103 | attributes[attribute.Key] = append(attributes[attribute.Key], domain) 104 | } 105 | } 106 | switch domain.Type { 107 | case routercommon.Domain_Plain: 108 | domains = append(domains, geosite.Item{ 109 | Type: geosite.RuleTypeDomainKeyword, 110 | Value: domain.Value, 111 | }) 112 | case routercommon.Domain_Regex: 113 | domains = append(domains, geosite.Item{ 114 | Type: geosite.RuleTypeDomainRegex, 115 | Value: domain.Value, 116 | }) 117 | case routercommon.Domain_RootDomain: 118 | if strings.Contains(domain.Value, ".") { 119 | domains = append(domains, geosite.Item{ 120 | Type: geosite.RuleTypeDomain, 121 | Value: domain.Value, 122 | }) 123 | } 124 | domains = append(domains, geosite.Item{ 125 | Type: geosite.RuleTypeDomainSuffix, 126 | Value: "." + domain.Value, 127 | }) 128 | case routercommon.Domain_Full: 129 | domains = append(domains, geosite.Item{ 130 | Type: geosite.RuleTypeDomain, 131 | Value: domain.Value, 132 | }) 133 | } 134 | } 135 | domainMap[code] = common.Uniq(domains) 136 | for attribute, attributeEntries := range attributes { 137 | attributeDomains := make([]geosite.Item, 0, len(attributeEntries)*2) 138 | for _, domain := range attributeEntries { 139 | switch domain.Type { 140 | case routercommon.Domain_Plain: 141 | attributeDomains = append(attributeDomains, geosite.Item{ 142 | Type: geosite.RuleTypeDomainKeyword, 143 | Value: domain.Value, 144 | }) 145 | case routercommon.Domain_Regex: 146 | attributeDomains = append(attributeDomains, geosite.Item{ 147 | Type: geosite.RuleTypeDomainRegex, 148 | Value: domain.Value, 149 | }) 150 | case routercommon.Domain_RootDomain: 151 | if strings.Contains(domain.Value, ".") { 152 | attributeDomains = append(attributeDomains, geosite.Item{ 153 | Type: geosite.RuleTypeDomain, 154 | Value: domain.Value, 155 | }) 156 | } 157 | attributeDomains = append(attributeDomains, geosite.Item{ 158 | Type: geosite.RuleTypeDomainSuffix, 159 | Value: "." + domain.Value, 160 | }) 161 | case routercommon.Domain_Full: 162 | attributeDomains = append(attributeDomains, geosite.Item{ 163 | Type: geosite.RuleTypeDomain, 164 | Value: domain.Value, 165 | }) 166 | } 167 | } 168 | domainMap[code+"@"+attribute] = common.Uniq(attributeDomains) 169 | } 170 | } 171 | return domainMap, nil 172 | } 173 | 174 | type filteredCodePair struct { 175 | code string 176 | badCode string 177 | } 178 | 179 | func filterTags(data map[string][]geosite.Item) { 180 | var codeList []string 181 | for code := range data { 182 | codeList = append(codeList, code) 183 | } 184 | var badCodeList []filteredCodePair 185 | var filteredCodeMap []string 186 | var mergedCodeMap []string 187 | for _, code := range codeList { 188 | codeParts := strings.Split(code, "@") 189 | if len(codeParts) != 2 { 190 | continue 191 | } 192 | leftParts := strings.Split(codeParts[0], "-") 193 | var lastName string 194 | if len(leftParts) > 1 { 195 | lastName = leftParts[len(leftParts)-1] 196 | } 197 | if lastName == "" { 198 | lastName = codeParts[0] 199 | } 200 | if lastName == codeParts[1] { 201 | delete(data, code) 202 | filteredCodeMap = append(filteredCodeMap, code) 203 | continue 204 | } 205 | if "!"+lastName == codeParts[1] { 206 | badCodeList = append(badCodeList, filteredCodePair{ 207 | code: codeParts[0], 208 | badCode: code, 209 | }) 210 | } else if lastName == "!"+codeParts[1] { 211 | badCodeList = append(badCodeList, filteredCodePair{ 212 | code: codeParts[0], 213 | badCode: code, 214 | }) 215 | } 216 | } 217 | for _, it := range badCodeList { 218 | badList := data[it.badCode] 219 | if badList == nil { 220 | panic("bad list not found: " + it.badCode) 221 | } 222 | delete(data, it.badCode) 223 | newMap := make(map[geosite.Item]bool) 224 | for _, item := range data[it.code] { 225 | newMap[item] = true 226 | } 227 | for _, item := range badList { 228 | delete(newMap, item) 229 | } 230 | newList := make([]geosite.Item, 0, len(newMap)) 231 | for item := range newMap { 232 | newList = append(newList, item) 233 | } 234 | data[it.code] = newList 235 | mergedCodeMap = append(mergedCodeMap, it.badCode) 236 | } 237 | sort.Strings(filteredCodeMap) 238 | sort.Strings(mergedCodeMap) 239 | os.Stderr.WriteString("filtered " + strings.Join(filteredCodeMap, ",") + "\n") 240 | os.Stderr.WriteString("merged " + strings.Join(mergedCodeMap, ",") + "\n") 241 | } 242 | 243 | func mergeTags(data map[string][]geosite.Item) { 244 | var codeList []string 245 | for code := range data { 246 | codeList = append(codeList, code) 247 | } 248 | var cnCodeList []string 249 | for _, code := range codeList { 250 | codeParts := strings.Split(code, "@") 251 | if len(codeParts) != 2 { 252 | continue 253 | } 254 | if codeParts[1] != "cn" { 255 | continue 256 | } 257 | if !strings.HasPrefix(codeParts[0], "category-") { 258 | continue 259 | } 260 | if strings.HasSuffix(codeParts[0], "-cn") || strings.HasSuffix(codeParts[0], "-!cn") { 261 | continue 262 | } 263 | cnCodeList = append(cnCodeList, code) 264 | } 265 | for _, code := range codeList { 266 | if !strings.HasPrefix(code, "category-") { 267 | continue 268 | } 269 | if !strings.HasSuffix(code, "-cn") { 270 | continue 271 | } 272 | if strings.Contains(code, "@") { 273 | continue 274 | } 275 | cnCodeList = append(cnCodeList, code) 276 | } 277 | newMap := make(map[geosite.Item]bool) 278 | for _, item := range data["geolocation-cn"] { 279 | newMap[item] = true 280 | } 281 | for _, code := range cnCodeList { 282 | for _, item := range data[code] { 283 | newMap[item] = true 284 | } 285 | } 286 | newList := make([]geosite.Item, 0, len(newMap)) 287 | for item := range newMap { 288 | newList = append(newList, item) 289 | } 290 | data["geolocation-cn"] = newList 291 | data["cn"] = append(newList, geosite.Item{ 292 | Type: geosite.RuleTypeDomainSuffix, 293 | Value: "cn", 294 | }) 295 | println("merged cn categories: " + strings.Join(cnCodeList, ",")) 296 | } 297 | 298 | func generate(release *github.RepositoryRelease, output string, cnOutput string, ruleSetOutput string, ruleSetUnstableOutput string) error { 299 | vData, err := download(release) 300 | if err != nil { 301 | return err 302 | } 303 | domainMap, err := parse(vData) 304 | if err != nil { 305 | return err 306 | } 307 | filterTags(domainMap) 308 | mergeTags(domainMap) 309 | outputPath, _ := filepath.Abs(output) 310 | os.Stderr.WriteString("write " + outputPath + "\n") 311 | outputFile, err := os.Create(output) 312 | if err != nil { 313 | return err 314 | } 315 | defer outputFile.Close() 316 | writer := bufio.NewWriter(outputFile) 317 | err = geosite.Write(writer, domainMap) 318 | if err != nil { 319 | return err 320 | } 321 | err = writer.Flush() 322 | if err != nil { 323 | return err 324 | } 325 | cnCodes := []string{ 326 | "geolocation-cn", 327 | } 328 | cnDomainMap := make(map[string][]geosite.Item) 329 | for _, cnCode := range cnCodes { 330 | cnDomainMap[cnCode] = domainMap[cnCode] 331 | } 332 | cnOutputFile, err := os.Create(cnOutput) 333 | if err != nil { 334 | return err 335 | } 336 | defer cnOutputFile.Close() 337 | writer.Reset(cnOutputFile) 338 | err = geosite.Write(writer, cnDomainMap) 339 | if err != nil { 340 | return err 341 | } 342 | err = writer.Flush() 343 | if err != nil { 344 | return err 345 | } 346 | os.RemoveAll(ruleSetOutput) 347 | os.RemoveAll(ruleSetUnstableOutput) 348 | err = os.MkdirAll(ruleSetOutput, 0o755) 349 | err = os.MkdirAll(ruleSetUnstableOutput, 0o755) 350 | if err != nil { 351 | return err 352 | } 353 | for code, domains := range domainMap { 354 | var headlessRule option.DefaultHeadlessRule 355 | defaultRule := geosite.Compile(domains) 356 | headlessRule.Domain = defaultRule.Domain 357 | headlessRule.DomainSuffix = defaultRule.DomainSuffix 358 | headlessRule.DomainKeyword = defaultRule.DomainKeyword 359 | headlessRule.DomainRegex = defaultRule.DomainRegex 360 | var plainRuleSet option.PlainRuleSet 361 | plainRuleSet.Rules = []option.HeadlessRule{ 362 | { 363 | Type: C.RuleTypeDefault, 364 | DefaultOptions: headlessRule, 365 | }, 366 | } 367 | srsPath, _ := filepath.Abs(filepath.Join(ruleSetOutput, "geosite-"+code+".srs")) 368 | unstableSRSPath, _ := filepath.Abs(filepath.Join(ruleSetUnstableOutput, "geosite-"+code+".srs")) 369 | // os.Stderr.WriteString("write " + srsPath + "\n") 370 | var ( 371 | outputRuleSet *os.File 372 | outputRuleSetUnstable *os.File 373 | ) 374 | outputRuleSet, err = os.Create(srsPath) 375 | if err != nil { 376 | return err 377 | } 378 | err = srs.Write(outputRuleSet, plainRuleSet, false) 379 | outputRuleSet.Close() 380 | if err != nil { 381 | return err 382 | } 383 | outputRuleSetUnstable, err = os.Create(unstableSRSPath) 384 | if err != nil { 385 | return err 386 | } 387 | err = srs.Write(outputRuleSetUnstable, plainRuleSet, true) 388 | outputRuleSetUnstable.Close() 389 | if err != nil { 390 | return err 391 | } 392 | } 393 | return nil 394 | } 395 | 396 | func setActionOutput(name string, content string) { 397 | os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n") 398 | } 399 | 400 | func release(source string, destination string, output string, cnOutput string, ruleSetOutput string, ruleSetOutputUnstable string) error { 401 | sourceRelease, err := fetch(source) 402 | if err != nil { 403 | return err 404 | } 405 | destinationRelease, err := fetch(destination) 406 | if err != nil { 407 | log.Warn("missing destination latest release") 408 | } else { 409 | if os.Getenv("NO_SKIP") != "true" && strings.Contains(*destinationRelease.Name, *sourceRelease.Name) { 410 | log.Info("already latest") 411 | setActionOutput("skip", "true") 412 | return nil 413 | } 414 | } 415 | err = generate(sourceRelease, output, cnOutput, ruleSetOutput, ruleSetOutputUnstable) 416 | if err != nil { 417 | return err 418 | } 419 | setActionOutput("tag", *sourceRelease.Name) 420 | return nil 421 | } 422 | 423 | func main() { 424 | err := release( 425 | "v2fly/domain-list-community", 426 | "sagernet/sing-geosite", 427 | "geosite.db", 428 | "geosite-cn.db", 429 | "rule-set", 430 | "rule-set-unstable", 431 | ) 432 | if err != nil { 433 | log.Fatal(err) 434 | } 435 | } 436 | --------------------------------------------------------------------------------