├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── go.yml │ ├── golangci-lint.yml │ ├── modver.yml │ └── zizmor.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── convert ├── convert.go └── convert_test.go ├── go.mod ├── go.sum └── main.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "Code scanning - action" 2 | 3 | on: 4 | pull_request: 5 | schedule: 6 | - cron: '0 12 * * 3' 7 | 8 | permissions: 9 | security-events: write # Used by this action. 10 | 11 | jobs: 12 | CodeQL-Build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Checkout repository 18 | uses: actions/checkout@v4 19 | with: 20 | # We must fetch at least the immediate parents so that if this is 21 | # a pull request then we can checkout the head. 22 | fetch-depth: 4 23 | persist-credentials: false 24 | 25 | # If this run was triggered by a pull request event, then checkout 26 | # the head of the pull request instead of the merge commit. 27 | - run: git checkout HEAD^2 28 | if: ${{ github.event_name == 'pull_request' }} 29 | 30 | # Initializes the CodeQL tools for scanning. 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@v3 33 | # Override language selection by uncommenting this and choosing your languages 34 | # with: 35 | # languages: go, javascript, csharp, python, cpp, java 36 | 37 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 38 | # If this step fails, then you should remove it and run the build manually (see below) 39 | - name: Autobuild 40 | uses: github/codeql-action/autobuild@v3 41 | 42 | # ℹ️ Command-line programs to run using the OS shell. 43 | # 📚 https://git.io/JvXDl 44 | 45 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 46 | # and modify them (or add more) to build your code if your project 47 | # uses a compiled language 48 | 49 | #- run: | 50 | # make bootstrap 51 | # make release 52 | 53 | - name: Perform CodeQL Analysis 54 | uses: github/codeql-action/analyze@v3 55 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: '6 15 * * SUN' 8 | 9 | permissions: {} 10 | 11 | jobs: 12 | 13 | build: 14 | strategy: 15 | matrix: 16 | go-version: [1.23.x, 1.24.x] 17 | platform: [ubuntu-latest, macos-latest, windows-latest] 18 | runs-on: ${{ matrix.platform }} 19 | name: "Build ${{ matrix.go-version }} test on ${{ matrix.platform }}" 20 | steps: 21 | - name: Set up Go 1.x 22 | uses: actions/setup-go@v5 23 | with: 24 | go-version: ${{ matrix.go-version }} 25 | id: go 26 | 27 | - name: Check out code into the Go module directory 28 | uses: actions/checkout@v4 29 | with: 30 | persist-credentials: false 31 | 32 | - name: Get dependencies 33 | run: go get -v -t -d ./... 34 | 35 | - name: Build 36 | run: go build -v ./... 37 | 38 | - name: Test 39 | run: go test -race -v ./... 40 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: '6 15 * * SUN' 8 | 9 | permissions: {} 10 | 11 | jobs: 12 | golangci: 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | with: 18 | persist-credentials: false 19 | - name: golangci-lint 20 | uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # 8.0.0 21 | with: 22 | version: latest 23 | -------------------------------------------------------------------------------- /.github/workflows/modver.yml: -------------------------------------------------------------------------------- 1 | name: modver 2 | 3 | on: 4 | pull_request: 5 | 6 | permissions: 7 | contents: read # This gets granted by default, so keep granting it. 8 | packages: read # This gets granted by default, so keep granting it. 9 | pull-requests: write # Needed to comment on the PR. 10 | 11 | jobs: 12 | modver: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | persist-credentials: false 19 | - uses: bobg/modver@0035b3b46089fc8f5ec9f3f5987e12fb618e120d # 2.11.0 20 | with: 21 | github_token: ${{ secrets.GITHUB_TOKEN }} 22 | pull_request_url: https://github.com/${{ github.repository }}/pull/${{ github.event.number }} 23 | -------------------------------------------------------------------------------- /.github/workflows/zizmor.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Actions Security Analysis with zizmor 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["**"] 8 | 9 | jobs: 10 | zizmor: 11 | name: zizmor latest via PyPI 12 | runs-on: ubuntu-latest 13 | permissions: 14 | security-events: write 15 | # required for workflows in private repositories 16 | contents: read 17 | actions: read 18 | steps: 19 | - name: Checkout repository 20 | uses: actions/checkout@v4 21 | with: 22 | persist-credentials: false 23 | 24 | - name: Install the latest version of uv 25 | uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # 6.1.0 26 | with: 27 | enable-cache: false 28 | 29 | - name: Run zizmor 30 | run: uvx zizmor@1.7.0 --format plain . 31 | env: 32 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | geoip2-csv-converter 2 | .sw? 3 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | # This is needed for precious, which may run multiple instances 4 | # in parallel 5 | go: "1.23" 6 | tests: true 7 | allow-parallel-runners: true 8 | linters: 9 | default: all 10 | disable: 11 | # The canonical form is not always the most common form for some headers 12 | # and there is a small chance that switching existing strings could 13 | # break something. 14 | - canonicalheader 15 | 16 | - cyclop 17 | 18 | # This forbids stuff like "_, _, _, err := ...". Although I agree that it 19 | # is not ideal, the actual uses in our code-base are mostly due to third- 20 | # party libraries. 21 | - dogsled 22 | 23 | # This seems to primarily go off for table-driven tests for us. I don't 24 | # know if more sharing between these tests would actually make the code 25 | # easier to follow. 26 | - dupl 27 | 28 | # This is very helpful for linting comments, but for strings in code it 29 | # is pretty questionable, e.g., we have repeated strings in GeoIP build 30 | # code for org names or in test cases. We should consider enabling if 31 | # they allow limiting it to certain cases in the future. 32 | - dupword 33 | 34 | # We don't follow its policy about not defining dynamic errors. 35 | - err113 36 | 37 | # We often don't initialize all of the struct fields. This is fine 38 | # generally 39 | - exhaustruct 40 | 41 | # We tried this linter but most places we do forced type asserts are 42 | # pretty safe, e.g., an atomic.Value when everything is encapsulated 43 | # in a small package. 44 | - forcetypeassert 45 | 46 | - funlen 47 | - gochecknoglobals 48 | - gochecknoinits 49 | 50 | # Similar to the exhaustive linter and I don't know that we use these 51 | # sorts of sum types 52 | - gochecksumtype 53 | 54 | - gocognit 55 | 56 | # We don't want to forbid TODO, FIXME, etc. 57 | - godox 58 | 59 | # This only "caught" one thing, and it seemed like a reasonable use 60 | # of Han script. Generally, I don't think we want to prevent the use 61 | # of particular scripts. The time.Local checks might be useful, but 62 | # this didn't actually catch anything of note there. 63 | - gosmopolitan 64 | 65 | # Seems too opinionated or at least would require going through all the 66 | # interfaces we have. 67 | - inamedparam 68 | 69 | # This is an ok rule generally, but we don't return that many interfaces 70 | # and where we do, we tend to have a particular reason to do so. 71 | - ireturn 72 | 73 | # We use golines instead. 74 | - lll 75 | 76 | # Maintainability Index. Seems like it could be a good idea, but a 77 | # lot of things fail and we would need to make some decisions about 78 | # what to allow. 79 | - maintidx 80 | 81 | # Using a const for every number doesn't necessarily increase code clarity, 82 | # and it would be a ton of work to move everything to that. 83 | - mnd 84 | 85 | # Causes panics, e.g., when processing mmerrors 86 | - musttag 87 | 88 | - nestif 89 | 90 | # We do end up with a lot of debates on PRs about nil, nil returns. Such 91 | # returns are often surprising, but there seem to be enough valid cases 92 | # in our codebase that I am reluctant to enforce this. 93 | - nilnil 94 | 95 | # Checks for a new line before a return. Although that often seems helpful, 96 | # there are many cases where it is not. 97 | - nlreturn 98 | 99 | # We occasionally use named returns for documentation, which is helpful. 100 | # Named returns are only really a problem when used in conjunction with 101 | # a bare return statement. I _think_ Revive's bare-return covers that 102 | # case. 103 | - nonamedreturns 104 | 105 | # This is not something we want to enforce throughout our code. 106 | - paralleltest 107 | 108 | # This seems like premature optimization for most programs. 109 | - prealloc 110 | 111 | # We have very few structs with multiple tags and for the couple we had, this 112 | # actually made it harder to read. 113 | - tagalign 114 | # We don't follow this. Sometimes we test internal code. 115 | - testpackage 116 | # We probably _should_ be doing this! 117 | - thelper 118 | - varnamelen 119 | 120 | # Although some fixes in this seem good (e.g., err cuddling), I was 121 | # not able to disable some of the less desirable whitespace changes. 122 | - wsl 123 | 124 | settings: 125 | # Please note that we only use depguard for blocking packages and 126 | # gomodguard for blocking modules. 127 | depguard: 128 | rules: 129 | main: 130 | deny: 131 | - pkg: github.com/likexian/gokit/assert 132 | desc: Use github.com/stretchr/testify/assert 133 | - pkg: golang.org/x/exp/maps 134 | desc: Use maps instead. 135 | - pkg: golang.org/x/exp/slices 136 | desc: Use slices instead. 137 | - pkg: golang.org/x/exp/slog 138 | desc: Use log/slog instead. 139 | - pkg: google.golang.org/api/compute/v1 140 | desc: Use cloud.google.com/go/compute/apiv1 instead. 141 | - pkg: google.golang.org/api/cloudresourcemanager/v1 142 | desc: Use cloud.google.com/go/resourcemanager/apiv3 instead. 143 | - pkg: google.golang.org/api/serviceusage/v1 144 | desc: Use cloud.google.com/go/serviceusage/apiv1 instead. 145 | - pkg: io/ioutil 146 | desc: Deprecated. Functions have been moved elsewhere. 147 | - pkg: k8s.io/utils/strings/slices 148 | desc: Use slices 149 | - pkg: math/rand$ 150 | desc: Use math/rand/v2 or crypto/rand as appropriate. 151 | - pkg: sort 152 | desc: Use slices instead 153 | 154 | errcheck: 155 | # Don't allow setting of error to the blank identifier. If there is a legitimate 156 | # reason, there should be a nolint with an explanation. 157 | check-blank: true 158 | exclude-functions: 159 | # If we are rolling back a transaction, we are often already in an error 160 | # state. 161 | - (*database/sql.Tx).Rollback 162 | 163 | # It is reasonable to ignore errors if Cleanup fails in most cases. 164 | - (*github.com/google/renameio/v2.PendingFile).Cleanup 165 | 166 | # We often do not care if unlocking failed as we are exiting anyway. 167 | - (*github.com/gofrs/flock.Flock).Unlock 168 | 169 | # We often don't care if removing a file failed (e.g., it doesn't exist) 170 | - os.Remove 171 | - os.RemoveAll 172 | 173 | errchkjson: 174 | report-no-exported: true 175 | 176 | errorlint: 177 | errorf: true 178 | asserts: true 179 | comparison: true 180 | 181 | exhaustive: 182 | default-signifies-exhaustive: true 183 | 184 | forbidigo: 185 | # Forbid the following identifiers 186 | forbid: 187 | - pattern: ^atomic.Value$ 188 | pkg: ^sync/atomic$ 189 | msg: Use atomic.Pointer instead. 190 | - pattern: GeoFeed 191 | msg: you should use the `Geofeed` qualifier instead 192 | - pattern: Geoip 193 | msg: you should use the `GeoIP` qualifier instead 194 | - pattern: geoIP 195 | msg: you should use the `geoip` qualifier instead 196 | - pattern: ^hubSpot 197 | msg: you should use the `hubspot` qualifier instead 198 | - pattern: Maxmind 199 | msg: you should use the `MaxMind` qualifier instead 200 | - pattern: ^maxMind 201 | msg: you should use the `maxmind` qualifier instead 202 | - pattern: Minfraud 203 | msg: you should use the `MinFraud` qualifier instead 204 | - pattern: ^minFraud 205 | msg: you should use the `minfraud` qualifier instead 206 | - pattern: "[Uu]ser[iI][dD]" 207 | msg: you should use the `accountID` or the `AccountID` qualifier instead 208 | - pattern: WithEnterpriseURLs 209 | msg: Use ghe.NewClient instead. 210 | - pattern: ^bigquery.NewClient 211 | msg: you should use mmgcloud.NewBigQueryClient instead. 212 | - pattern: ^drive.NewService 213 | msg: you should use mmgdrive.NewGDrive instead. 214 | - pattern: ^filepath.Walk$ 215 | msg: you should use filepath.WalkDir instead as it doesn't call os.Lstat on every entry. 216 | - pattern: ^math.Min$ 217 | msg: you should use the min built-in instead. 218 | - pattern: ^mux.Vars$ 219 | msg: use req.PathValue instead. 220 | - pattern: ^net.ParseCIDR 221 | msg: you should use netip.ParsePrefix unless you really need a *net.IPNet 222 | - pattern: ^net.ParseIP 223 | msg: you should use netip.ParseAddr unless you really need a net.IP 224 | - pattern: ^pgtype.NewMap 225 | msg: you should use mmdatabase.NewTypeMap instead 226 | - pattern: ^sheets.NewService 227 | msg: you should use mmgcloud.NewSheetsService instead. 228 | - pattern: ^storage.NewClient 229 | msg: you should use gstorage.NewClient instead. This sets the HTTP client settings that we need for internal use. 230 | - pattern: ^os.IsNotExist 231 | msg: As per their docs, new code should use errors.Is(err, fs.ErrNotExist). 232 | - pattern: ^os.IsExist 233 | msg: As per their docs, new code should use errors.Is(err, fs.ErrExist) 234 | - pattern: ^net.LookupIP 235 | msg: You should use net.Resolver functions instead. 236 | - pattern: ^net.LookupCNAME 237 | msg: You should use net.Resolver functions instead. 238 | - pattern: ^net.LookupHost 239 | msg: You should use net.Resolver functions instead. 240 | - pattern: ^net.LookupPort 241 | msg: You should use net.Resolver functions instead. 242 | - pattern: ^net.LookupTXT 243 | msg: You should use net.Resolver functions instead. 244 | - pattern: ^net.LookupAddr 245 | msg: You should use net.Resolver functions instead. 246 | - pattern: ^net.LookupMX 247 | msg: You should use net.Resolver functions instead. 248 | - pattern: ^net.LookupNS 249 | msg: You should use net.Resolver functions instead. 250 | - pattern: ^net.LookupSRV 251 | msg: You should use net.Resolver functions instead. 252 | 253 | gocritic: 254 | enable-all: true 255 | disabled-checks: 256 | # Revive's defer rule already captures this. This caught no extra cases. 257 | - deferInLoop 258 | 259 | # Given that all of our code runs on Linux and the / separate should 260 | # work fine, this seems less important. 261 | - filepathJoin 262 | 263 | # This might be good, but we would have to revisit a lot of code. 264 | - hugeParam 265 | 266 | # This might be good, but I don't think we want to encourage 267 | # significant changes to regexes as we port stuff from Perl. 268 | - regexpSimplify 269 | 270 | # This seems like it might also be good, but a lot of existing code 271 | # fails. 272 | - sloppyReassign 273 | 274 | # I am not sure we would want this linter and a lot of existing 275 | # code fails. 276 | - unnamedResult 277 | 278 | # Covered by nolintlint 279 | - whyNoLint 280 | 281 | gomoddirectives: 282 | replace-allow-list: 283 | # We want to use an old version due to race conditions in the latest 284 | # release. 285 | - github.com/fsnotify/fsnotify 286 | # We want to use an old version due to a logging issue. 287 | - github.com/snowflakedb/gosnowflake 288 | toolchain-forbidden: true 289 | go-version-pattern: \d\.\d+(\.0)? 290 | 291 | # IMPORTANT: gomodguard blocks _modules_, not arbitrary packages. Be 292 | # sure to use the module path from the go.mod file for these. 293 | # See https://github.com/ryancurrah/gomodguard/issues/12 294 | gomodguard: 295 | blocked: 296 | modules: 297 | - github.com/avct/uasurfer: 298 | recommendations: 299 | - github.com/xavivars/uasurfer 300 | reason: The original avct module appears abandoned. 301 | - github.com/BurntSushi/toml: 302 | recommendations: 303 | - github.com/pelletier/go-toml/v2 304 | reason: This library panics frequently on invalid input. 305 | - github.com/pelletier/go-toml: 306 | recommendations: 307 | - github.com/pelletier/go-toml/v2 308 | reason: This is an outdated version. 309 | - github.com/gofrs/uuid: 310 | recommendations: 311 | - github.com/google/uuid 312 | - github.com/gofrs/uuid/v5: 313 | recommendations: 314 | - github.com/google/uuid 315 | - github.com/satori/go.uuid: 316 | recommendations: 317 | - github.com/google/uuid 318 | - github.com/google/uuid: 319 | recommendations: 320 | - github.com/google/uuid 321 | - github.com/lib/pq: 322 | recommendations: 323 | - github.com/jackc/pgx 324 | reason: This library is no longer actively maintained. 325 | - github.com/neilotoole/errgroup: 326 | recommendations: 327 | - golang.org/x/sync/errgroup 328 | reason: This library can lead to subtle deadlocks in certain use cases. 329 | - github.com/pariz/gountries: 330 | reason: This library's data is not actively maintained. Use GeoInfo data. 331 | github.com/pkg/errors: 332 | recommendations: 333 | - errors 334 | reason: pkg/errors is no longer maintained. 335 | - github.com/RackSec/srslog: 336 | recommendations: 337 | - log/syslog 338 | reason: This library's data is not actively maintained. 339 | - github.com/ua-parser/uap-go: 340 | recommendations: 341 | - github.com/xavivars/uasurfer 342 | reason: The performance of this library is absolutely abysmal. 343 | - github.com/ugorji/go: 344 | recommendations: 345 | - encoding/json 346 | - github.com/mailru/easyjson 347 | reason: This library is poorly maintained. We should default to using encoding/json and use easyjson where performance really matters. 348 | - github.com/zeebo/assert: 349 | recommendations: 350 | - github.com/stretchr/testify/assert 351 | reason: Use github.com/stretchr/testify/assert 352 | - gopkg.in/yaml.v2: 353 | recommendations: 354 | - github.com/goccy/go-yaml 355 | reason: Not actively maintained. 356 | - gopkg.in/yaml.v3: 357 | recommendations: 358 | - github.com/goccy/go-yaml 359 | reason: Not actively maintained. 360 | - gotest.tools/v3: 361 | recommendations: 362 | - github.com/stretchr/testify/assert 363 | reason: Use github.com/stretchr/testify/assert 364 | - inet.af/netaddr: 365 | recommendations: 366 | - net/netip 367 | - go4.org/netipx 368 | reason: inet.af/netaddr has been deprecated. 369 | versions: 370 | - github.com/jackc/pgconn: 371 | reason: Use github.com/jackc/pgx/v5 372 | - github.com/jackc/pgtype: 373 | reason: Use github.com/jackc/pgx/v5 374 | - github.com/jackc/pgx: 375 | version: < 5.0.0 376 | reason: Use github.com/jackc/pgx/v5 377 | 378 | gosec: 379 | excludes: 380 | # G104 - "Audit errors not checked." We use errcheck for this. 381 | - G104 382 | 383 | # G306 - "Expect WriteFile permissions to be 0600 or less". 384 | - G306 385 | 386 | # Prohibits defer (*os.File).Close, which we allow when reading from file. 387 | - G307 388 | 389 | # We use md5 in geoipupdate 390 | - G401 391 | - G501 392 | 393 | # no longer relevant with 1.22 394 | - G601 395 | 396 | govet: 397 | disable: 398 | # Although it is very useful in particular cases where we are trying to 399 | # use as little memory as possible, there are even more cases where 400 | # other organizations may make more sense. 401 | - fieldalignment 402 | enable-all: true 403 | settings: 404 | shadow: 405 | strict: true 406 | 407 | loggercheck: 408 | # although it seems like this should remove the need for our custom 409 | # Ruleguard check, it misses things that seems to catch. However, it 410 | # is possible that this will catch things that misses as that check 411 | # is very simple. 412 | no-printf-like: true 413 | 414 | misspell: 415 | locale: US 416 | extra-words: 417 | - typo: marshall 418 | correction: marshal 419 | - typo: marshalling 420 | correction: marshaling 421 | - typo: marshalls 422 | correction: marshals 423 | - typo: unmarshall 424 | correction: unmarshal 425 | - typo: unmarshalling 426 | correction: unmarshaling 427 | - typo: unmarshalls 428 | correction: unmarshals 429 | 430 | nolintlint: 431 | require-explanation: true 432 | require-specific: true 433 | allow-no-explanation: 434 | - misspell 435 | # Setting allow-unused to false would be nice. We previously had it that 436 | # way. However there is an unresolved issue where it occasionally fails 437 | # when it should not. We think there might be some kind of caching bug in 438 | # golangci-lint. See 439 | # https://maxmind.slack.com/archives/C07ER81BQ4T/p1739389216305519. 440 | allow-unused: false 441 | 442 | revive: 443 | severity: warning 444 | enable-all-rules: true 445 | rules: 446 | # This might be nice but it is so common that it is hard 447 | # to enable. 448 | - name: add-constant 449 | disabled: true 450 | 451 | - name: argument-limit 452 | disabled: true 453 | 454 | - name: cognitive-complexity 455 | disabled: true 456 | 457 | - name: comment-spacings 458 | arguments: 459 | - easyjson 460 | - nolint 461 | disabled: false 462 | 463 | # Probably a good rule, but we have a lot of names that 464 | # only have case differences. 465 | - name: confusing-naming 466 | disabled: true 467 | 468 | - name: cyclomatic 469 | disabled: true 470 | 471 | # Although being consistent might be nice, I don't know that it 472 | # is worth the effort enabling this rule. It doesn't have an 473 | # autofix option. 474 | - name: enforce-repeated-arg-type-style 475 | arguments: 476 | - short 477 | disabled: true 478 | 479 | - name: enforce-map-style 480 | arguments: 481 | - literal 482 | disabled: false 483 | 484 | # We have very few of these as we force nil slices in most places, 485 | # but there are a couple of cases. 486 | - name: enforce-slice-style 487 | arguments: 488 | - literal 489 | disabled: false 490 | 491 | - name: file-header 492 | disabled: true 493 | 494 | # We have a lot of flag parameters. This linter probably makes 495 | # a good point, but we would need some cleanup or a lot of nolints. 496 | - name: flag-parameter 497 | disabled: true 498 | 499 | - name: function-length 500 | disabled: true 501 | 502 | - name: function-result-limit 503 | disabled: true 504 | 505 | - name: line-length-limit 506 | disabled: true 507 | 508 | - name: max-public-structs 509 | disabled: true 510 | 511 | # We frequently use nested structs, particularly in tests. 512 | - name: nested-structs 513 | disabled: true 514 | 515 | # This doesn't make sense with 1.22 loop var changes. 516 | - name: range-val-address 517 | disabled: true 518 | 519 | # This flags things that do not seem like a problem, e.g. "sixHours". 520 | - name: time-naming 521 | disabled: true 522 | 523 | # This causes a ton of failures. Many are fairly safe. It might be nice to 524 | # enable, but probably not worth the effort. 525 | - name: unchecked-type-assertion 526 | disabled: true 527 | 528 | # This seems to give many false positives. 529 | - name: unconditional-recursion 530 | disabled: true 531 | 532 | # This is covered elsewhere and we want to ignore some 533 | # functions such as fmt.Fprintf. 534 | - name: unhandled-error 535 | disabled: true 536 | 537 | # We generally have unused receivers in tests for meeting the 538 | # requirements of an interface. 539 | - name: unused-receiver 540 | disabled: true 541 | 542 | sloglint: 543 | # Enforce not mixing key-value pairs and attributes. 544 | no-mixed-args: true 545 | 546 | # Enforce not using global loggers. 547 | no-global: all 548 | 549 | # Enforce a snake case for keys. 550 | key-naming-case: snake 551 | 552 | # Make sure we don't use reserved keys. 553 | forbidden-keys: 554 | # These are included by our handler. 555 | - time 556 | - level 557 | - logged_from 558 | - message 559 | 560 | # We don't use these two, but they are slog defaults. It would 561 | # be better to avoid using them to reduce confusion and to make 562 | # it easier to potentially use the defaults in the future. 563 | - msg 564 | - source 565 | 566 | staticcheck: 567 | checks: 568 | - all 569 | 570 | # SA1019: Using a deprecated function, variable, constant or field 571 | # 572 | # This is disabled as it interacts poorly with golangci-lint's caching. 573 | # I believe https://github.com/golangci/golangci-lint-action/issues/420 574 | # is the same underlying issue. 575 | - -SA1019 576 | 577 | # SA5008: unknown JSON option "intern" - easyjson specific option. 578 | - -SA5008 579 | 580 | tagliatelle: 581 | case: 582 | rules: 583 | avro: snake 584 | bson: snake 585 | env: upperSnake 586 | envconfig: upperSnake 587 | json: snake 588 | mapstructure: snake 589 | xml: snake 590 | yaml: snake 591 | 592 | testifylint: 593 | enable-all: true 594 | 595 | unparam: 596 | check-exported: true 597 | 598 | usestdlibvars: 599 | time-layout: true 600 | 601 | usetesting: 602 | os-temp-dir: true 603 | 604 | wrapcheck: 605 | ignore-sigs: 606 | - .Errorf( 607 | - errgroup.NewMultiError( 608 | - errors.Join( 609 | - errors.New( 610 | - .Wait( 611 | - .WithStack( 612 | - .Wrap( 613 | - .Wrapf( 614 | - v5.Retry[T any]( 615 | 616 | exclusions: 617 | generated: lax 618 | rules: 619 | # This rule doesn't really make sense for tests where we don't have an open 620 | # connection and we might be passing around the response for other reasons. 621 | - linters: 622 | - bodyclose 623 | path: _test.go 624 | 625 | # There are many cases where we want to just close resources and ignore the 626 | # error (e.g., for defer f.Close on a read). errcheck removed its built-in 627 | # wildcard ignore. I tried listing all of the cases, but it was too many 628 | # and some were very specific. 629 | - linters: 630 | - errcheck 631 | source: \.Close 632 | 633 | # This refers to a minFraud field, not the MaxMind Account ID 634 | - linters: 635 | - forbidigo 636 | source: "[Aa]ccountUserID|Account\\.UserID" 637 | 638 | # we include both a source and text exclusion as the source exclusion 639 | # misses matches where forbidigo reports the error on the first line 640 | # of a chunk of a function call even though the use is on a later line. 641 | - linters: 642 | - forbidigo 643 | text: "[Aa]ccountUserID|Account\\.UserID" 644 | 645 | # For some reason the imports stuff in ruleguard doesn't work in golangci-lint. 646 | # Perhaps it has an outdated version or something 647 | - linters: 648 | - gocritic 649 | path: _test.go 650 | text: "ruleguard: Prefer the alternative Context method instead" 651 | 652 | # The nolintlint linter behaves oddly with ruleguard rules 653 | - linters: 654 | - gocritic 655 | source: // *no-ruleguard 656 | 657 | # The contextcheck linter also uses "nolint" in a slightly different way, 658 | # leading to falso positives from nolintlint. 659 | - linters: 660 | - nolintlint 661 | source: //nolint:contextcheck //.* 662 | 663 | # These are usually fine to shadow and not allowing shadowing for them can 664 | # make the code unnecessarily verbose. 665 | - linters: 666 | - govet 667 | text: 'shadow: declaration of "(ctx|err|ok)" shadows declaration' 668 | 669 | - linters: 670 | - contextcheck 671 | # With recent changes to the linter, there were a lot of failures in 672 | # the tests and it wasn't clear to me that fixing them would actually 673 | # improve the readability. 674 | - goconst 675 | - nilerr 676 | - wrapcheck 677 | path: _test.go 678 | 679 | # ST1016 - methods on the same type should have the same receiver name. 680 | # easyjson doesn't interact well with this. 681 | - linters: 682 | - staticcheck 683 | text: ST1016 684 | 685 | - linters: 686 | - wrapcheck 687 | text: github.com/maxmind/geoip2-csv-converter 688 | 689 | - linters: 690 | - wrapcheck 691 | path: _easyjson.go 692 | 693 | - linters: 694 | - gocritic 695 | text: octalLiteral 696 | source: Chmod|WriteFile 697 | 698 | paths: 699 | - _easyjson\.go$ 700 | - _easyjson_test\.go$ 701 | - _xgb2code\.go$ 702 | - _json2vector\.go$ 703 | - geoip-build/mmcsv 704 | 705 | formatters: 706 | enable: 707 | - gci 708 | - gofumpt 709 | - goimports 710 | - golines 711 | 712 | settings: 713 | gci: 714 | sections: 715 | - standard 716 | - default 717 | - prefix(github.com/maxmind/geoip2-csv-converter) 718 | 719 | gofumpt: 720 | module-path: github.com/maxmind/geoip2-csv-converter 721 | extra-rules: true 722 | 723 | goimports: 724 | local-prefixes: 725 | - github.com/maxmind/geoip2-csv-converter 726 | 727 | golines: 728 | shorten-comments: true 729 | 730 | exclusions: 731 | generated: lax 732 | paths: 733 | - _easyjson\.go$ 734 | - _easyjson_test\.go$ 735 | - _xgb2code\.go$ 736 | - _json2vector\.go$ 737 | - geoip-build/mmcsv 738 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: 'geoip2-csv-converter' 2 | version: 2 3 | 4 | builds: 5 | - id: 'geoip2-csv-converter' 6 | binary: 'geoip2-csv-converter' 7 | goos: 8 | - 'darwin' 9 | - 'linux' 10 | - 'windows' 11 | ignore: 12 | - goos: 'darwin' 13 | goarch: '386' 14 | 15 | archives: 16 | - id: 'geoip2-csv-converter' 17 | builds: 18 | - 'geoip2-csv-converter' 19 | wrap_in_directory: true 20 | format_overrides: 21 | - goos: windows 22 | format: zip 23 | files: 24 | - 'CHANGELOG.md' 25 | - 'LICENSE' 26 | - 'README.md' 27 | 28 | checksum: 29 | name_template: 'checksums.txt' 30 | 31 | snapshot: 32 | name_template: "{{ .Tag }}-next" 33 | 34 | changelog: 35 | disable: true 36 | 37 | nfpms: 38 | - id: 'geoip2-csv-converter' 39 | builds: 40 | - 'geoip2-csv-converter' 41 | vendor: 'MaxMind, Inc.' 42 | homepage: 'https://www.maxmind.com/' 43 | maintainer: 'MaxMind, Inc. ' 44 | description: 'Convert GeoIP2 and GeoLite2 CSVs to different formats.' 45 | license: 'Apache 2.0' 46 | formats: 47 | - 'deb' 48 | - 'rpm' 49 | contents: 50 | - src: CHANGELOG.md 51 | dst: /usr/share/doc/geoip2-csv-converter/CHANGELOG.md 52 | - src: LICENSE 53 | dst: /usr/share/doc/geoip2-csv-converter/LICENSE 54 | - src: README.md 55 | dst: /usr/share/doc/geoip2-csv-converter/README.md 56 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 1.4.1 (2024-08-06) 4 | 5 | * The converter now checks for errors after flushing the CSV writer. 6 | 7 | ## 1.4.0 (2023-09-18) 8 | 9 | * Use goreleaser to release. 10 | * Add arm64 architecture builds. 11 | 12 | ## 1.3.0 (2021-01-15) 13 | 14 | * Added `-include-hex-range` flag. If set, this will include the IP range 15 | in hexadecimal format. Pull request by Alexander Sinitsyn. GitHub #33. 16 | 17 | ## 1.2.0 (2020-12-03) 18 | 19 | * The output file is now synced before it is closed and the program exits. 20 | Requested by orang3-juic3. GitHub #30. 21 | * Dependencies have been updated. 22 | 23 | ## 1.1.0 (2018-12-06) 24 | 25 | * The help output is now improved on errors. 26 | 27 | ## 1.0.0 (2016-11-04) 28 | 29 | * Compiled with Go 1.7.3. This fixes issues on macOS Sierra. Closes #6. 30 | * Updated to new version of github.com/mikioh/ipaddr 31 | 32 | ## 0.0.1 (2014-12-09) 33 | 34 | * Initial release. 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GeoIP2 CSV Format Converter 2 | --------------------------- 3 | 4 | This is a simple utility for converting the MaxMind GeoIP2 and GeoLite2 CSVs 5 | to different formats for representing IP addresses such as IP ranges or 6 | integer ranges. 7 | 8 | Compiled binaries for Linux/x86_64, Windows, and macOS (darwin) can be 9 | downloaded from the GitHub releases page. 10 | 11 | Usage 12 | ===== 13 | 14 | Required: 15 | 16 | * -block-file=[FILENAME] - The name of the block CSV file to use as input. 17 | * -output-file=[FILENAME] - The file name to the output CSV 18 | 19 | In addition, at least one of these is required: 20 | 21 | * -include-cidr - Include the network in CIDR format 22 | * -include-range - Include the IP range of the network in string format 23 | * -include-integer-range - Include the IP range of the network in integer format 24 | * -include-hex-range - Include the IP range of the network in hexadecimal format 25 | 26 | Output 27 | ====== 28 | 29 | ### CIDR (-include-cidr) 30 | 31 | This will include the network in CIDR notation in the `network` column as it 32 | is in the original CSV. 33 | 34 | ### Range (-include-range) 35 | 36 | This adds `network_start_ip` and `network_last_ip` columns. These 37 | are string representations of the first and last IP address in the network. 38 | 39 | ### Integer Range (-include-integer-range) 40 | 41 | This adds `network_start_integer` and `network_last_integer` columns. These 42 | are integer representations of the first and last IP address in the network. 43 | 44 | ### Hex Range (-include-hex-range) 45 | 46 | This adds `network_start_hex` and `network_last_hex` columns. These 47 | are hexadecimal representations of the first and last IP address in the network. 48 | 49 | Copyright and License 50 | ===================== 51 | 52 | This software is Copyright (c) 2014 - 2024 by MaxMind, Inc. 53 | 54 | This is free software, licensed under the Apache License, Version 2.0. 55 | -------------------------------------------------------------------------------- /convert/convert.go: -------------------------------------------------------------------------------- 1 | // Package convert transforms a GeoIP2/GeoLite2 CSV to various formats. 2 | package convert 3 | 4 | import ( 5 | "encoding/csv" 6 | "encoding/hex" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "math/big" 11 | "net/netip" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | 16 | "go4.org/netipx" 17 | ) 18 | 19 | type ( 20 | headerFunc func([]string) []string 21 | lineFunc func(netip.Prefix, []string) []string 22 | ) 23 | 24 | // ConvertFile converts the MaxMind GeoIP2 or GeoLite2 CSV file `inputFile` to 25 | // `outputFile` file using a different representation of the network. The 26 | // representation can be specified by setting one or more of `cidr`, 27 | // `ipRange`, `intRange` or `hexRange` to true. If none of these are set to true, it will 28 | // strip off the network information. 29 | func ConvertFile( //nolint: revive // too late to change name 30 | inputFile string, 31 | outputFile string, 32 | cidr bool, 33 | ipRange bool, 34 | intRange bool, 35 | hexRange bool, 36 | ) error { 37 | outFile, err := os.Create(filepath.Clean(outputFile)) 38 | if err != nil { 39 | return fmt.Errorf("creating output file (%s): %w", outputFile, err) 40 | } 41 | 42 | inFile, err := os.Open(filepath.Clean(inputFile)) 43 | if err != nil { 44 | outFile.Close() 45 | return fmt.Errorf("opening input file (%s): %w", inputFile, err) 46 | } 47 | 48 | err = Convert(inFile, outFile, cidr, ipRange, intRange, hexRange) 49 | if err != nil { 50 | inFile.Close() 51 | outFile.Close() 52 | return err 53 | } 54 | err = outFile.Sync() 55 | if err != nil { 56 | inFile.Close() 57 | outFile.Close() 58 | return fmt.Errorf("syncing file (%s): %w", outputFile, err) 59 | } 60 | if err := inFile.Close(); err != nil { 61 | return fmt.Errorf("closing file (%s): %w", inputFile, err) 62 | } 63 | if err := outFile.Close(); err != nil { 64 | return fmt.Errorf("closing file (%s): %w", outputFile, err) 65 | } 66 | return nil 67 | } 68 | 69 | // Convert writes the MaxMind GeoIP2 or GeoLite2 CSV in the `input` io.Reader 70 | // to the Writer `output` using the network representation specified by setting 71 | // `cidr`, ipRange`, or `intRange` to true. If none of these are set to true, 72 | // it will strip off the network information. 73 | func Convert( 74 | input io.Reader, 75 | output io.Writer, 76 | cidr bool, 77 | ipRange bool, 78 | intRange bool, 79 | hexRange bool, 80 | ) error { 81 | makeHeader := func(orig []string) []string { return orig } 82 | makeLine := func(_ netip.Prefix, orig []string) []string { return orig } 83 | 84 | if hexRange { 85 | makeHeader = addHeaderFunc(makeHeader, hexRangeHeader) 86 | makeLine = addLineFunc(makeLine, hexRangeLine) 87 | } 88 | 89 | if intRange { 90 | makeHeader = addHeaderFunc(makeHeader, intRangeHeader) 91 | makeLine = addLineFunc(makeLine, intRangeLine) 92 | } 93 | 94 | if ipRange { 95 | makeHeader = addHeaderFunc(makeHeader, rangeHeader) 96 | makeLine = addLineFunc(makeLine, rangeLine) 97 | } 98 | 99 | if cidr { 100 | makeHeader = addHeaderFunc(makeHeader, cidrHeader) 101 | makeLine = addLineFunc(makeLine, cidrLine) 102 | } 103 | 104 | return convert(input, output, makeHeader, makeLine) 105 | } 106 | 107 | func addHeaderFunc(first, second headerFunc) headerFunc { 108 | return func(header []string) []string { 109 | return second(first(header)) 110 | } 111 | } 112 | 113 | func addLineFunc(first, second lineFunc) lineFunc { 114 | return func(network netip.Prefix, line []string) []string { 115 | return second(network, first(network, line)) 116 | } 117 | } 118 | 119 | func cidrHeader(orig []string) []string { 120 | return append([]string{"network"}, orig...) 121 | } 122 | 123 | func cidrLine(network netip.Prefix, orig []string) []string { 124 | return append([]string{network.String()}, orig...) 125 | } 126 | 127 | func rangeHeader(orig []string) []string { 128 | return append([]string{"network_start_ip", "network_last_ip"}, orig...) 129 | } 130 | 131 | func rangeLine(network netip.Prefix, orig []string) []string { 132 | return append( 133 | []string{network.Addr().String(), netipx.PrefixLastIP(network).String()}, 134 | orig..., 135 | ) 136 | } 137 | 138 | func intRangeHeader(orig []string) []string { 139 | return append([]string{"network_start_integer", "network_last_integer"}, orig...) 140 | } 141 | 142 | func intRangeLine(network netip.Prefix, orig []string) []string { 143 | startInt := new(big.Int) 144 | 145 | startInt.SetBytes(network.Addr().AsSlice()) 146 | 147 | endInt := new(big.Int) 148 | endInt.SetBytes(netipx.PrefixLastIP(network).AsSlice()) 149 | 150 | return append( 151 | []string{startInt.String(), endInt.String()}, 152 | orig..., 153 | ) 154 | } 155 | 156 | func hexRangeHeader(orig []string) []string { 157 | return append([]string{"network_start_hex", "network_last_hex"}, orig...) 158 | } 159 | 160 | func hexRangeLine(network netip.Prefix, orig []string) []string { 161 | return append( 162 | []string{ 163 | toHex(network.Addr()), 164 | toHex(netipx.PrefixLastIP(network)), 165 | }, 166 | orig..., 167 | ) 168 | } 169 | 170 | func toHex(ip netip.Addr) string { 171 | return strings.TrimPrefix(hex.EncodeToString(ip.AsSlice()), "0") 172 | } 173 | 174 | func convert( 175 | input io.Reader, 176 | output io.Writer, 177 | makeHeader headerFunc, 178 | makeLine lineFunc, 179 | ) error { 180 | reader := csv.NewReader(input) 181 | writer := csv.NewWriter(output) 182 | 183 | header, err := reader.Read() 184 | if err != nil { 185 | return fmt.Errorf("reading CSV header: %w", err) 186 | } 187 | 188 | newHeader := makeHeader(header[1:]) 189 | err = writer.Write(newHeader) 190 | if err != nil { 191 | return fmt.Errorf("writing CSV header: %w", err) 192 | } 193 | 194 | for { 195 | record, err := reader.Read() 196 | if errors.Is(err, io.EOF) { 197 | break 198 | } else if err != nil { 199 | return fmt.Errorf("reading CSV: %w", err) 200 | } 201 | 202 | prefix, err := netip.ParsePrefix(record[0]) 203 | if err != nil { 204 | return fmt.Errorf("parsing network (%s): %w", record[0], err) 205 | } 206 | 207 | err = writer.Write(makeLine(prefix, record[1:])) 208 | if err != nil { 209 | return fmt.Errorf("writing CSV: %w", err) 210 | } 211 | } 212 | 213 | writer.Flush() 214 | 215 | if err := writer.Error(); err != nil { 216 | return fmt.Errorf("flushing CSV: %w", err) 217 | } 218 | 219 | return nil 220 | } 221 | -------------------------------------------------------------------------------- /convert/convert_test.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/netip" 8 | "os" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestCIDR(t *testing.T) { 17 | checkHeader( 18 | t, 19 | cidrHeader, 20 | []string{"network"}, 21 | ) 22 | 23 | v4net := "1.1.1.0/24" 24 | checkLine( 25 | t, 26 | cidrLine, 27 | v4net, 28 | []string{v4net}, 29 | ) 30 | 31 | v6net := "2001:db8:85a3:42::/64" 32 | checkLine( 33 | t, 34 | cidrLine, 35 | v6net, 36 | []string{v6net}, 37 | ) 38 | } 39 | 40 | func TestRange(t *testing.T) { 41 | checkHeader( 42 | t, 43 | rangeHeader, 44 | []string{"network_start_ip", "network_last_ip"}, 45 | ) 46 | 47 | checkLine( 48 | t, 49 | rangeLine, 50 | "1.1.1.0/24", 51 | []string{"1.1.1.0", "1.1.1.255"}, 52 | ) 53 | 54 | checkLine( 55 | t, 56 | rangeLine, 57 | "2001:0db8:85a3:0042::/64", 58 | []string{"2001:db8:85a3:42::", "2001:db8:85a3:42:ffff:ffff:ffff:ffff"}, 59 | ) 60 | } 61 | 62 | func TestIntRange(t *testing.T) { 63 | checkHeader( 64 | t, 65 | intRangeHeader, 66 | []string{"network_start_integer", "network_last_integer"}, 67 | ) 68 | 69 | checkLine( 70 | t, 71 | intRangeLine, 72 | "1.1.1.0/24", 73 | []string{"16843008", "16843263"}, 74 | ) 75 | 76 | checkLine( 77 | t, 78 | intRangeLine, 79 | "2001:0db8:85a3:0042::/64", 80 | []string{ 81 | "42540766452641155289225172512357220352", 82 | "42540766452641155307671916586066771967", 83 | }, 84 | ) 85 | } 86 | 87 | func TestHexRange(t *testing.T) { 88 | checkHeader( 89 | t, 90 | hexRangeHeader, 91 | []string{"network_start_hex", "network_last_hex"}, 92 | ) 93 | 94 | checkLine( 95 | t, 96 | hexRangeLine, 97 | "1.1.1.0/24", 98 | []string{"1010100", "10101ff"}, 99 | ) 100 | 101 | checkLine( 102 | t, 103 | hexRangeLine, 104 | "2001:0db8:85a3:0042::/64", 105 | []string{ 106 | "20010db885a300420000000000000000", 107 | "20010db885a30042ffffffffffffffff", 108 | }, 109 | ) 110 | } 111 | 112 | func checkHeader( 113 | t *testing.T, 114 | makeHeader headerFunc, 115 | expected []string, 116 | ) { 117 | suffix := []string{"city", "country"} 118 | assert.Equal( 119 | t, 120 | append(expected, suffix...), 121 | makeHeader(suffix), 122 | ) 123 | } 124 | 125 | func checkLine( 126 | t *testing.T, 127 | makeLine lineFunc, 128 | network string, 129 | expected []string, 130 | ) { 131 | p, err := netip.ParsePrefix(network) 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | 136 | suffix := []string{"1", "2"} 137 | assert.Equal( 138 | t, 139 | append(expected, suffix...), 140 | makeLine(p, suffix), 141 | ) 142 | } 143 | 144 | func TestCIDROutput(t *testing.T) { 145 | checkOutput( 146 | t, 147 | "CIDR only", 148 | true, 149 | false, 150 | false, 151 | false, 152 | []any{ 153 | "network", 154 | "1.0.0.0/24", 155 | "4.69.140.16/29", 156 | "5.61.192.0/21", 157 | "2001:4220::/32", 158 | "2402:d000::/32", 159 | "2406:4000::/32", 160 | }, 161 | ) 162 | } 163 | 164 | func TestRangeOutput(t *testing.T) { 165 | checkOutput( 166 | t, 167 | "range only", 168 | false, 169 | true, 170 | false, 171 | false, 172 | []any{ 173 | "network_start_ip,network_last_ip", 174 | "1.0.0.0,1.0.0.255", 175 | "4.69.140.16,4.69.140.23", 176 | "5.61.192.0,5.61.199.255", 177 | "2001:4220::,2001:4220:ffff:ffff:ffff:ffff:ffff:ffff", 178 | "2402:d000::,2402:d000:ffff:ffff:ffff:ffff:ffff:ffff", 179 | "2406:4000::,2406:4000:ffff:ffff:ffff:ffff:ffff:ffff", 180 | }, 181 | ) 182 | } 183 | 184 | func TestIntRangeOutput(t *testing.T) { 185 | checkOutput( 186 | t, 187 | "integer range only", 188 | false, 189 | false, 190 | true, 191 | false, 192 | []any{ 193 | "network_start_integer,network_last_integer", 194 | "16777216,16777471", 195 | "71666704,71666711", 196 | "87932928,87934975", 197 | "42541829336310884227257139937291534336,42541829415539046741521477530835484671", 198 | "47866811183171600627242296191018336256,47866811262399763141506633784562286591", 199 | "47884659703622814097215369772150030336,47884659782850976611479707365693980671", 200 | }, 201 | ) 202 | } 203 | 204 | func TestHexRangeOutput(t *testing.T) { 205 | checkOutput( 206 | t, 207 | "hex range only", 208 | false, 209 | false, 210 | false, 211 | true, 212 | []any{ 213 | "network_start_hex,network_last_hex", 214 | "1000000,10000ff", 215 | "4458c10,4458c17", 216 | "53dc000,53dc7ff", 217 | "20014220000000000000000000000000,20014220ffffffffffffffffffffffff", 218 | "2402d000000000000000000000000000,2402d000ffffffffffffffffffffffff", 219 | "24064000000000000000000000000000,24064000ffffffffffffffffffffffff", 220 | }, 221 | ) 222 | } 223 | 224 | func TestAllOutput(t *testing.T) { 225 | checkOutput( 226 | t, 227 | "all output options", 228 | true, 229 | true, 230 | true, 231 | true, 232 | []any{ 233 | "network,network_start_ip,network_last_ip,network_start_integer,network_last_integer,network_start_hex,network_last_hex", 234 | "1.0.0.0/24,1.0.0.0,1.0.0.255,16777216,16777471,1000000,10000ff", 235 | "4.69.140.16/29,4.69.140.16,4.69.140.23,71666704,71666711,4458c10,4458c17", 236 | "5.61.192.0/21,5.61.192.0,5.61.199.255,87932928,87934975,53dc000,53dc7ff", 237 | "2001:4220::/32,2001:4220::,2001:4220:ffff:ffff:ffff:ffff:ffff:ffff,42541829336310884227257139937291534336,42541829415539046741521477530835484671,20014220000000000000000000000000,20014220ffffffffffffffffffffffff", 238 | "2402:d000::/32,2402:d000::,2402:d000:ffff:ffff:ffff:ffff:ffff:ffff,47866811183171600627242296191018336256,47866811262399763141506633784562286591,2402d000000000000000000000000000,2402d000ffffffffffffffffffffffff", 239 | "2406:4000::/32,2406:4000::,2406:4000:ffff:ffff:ffff:ffff:ffff:ffff,47884659703622814097215369772150030336,47884659782850976611479707365693980671,24064000000000000000000000000000,24064000ffffffffffffffffffffffff", 240 | }, 241 | ) 242 | } 243 | 244 | func checkOutput( 245 | t *testing.T, 246 | name string, 247 | cidr bool, 248 | ipRange bool, 249 | intRange bool, 250 | hexRange bool, 251 | expected []any, 252 | ) { 253 | input := `network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider 254 | 1.0.0.0/24,2077456,2077456,,0,0 255 | 4.69.140.16/29,6252001,6252001,,0,0 256 | 5.61.192.0/21,2635167,2635167,,0,0 257 | 2001:4220::/32,357994,357994,,0,0 258 | 2402:d000::/32,1227603,1227603,,0,0 259 | 2406:4000::/32,1835841,1835841,,0,0 260 | ` 261 | var outbuf bytes.Buffer 262 | 263 | err := Convert(strings.NewReader(input), &outbuf, cidr, ipRange, intRange, hexRange) 264 | if err != nil { 265 | t.Fatal(err) 266 | } 267 | 268 | // This is a regexp as Go 1.4 does not quote empty fields while earlier 269 | // versions do 270 | outTMPL := `%s,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider 271 | %s,2077456,2077456,(?:"")?,0,0 272 | %s,6252001,6252001,(?:"")?,0,0 273 | %s,2635167,2635167,(?:"")?,0,0 274 | %s,357994,357994,(?:"")?,0,0 275 | %s,1227603,1227603,(?:"")?,0,0 276 | %s,1835841,1835841,(?:"")?,0,0 277 | ` 278 | 279 | assert.Regexp( 280 | t, 281 | fmt.Sprintf(outTMPL, expected...), 282 | outbuf.String(), 283 | name, 284 | ) 285 | } 286 | 287 | func TestFileWriting(t *testing.T) { 288 | input := `network,something 289 | 1.0.0.0/24,"some more" 290 | ` 291 | 292 | expected := `network,network_start_ip,network_last_ip,network_start_integer,network_last_integer,network_start_hex,network_last_hex,something 293 | 1.0.0.0/24,1.0.0.0,1.0.0.255,16777216,16777471,1000000,10000ff,some more 294 | ` 295 | 296 | inFile, err := os.CreateTemp(t.TempDir(), "input") 297 | if err != nil { 298 | t.Fatal(err) 299 | } 300 | defer inFile.Close() 301 | 302 | outFile, err := os.CreateTemp(t.TempDir(), "output") 303 | if err != nil { 304 | t.Fatal(err) 305 | } 306 | defer outFile.Close() 307 | 308 | _, err = inFile.WriteString(input) 309 | require.NoError(t, err) 310 | 311 | err = ConvertFile(inFile.Name(), outFile.Name(), true, true, true, true) 312 | if err != nil { 313 | t.Fatal(err) 314 | } 315 | 316 | buf := bytes.NewBuffer(nil) 317 | _, err = io.Copy(buf, outFile) 318 | require.NoError(t, err) 319 | 320 | assert.Equal(t, expected, buf.String()) 321 | } 322 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/maxmind/geoip2-csv-converter 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/stretchr/testify v1.10.0 7 | go4.org/netipx v0.0.0-20230824141953-6213f710f925 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | gopkg.in/yaml.v3 v3.0.1 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 6 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 7 | go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ= 8 | go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 10 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 11 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 12 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 13 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // geoip2-csv-converter is a utility for converting the MaxMind GeoIP2 and 2 | // GeoLite2 CSVs to different formats for representing IP addresses such as IP 3 | // ranges or integer ranges. 4 | package main 5 | 6 | import ( 7 | "flag" 8 | "fmt" 9 | "os" 10 | "strings" 11 | 12 | "github.com/maxmind/geoip2-csv-converter/convert" 13 | ) 14 | 15 | func main() { 16 | input := flag.String( 17 | "block-file", 18 | "", 19 | "The path to the block CSV file to use as input (REQUIRED)", 20 | ) 21 | output := flag.String("output-file", "", "The path to the output CSV (REQUIRED)") 22 | ipRange := flag.Bool( 23 | "include-range", 24 | false, 25 | "Include the IP range of the network in string format", 26 | ) 27 | intRange := flag.Bool( 28 | "include-integer-range", 29 | false, 30 | "Include the IP range of the network in integer format", 31 | ) 32 | hexRange := flag.Bool( 33 | "include-hex-range", 34 | false, 35 | "Include the IP range of the network in hexadecimal format", 36 | ) 37 | cidr := flag.Bool("include-cidr", false, "Include the network in CIDR format") 38 | 39 | flag.Parse() 40 | 41 | var errors []string 42 | 43 | if *input == "" { 44 | errors = append(errors, "-block-file is required") 45 | } 46 | 47 | if *output == "" { 48 | errors = append(errors, "-output-file is required") 49 | } 50 | 51 | if *input != "" && *output != "" && *output == *input { 52 | errors = append( 53 | errors, 54 | "Your output file must be different than your block file(input file).", 55 | ) 56 | } 57 | 58 | if !*ipRange && !*intRange && !*cidr && !*hexRange { 59 | errors = append(errors, "-include-cidr, -include-range, -include-integer-range,"+ 60 | " or -include-hex-range is required") 61 | } 62 | 63 | args := flag.Args() 64 | if len(args) > 0 { 65 | errors = append(errors, "unknown argument(s): "+strings.Join(args, ", ")) 66 | } 67 | 68 | if len(errors) != 0 { 69 | printHelp(errors) 70 | os.Exit(1) 71 | } 72 | 73 | err := convert.ConvertFile(*input, *output, *cidr, *ipRange, *intRange, *hexRange) 74 | if err != nil { 75 | //nolint:errcheck // We are exiting and there isn't much we can do. 76 | fmt.Fprintf(flag.CommandLine.Output(), "Error: %v\n", err) 77 | os.Exit(1) 78 | } 79 | } 80 | 81 | func printHelp(errors []string) { 82 | var passedFlags []string 83 | flag.Visit(func(f *flag.Flag) { 84 | passedFlags = append(passedFlags, "-"+f.Name) 85 | }) 86 | 87 | if len(passedFlags) > 0 { 88 | errors = append(errors, "flags passed: "+strings.Join(passedFlags, ", ")) 89 | } 90 | 91 | for _, message := range errors { 92 | //nolint:errcheck // There isn't much to do if we can't print to the output. 93 | fmt.Fprintln(flag.CommandLine.Output(), message) 94 | } 95 | 96 | flag.Usage() 97 | } 98 | --------------------------------------------------------------------------------