├── .air.toml ├── .github ├── ISSUE_TEMPLATE │ └── Service_Domain.md └── images │ ├── Github-logo-black.png │ ├── Github-logo-white.png │ └── github-status.png ├── .gitignore ├── .golangci.yml ├── INSTALLING.md ├── LICENSE ├── Makefile ├── README.md ├── app.env.example ├── cmd ├── api │ └── main.go └── v6manage │ ├── cmd │ ├── campaign.go │ ├── campaign_crawl.go │ ├── campaign_import.go │ ├── campaign_list.go │ ├── changelog.go │ ├── crawl.go │ ├── disabledomain.go │ ├── dump.go │ ├── geoip.go │ ├── import.go │ ├── root.go │ └── services.go │ └── main.go ├── db ├── geoip │ ├── GeoLite2-ASN.mmdb │ └── GeoLite2-Country.mmdb ├── migrations │ ├── 01_schema.down.sql │ ├── 01_schema.up.sql │ ├── 02_data.down.sql │ └── 02_data.up.sql └── query │ ├── asn.sql │ ├── campaign.sql │ ├── changelog.sql │ ├── country.sql │ ├── domain.sql │ ├── metrics.sql │ ├── sites.sql │ └── stats.sql ├── extra ├── grafana_dashboard.json ├── logrotate │ └── whynoipv6-nginx.conf ├── nginx │ ├── ipv6.fail.conf │ └── whynoipv6.com.conf ├── systemd │ ├── whynoipv6-api.service │ ├── whynoipv6-campaign-crawler.service │ └── whynoipv6-crawler.service ├── whynoipv6-campaign-import.service └── whynoipv6-campaign-import.timer ├── go.mod ├── go.sum ├── internal ├── config │ └── config.go ├── core │ ├── asn.go │ ├── campaign.go │ ├── changelog.go │ ├── country.go │ ├── domain.go │ ├── metric.go │ ├── null.go │ ├── site.go │ └── stats.go ├── geoip │ └── geoip.go ├── logger │ └── logger.go ├── postgres │ ├── db │ │ ├── asn.sql.go │ │ ├── campaign.sql.go │ │ ├── changelog.sql.go │ │ ├── country.sql.go │ │ ├── db.go │ │ ├── domain.sql.go │ │ ├── metrics.sql.go │ │ ├── models.go │ │ ├── sites.sql.go │ │ └── stats.sql.go │ └── postgres.go ├── resolver │ └── resolver.go ├── rest │ ├── campaign.go │ ├── changelog.go │ ├── country.go │ ├── domain.go │ ├── error.go │ ├── metric.go │ └── server.go └── toolbox │ ├── healthcheck.go │ └── notify.go ├── service_domains.yml ├── sqlc.yaml.example └── tldbwriter.toml.example /.air.toml: -------------------------------------------------------------------------------- 1 | root = "." 2 | testdata_dir = "tmp" 3 | tmp_dir = "tmp" 4 | 5 | [build] 6 | bin = "./tmp/main" 7 | cmd = "go build -o ./tmp/main ./cmd/api" 8 | delay = 1000 9 | exclude_dir = ["assets", "tmp", "vendor", "testdata"] 10 | exclude_file = [] 11 | exclude_regex = ["_test.go"] 12 | exclude_unchanged = false 13 | follow_symlink = false 14 | full_bin = "" 15 | include_dir = [] 16 | include_ext = ["go", "tpl", "tmpl", "html"] 17 | kill_delay = "0s" 18 | log = "./tmp/build-errors.log" 19 | send_interrupt = false 20 | stop_on_error = true 21 | 22 | [color] 23 | app = "" 24 | build = "yellow" 25 | main = "magenta" 26 | runner = "green" 27 | watcher = "cyan" 28 | 29 | [log] 30 | time = true 31 | 32 | [misc] 33 | clean_on_exit = false 34 | 35 | [screen] 36 | clear_on_rebuild = false 37 | 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Service_Domain.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Disable Service Domain 3 | about: Remove a domain from the list at https://whynoipv6.com/ 4 | title: '❌ Disable Service Domain' 5 | labels: '' 6 | assignees: lasseh 7 | 8 | --- 9 | 10 | ## Reason 11 | 12 | Please describe the reason for excluding this domain 13 | 14 | 15 | ## Domains 16 | 17 | * domain1.com 18 | * domain2.net 19 | -------------------------------------------------------------------------------- /.github/images/Github-logo-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/.github/images/Github-logo-black.png -------------------------------------------------------------------------------- /.github/images/Github-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/.github/images/Github-logo-white.png -------------------------------------------------------------------------------- /.github/images/github-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/.github/images/github-status.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Environment files 2 | app.env 3 | *.env 4 | 5 | tldbwriter.toml 6 | sqlc.yaml 7 | cmd/api/api 8 | extra/vector/ 9 | 10 | # Build and dependency directories 11 | /build/ 12 | /dist/ 13 | /tmp/ 14 | 15 | # Logs 16 | /logs/ 17 | *.log 18 | 19 | # IDE and editor files 20 | .idea/ 21 | .vscode/ 22 | *.swp 23 | *.swo 24 | *~ 25 | .DS_Store 26 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: standard # standard/all/none/fast 4 | formatters: 5 | enable: 6 | - gofmt 7 | - goimports 8 | settings: 9 | gofmt: 10 | rewrite-rules: 11 | - pattern: 'interface{}' 12 | replacement: 'any' 13 | goimports: 14 | local-prefixes: 15 | - github.com/lasseh/whynoipv6 16 | -------------------------------------------------------------------------------- /INSTALLING.md: -------------------------------------------------------------------------------- 1 | # Using the Project 2 | 3 | This guide provides instructions on how to set up and use the project. 4 | 5 | ## Setting Up the Database 6 | 7 | 1. Create a new database and user: 8 | ``` 9 | CREATE DATABASE whynoipv6; 10 | CREATE USER whynoipv6 with encrypted password ''; 11 | GRANT ALL PRIVILEGES ON DATABASE whynoipv6 TO whynoipv6; 12 | 13 | GRANT USAGE ON SCHEMA public TO whynoipv6; 14 | GRANT CREATE ON SCHEMA public TO whynoipv6; 15 | GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO whynoipv6; 16 | 17 | CREATE EXTENSION pgcrypto; 18 | ``` 19 | 20 | # User 21 | Create a new user and install folders 22 | ```bash 23 | useradd -m -d /opt/whynoipv6/ ipv6 24 | mkdir -p /opt/whynoipv6/{whynoipv6,whynoipv6-web,whynoipv6-campaign} 25 | chown -R ipv6:ipv6 /opt/whynoipv6 26 | ```` 27 | 28 | ## Clone 29 | Clone all the repo's 30 | ```bash 31 | su - ipv6 32 | git clone https://github.com/lasseh/whynoipv6.git /opt/whynoipv6/whynoipv6/ 33 | git clone https://github.com/lasseh/whynoipv6-web.git /opt/whynoipv6/whynoipv6-web/ 34 | git clone https://github.com/lasseh/whynoipv6-campaign /opt/whynoipv6/whynoipv6-campaign/ 35 | ``` 36 | 37 | 38 | ## Install 39 | 1. Edit the env file: 40 | ```bash 41 | cp app.env.example app.env 42 | vim app.env 43 | ``` 44 | 45 | 1. Run the database migrations: 46 | ``` 47 | make migrateup 48 | ``` 49 | 50 | 2. Start Tranco List downloader 51 | Edit the tldbwriter.toml file with database details 52 | ```bash 53 | cp tldbwriter.toml.example tldbwriter.toml 54 | vim tldbwriter.toml 55 | make tldbwriter 56 | ``` 57 | ctrl-c after: time until next check: 1h0m9s 58 | 59 | 60 | ## Importing Data and Crawling Domains 61 | 62 | 1. Copy the service files 63 | ```bash 64 | cp /opt/whynoipv6/whynoipv6/extra/systemd/* /etc/systemd/system 65 | systemctl daemon-reload 66 | ``` 67 | 68 | 1. Import data: 69 | ``` 70 | /opt/whynoipv6/go/bin/v6manage import 71 | /opt/whynoipv6/go/bin/v6manage campaign import 72 | ``` 73 | 74 | 2. Start services 75 | ``` 76 | systemctl enable --now whynoipv6-api 77 | systemctl enable --now whynoipv6-crawler 78 | systemctl enable --now whynoipv6-campaign-crawler 79 | ``` 80 | 81 | # Frontend 82 | Create folder for the html 83 | ```bash 84 | mkdir /var/www/whynoipv6.com/ 85 | chown ipv6:ipv6 /var/www/whynoipv6.com/ 86 | ``` 87 | 88 | ## Updating the MaxMind Geo Database 89 | 90 | 1. Download the latest MaxMind Geo database from the following link: 91 | `https://github.com/P3TERX/GeoLite.mmdb/releases` 92 | 93 | 2. Replace the existing database file with the downloaded file to update the database. 94 | 95 | 96 | # Monitor services 97 | ```bash 98 | journalctl -o cat -fu whynoipv6-c* | ccze -A 99 | journalctl -o cat -fu whynoipv6-api | ccze -A 100 | ``` 101 | 102 | 103 | # Grafana 104 | 105 | 1. Create a read-only sql user: 106 | ```sql 107 | -- Create the read-only user 108 | CREATE USER v6stats WITH PASSWORD ''; 109 | 110 | -- Grant CONNECT access to the database 111 | GRANT CONNECT ON DATABASE whynoipv6 TO v6stats; 112 | 113 | -- Switch to the target database 114 | \c whynoipv6 115 | 116 | -- Grant USAGE on all schemas 117 | GRANT USAGE ON SCHEMA public TO v6stats; 118 | 119 | -- Grant SELECT on all existing tables and sequences in the schema 120 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO v6stats; 121 | GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO v6stats; 122 | 123 | -- Ensure the user has SELECT rights on future tables 124 | ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO v6stats; 125 | ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO v6stats; 126 | ``` 127 | 128 | 1. Add the following to the `pg_hba.conf` file: 129 | ``` 130 | host graph.domain.com whynoipv6_read 131 | ``` 132 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Load environment variables from .env if the file exists 2 | include ./app.env 3 | 4 | .PHONY: run 5 | run: ## Runs the application 6 | go run cmd/api/*.go 7 | 8 | .PHONY: build 9 | build: ## Builds the CLI application 10 | go build -o v6manage cmd/v6manage/main.go 11 | 12 | .PHONY: install 13 | install: ## Builds the CLI application 14 | go build -o $HOME/go/bin/v6manage cmd/v6manage/main.go 15 | go build -o $HOME/go/bin/v6-api cmd/api/main.go 16 | 17 | .PHONY: test 18 | test: ## Runs short tests 19 | go test ./... -short 20 | 21 | .PHONY: test-all 22 | test-all: ## Runs all tests 23 | go test ./... 24 | 25 | .PHONY: upgrade 26 | upgrade: ## Upgrades dependencies 27 | go get -u ./... 28 | go mod tidy 29 | 30 | .PHONY: lint 31 | lint: ## Runs the linter 32 | gofumpt -w . 33 | goimports -w . 34 | golines -w . 35 | golangci-lint run 36 | govulncheck ./... 37 | 38 | .PHONY: migrateup 39 | migrateup: ## Migrates up the database 40 | @command -v migrate >/dev/null 2>&1 || { \ 41 | echo >&2 "migrate command not found, installing..."; \ 42 | go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest; \ 43 | } 44 | @echo "📥 Running database migrations (up)..." 45 | migrate -path ./db/migrations -database $(DB_SOURCE) -verbose up 46 | @echo "✅ Migrations applied successfully!" 47 | 48 | .PHONY: migratedown 49 | migratedown: ## Migrates down the database 50 | migrate -path ./db/migrations -database $(DB_SOURCE) -verbose down 51 | 52 | .PHONY: pgcli 53 | pgcli: ## Launches pgcli tool 54 | pgcli $(DB_SOURCE) 55 | 56 | .PHONY: pgdump 57 | pgdump: ## Dumps the database 58 | pg_dump -d "$(DB_SOURCE)" --format plain --data-only --use-set-session-authorization --quote-all-identifiers --column-inserts --file "tmp/dump-$$(date +%Y%m%d).sql" 59 | 60 | .PHONY: tldbwriter 61 | tldbwriter: ## Runs the tldbwriter tool 62 | @command -v tldbwriter >/dev/null 2>&1 || { \ 63 | echo >&2 "tldbwriter command not found, installing..."; \ 64 | go install -v github.com/eest/tranco-list-api/cmd/tldbwriter@latest; \ 65 | } 66 | tldbwriter -config=tldbwriter.toml 67 | 68 | .PHONY: sqlc 69 | sqlc: ## Generates Go code from SQL 70 | sqlc generate 71 | 72 | .PHONY: live 73 | live: ## Live reload of the application 74 | air . 75 | 76 | ## Help display. 77 | ## Pulls comments from beside commands and prints a nicely formatted 78 | ## display with the commands and their usage information. 79 | .DEFAULT_GOAL := help 80 | 81 | help: ## Prints this help 82 | @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 5 | Shame! 6 | 7 |
8 |
9 |
10 | 11 | [![License](https://img.shields.io/github/license/lasseh/whynoipv6)](#license) 12 | [![Website](https://img.shields.io/website?url=https%3A%2F%2Fwhynoipv6.com)](https://whynoipv6.com/) 13 | [![Issues - whynoipv6](https://img.shields.io/github/issues/lasseh/whynoipv6)](https://github.com/lasseh/whynoipv6/issues) 14 | [![Github status](https://img.shields.io/badge/Github_IPv6-Missing-red?logo=github)](https://whynoipv6.com/domain/github.com/) 15 | 16 |
17 |

Shame as a Service

18 |
19 | Shaming the largest websites in the world lacking IPv6 support. 20 |
21 |
22 |

23 | Report Bug 24 | · 25 | Request Feature 26 | · 27 | Twitter 28 |

29 | 30 |
31 | 32 | 33 | 34 |
35 | 36 | ## What is WhyNoIPv6.com? 37 | WhyNoIPv6.com is a specialized platform committed to monitoring and promoting the adoption of IPv6 among the 1 Million top-ranked websites and user-submitted campaigns. We offer insightful metrics to help you assess the current landscape of IPv6 implementation. 38 | 39 | ## Why is IPv6 Important? 40 | IPv6 is not merely an upgrade; it's a fundamental pillar for the Internet's sustainable future. As we edge closer to exhausting the IPv4 address space, the immense address capacity of IPv6 becomes indispensable. Beyond the scalability, IPv6 brings along robust security protocols and superior performance, making it the linchpin for modern, efficient, and secure internet communications. 41 | 42 | Failing to adopt IPv6 is tantamount to inhibiting the Internet's evolution. For top websites, this isn't just negligence—it's an abdication of their role as industry leaders. That's why our mission at WhyNoIPv6.com is not just to monitor, but to actively push for the closing of these alarming gaps in IPv6 adoption. 43 | 44 | ## How does WhyNoIPv6.com work? 45 | At WhyNoIPv6.com, we meticulously scan each domain from Tranco's top-ranked list every 3 days to evaluate critical IPv6 adoption metrics. Specifically, we check for the existence of IPv6 DNS records and MX records. The data gleaned from these scans is then aggregated, analyzed, and made publicly available, providing a comprehensive and up-to-date snapshot of IPv6 implementation across influential websites. 46 | 47 | ## Tranco? 48 | The [Tranco List](https://tranco-list.eu/) offers an alternative way to gauge a website's standing on the internet, diverging from traditional metrics such as those provided by Alexa rankings. Unlike Alexa, which ranks websites based on a combination of average daily visitors and pageviews over a three-month period, the Tranco List employs a robust methodology that aggregates data from various sources to compile its rankings. 49 | 50 | This approach aims to provide a more comprehensive and reliable measure of a website's popularity and traffic, addressing some of the accuracy concerns associated with Alexa's data. As a result, the Tranco List is increasingly recognized as a valuable tool for understanding website prominence in a way that accounts for a broader spectrum of internet activity. 51 | 52 | ## Crawler 53 | The crawler uses 1.1.1.1 as nameserver and will check for ipv6 records on domain.com, wwww.domain.com, ns and mx records. 54 | It will check each domain every 3 days. 55 | 56 | ## Campaigns 57 | In addition to displaying the IPv6 status of the top 1 million domains, WhyNoIPv6.com also has a campaign feature that encourages users to create their own lists of domains to check and shame. This feature allows users to generate their own personalized list of domains and monitor their IPv6 adoption progress. Users can also share their lists on social media to spread awareness about the importance of IPv6 adoption and encourage more websites to adopt IPv6. By empowering users to create their own lists, WhyNoIPv6.com aims to create a community-driven effort to promote IPv6 adoption and help build a more resilient and future-proof Internet. 58 | To create a campaign, create a new issue here: https://github.com/lasseh/whynoipv6-campaign 59 | 60 | ## Repositories 61 | The complete project consists of 3 repo's, check them out here: 62 | [WhyNoIPv6 Backend](https://github.com/lasseh/whynoipv6) 63 | [WhyNoIPv6 Frontend](https://github.com/lasseh/whynoipv6-web) 64 | [WhyNoIPv6 Campaigns](https://github.com/lasseh/whynoipv6-campaign) 65 | 66 | ## Contributors 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /app.env.example: -------------------------------------------------------------------------------- 1 | DB_SOURCE=postgresql://username:password@localhost:5432/database?sslmode=disable 2 | API_PORT=9001 3 | GEOIP_PATH="db/geoip/" 4 | CAMPAIGN_PATH="$GOPATH/src/github.com/lasseh/whynoipv6-campaign/" 5 | IRC_TOKEN="" 6 | HEALTHCHECK_CRAWLER="" 7 | HEALTHCHECK_CAMPAIGN="" 8 | NAMESERVER="nameserver 1.1.1.1" 9 | -------------------------------------------------------------------------------- /cmd/api/main.go: -------------------------------------------------------------------------------- 1 | // Package main contains the entry point for the whynoipv6 API server. 2 | package main 3 | 4 | import ( 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "time" 9 | 10 | "whynoipv6/internal/config" 11 | "whynoipv6/internal/core" 12 | "whynoipv6/internal/postgres" 13 | "whynoipv6/internal/rest" 14 | ) 15 | 16 | func main() { 17 | log.Println("Starting api server") 18 | 19 | // Read the application configuration. 20 | cfg, err := config.Read() 21 | if err != nil { 22 | log.Fatalf("Failed to read config: %v", err) 23 | } 24 | 25 | // Connect to the database 26 | const maxRetries = 5 27 | const timeout = 10 * time.Second 28 | dbSource := cfg.DatabaseSource + "&application_name=api" 29 | db, err := postgres.NewPostgreSQL(dbSource, maxRetries, timeout) 30 | if err != nil { 31 | log.Fatalln("Error connecting to database", err) 32 | } 33 | defer db.Close() 34 | 35 | // Initialize the router for handling HTTP requests. 36 | router, err := rest.NewRouter() 37 | if err != nil { 38 | log.Fatalf("Failed to create router: %v", err) 39 | } 40 | 41 | // Initialize core services for managing various resources. 42 | changelogService := core.NewChangelogService(db) 43 | domainService := core.NewDomainService(db) 44 | countryService := core.NewCountryService(db) 45 | campaignService := core.NewCampaignService(db) 46 | metricService := core.NewMetricService(db) 47 | 48 | // Message for the / endpoint. 49 | router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 50 | w.Header().Set("Content-Type", "application/json") 51 | w.WriteHeader(http.StatusOK) 52 | _, _ = w.Write([]byte(`{"message": "ok"}`)) 53 | }) 54 | 55 | // Register API endpoints with their respective handlers. 56 | router.Mount("/domain", rest.DomainHandler{Repo: domainService}.Routes()) 57 | router.Mount("/country", rest.CountryHandler{Repo: countryService}.Routes()) 58 | router.Mount("/changelog", rest.ChangelogHandler{Repo: changelogService}.Routes()) 59 | router.Mount("/campaign", rest.CampaignHandler{Repo: campaignService}.Routes()) 60 | router.Mount("/metric", rest.MetricHandler{Repo: metricService}.Routes()) 61 | 62 | // Print the registered routes for debugging purposes. 63 | rest.PrintRoutes(router) 64 | 65 | // Start the API server with the configured listening address. 66 | listenAddr := fmt.Sprintf("[::1]:%v", cfg.APIPort) 67 | log.Printf("Starting server on %s", listenAddr) 68 | log.Fatal(http.ListenAndServe(listenAddr, router)) 69 | } 70 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/campaign.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | // campaignCmd represents the campaign command 10 | var campaignCmd = &cobra.Command{ 11 | Use: "campaign", 12 | Short: "Manage campaigns", 13 | Long: "Manage campaigns", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | if len(args) == 0 { 16 | if err := cmd.Help(); err != nil { 17 | os.Exit(1) 18 | } 19 | os.Exit(0) 20 | } 21 | }, 22 | } 23 | 24 | func init() { 25 | rootCmd.AddCommand(campaignCmd) 26 | } 27 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/campaign_import.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "whynoipv6/internal/core" 12 | "whynoipv6/internal/resolver" 13 | 14 | "github.com/google/uuid" 15 | "github.com/spf13/cobra" 16 | "gopkg.in/yaml.v3" 17 | ) 18 | 19 | // campaignImportCmd represents the command for importing domains to a campaign 20 | var campaignImportCmd = &cobra.Command{ 21 | Use: "import", 22 | Short: "Import list of domains to a campaign", 23 | Long: "Import list of domains to a campaign from a file", 24 | Run: func(cmd *cobra.Command, args []string) { 25 | campaignService = *core.NewCampaignService(db) 26 | importDomainsToCampaign() 27 | }, 28 | } 29 | 30 | func init() { 31 | campaignCmd.AddCommand(campaignImportCmd) 32 | } 33 | 34 | // CampaignYAML represents a campaign in YAML format 35 | type CampaignYAML struct { 36 | Title string `yaml:"title"` 37 | Description string `yaml:"description"` 38 | UUID string `yaml:"uuid"` 39 | DomainNames []string `yaml:"domains"` 40 | } 41 | 42 | // importDomainsToCampaign imports domains from a file to the specified campaign. 43 | // It reads and processes the YAML files in the campaign folder and imports the domains into the campaign. 44 | // The function also handles updating the campaign's title and description if needed, 45 | // as well as adding and removing domains from the campaign. 46 | func importDomainsToCampaign() { 47 | ctx := context.Background() 48 | log.Println("Starting Campaign Import from", cfg.CampaignPath) 49 | 50 | // Read all the files in the campaign folder 51 | err = filepath.Walk( 52 | cfg.CampaignPath, 53 | func(campaignFile string, info os.FileInfo, err error) error { 54 | if err != nil { 55 | return err 56 | } 57 | 58 | // Check if the file is a YAML file 59 | if !info.IsDir() && strings.EqualFold(filepath.Ext(campaignFile), ".yml") { 60 | // Unmarshal YAML data from the file 61 | yamlData, err := unmarshalYAMLFile(campaignFile) 62 | if err != nil { 63 | log.Println(err) 64 | return nil 65 | } 66 | 67 | // Create or update campaign 68 | err = createOrUpdateCampaign(ctx, campaignFile, yamlData) 69 | if err != nil { 70 | log.Println(err) 71 | return nil 72 | } 73 | 74 | // Sync domains with the database 75 | err = syncDomainsWithDatabase(ctx, yamlData) 76 | if err != nil { 77 | log.Println(err) 78 | return nil 79 | } 80 | } 81 | return nil 82 | }, 83 | ) 84 | if err != nil { 85 | fmt.Printf("Error walking through directory %s: %v\n", cfg.CampaignPath, err) 86 | } 87 | } 88 | 89 | // unmarshalYAMLFile reads and unmarshals the YAML data from the given file and returns the unmarshalled data as CampaignYAML. 90 | func unmarshalYAMLFile(campaignFile string) (*CampaignYAML, error) { 91 | data, err := os.ReadFile(campaignFile) 92 | if err != nil { 93 | return nil, fmt.Errorf("error reading file %s: %v", campaignFile, err) 94 | } 95 | fmt.Println("Processing file:", filepath.Base(campaignFile)) 96 | 97 | // Unmarshal the YAML data 98 | var yamlData CampaignYAML 99 | err = yaml.Unmarshal(data, &yamlData) 100 | if err != nil { 101 | return nil, fmt.Errorf("error unmarshalling YAML data from %s: %v", campaignFile, err) 102 | } 103 | 104 | return &yamlData, nil 105 | } 106 | 107 | // createOrUpdateCampaign creates a new campaign if the UUID is empty, otherwise it updates the campaign. 108 | func createOrUpdateCampaign( 109 | ctx context.Context, 110 | campaignFile string, 111 | yamlData *CampaignYAML, 112 | ) error { 113 | // Validate Campaign title and description 114 | if yamlData.Title == "" || yamlData.Description == "" { 115 | return fmt.Errorf("error: Campaign title or description is empty in %s", campaignFile) 116 | } 117 | 118 | // Check if the UUID is empty, if so, create a new campaign and update the YAML file 119 | if yamlData.UUID == "" { 120 | newUUID, err := updateCampaignUUID(campaignFile, yamlData) 121 | if err != nil { 122 | return fmt.Errorf("error updating campaign UUID: %v", err) 123 | } 124 | yamlData.UUID = newUUID 125 | } 126 | 127 | newCampaign := core.CampaignModel{ 128 | UUID: uuid.MustParse(yamlData.UUID), 129 | Name: yamlData.Title, 130 | Description: yamlData.Description, 131 | } 132 | 133 | // Update the campaign 134 | _, err := campaignService.CreateOrUpdateCampaign(ctx, newCampaign) 135 | if err != nil { 136 | return fmt.Errorf("error creating or updating campaign: %v", err) 137 | } 138 | 139 | // log.Println("Campaign created or updated:", campaign.UUID) 140 | return nil 141 | } 142 | 143 | // updateCampaignUUID updates the campaign's UUID in the given YAML file and stores the updated content. 144 | // It creates a new campaign using campaignService, extracts the UUID and updates the YAML file with the new UUID. 145 | // And returns the new UUID. 146 | func updateCampaignUUID(yamlFilePath string, campaignData *CampaignYAML) (string, error) { 147 | ctx := context.Background() 148 | 149 | // Create a new campaign using the provided YAML data. 150 | newCampaign, err := campaignService.CreateCampaign( 151 | ctx, 152 | campaignData.Title, 153 | campaignData.Description, 154 | ) 155 | if err != nil { 156 | return "", fmt.Errorf("failed to create campaign: %w", err) 157 | } 158 | fmt.Printf("Created new campaign with UUID %s\n", newCampaign.UUID) 159 | 160 | // Update the campaign data with the new UUID. 161 | campaignData.UUID = newCampaign.UUID.String() 162 | 163 | // Marshal the updated campaign data back into YAML format. 164 | updatedYAMLData, err := yaml.Marshal(campaignData) 165 | if err != nil { 166 | return "", fmt.Errorf("failed to marshal updated campaign data: %w", err) 167 | } 168 | 169 | // Write the updated YAML data to the file. 170 | err = os.WriteFile(yamlFilePath, updatedYAMLData, 0o644) 171 | if err != nil { 172 | return "", fmt.Errorf("failed to write updated YAML data to file: %w", err) 173 | } 174 | 175 | fmt.Println("Updated YAML file with new UUID") 176 | 177 | return newCampaign.UUID.String(), nil 178 | } 179 | 180 | // syncDomainsWithDatabase syncs domains from the yamlData with the database, inserting new domains and 181 | // removing the ones that are not present in the yamlData anymore. 182 | func syncDomainsWithDatabase(ctx context.Context, yamlData *CampaignYAML) error { 183 | campaignUUID := uuid.MustParse(yamlData.UUID) 184 | 185 | // Loop over domains 186 | for _, domain := range yamlData.DomainNames { 187 | // Validate domain 188 | // Ignore rcode here. Manually disable/remove domains from campaigns if they are not valid. 189 | _, err := resolver.ValidateDomain(domain) 190 | if err != nil { 191 | log.Printf("error validating domain %s: %v", domain, err.Error()) 192 | continue 193 | } 194 | 195 | // Insert domain into campaign 196 | if err := campaignService.InsertCampaignDomain(ctx, campaignUUID, domain); err != nil { 197 | log.Printf("error inserting domain: %v", err) 198 | continue 199 | } 200 | } 201 | 202 | // Check if domains in the database are in the YAML file, if not, remove them from the campaign table 203 | domains, err := campaignService.ListCampaignDomain(ctx, campaignUUID, 0, 10000) 204 | if err != nil { 205 | return fmt.Errorf("error listing domains for campaign with UUID %s: %v", yamlData.UUID, err) 206 | } 207 | 208 | // Use a map to store the domains from yamlData.DomainNames for faster lookup. 209 | domainMap := make(map[string]bool) 210 | for _, domain := range yamlData.DomainNames { 211 | domainMap[domain] = true 212 | } 213 | 214 | // Iterate over all domains in the database. 215 | for _, domainRecord := range domains { 216 | // Check if the domain is not present in the domainMap. 217 | if !domainMap[domainRecord.Site] { 218 | fmt.Printf("Removing domain %s from campaign '%s'\n", domainRecord.Site, yamlData.Title) 219 | // Delete the domain from the campaign. 220 | err := campaignService.DeleteCampaignDomain(ctx, campaignUUID, domainRecord.Site) 221 | if err != nil { 222 | return fmt.Errorf( 223 | "error deleting domain %s from campaign with UUID %s: %v", 224 | domainRecord.Site, 225 | yamlData.UUID, 226 | err, 227 | ) 228 | } 229 | } 230 | } 231 | 232 | return nil 233 | } 234 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/campaign_list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "whynoipv6/internal/core" 8 | 9 | "github.com/alexeyco/simpletable" 10 | 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | // listCmd represents the list command 15 | var listCmd = &cobra.Command{ 16 | Use: "list", 17 | Short: "Lists all campaigns", 18 | Long: `Lists all campaigns`, 19 | Run: func(cmd *cobra.Command, args []string) { 20 | campaignService = *core.NewCampaignService(db) 21 | listCampaign() 22 | }, 23 | } 24 | 25 | func init() { 26 | campaignCmd.AddCommand(listCmd) 27 | } 28 | 29 | // listCampaign displays a table of campaigns with their UUID, name, and domain count. 30 | func listCampaign() { 31 | ctx := context.Background() 32 | 33 | // Create a new table with simpletable package. 34 | table := simpletable.New() 35 | 36 | // Fetch campaigns using the campaignService. 37 | campaigns, err := campaignService.ListCampaign(ctx) 38 | if err != nil { 39 | fmt.Printf("Error fetching campaigns: %v\n", err) 40 | return 41 | } 42 | 43 | // Set table header. 44 | table.Header = &simpletable.Header{ 45 | Cells: []*simpletable.Cell{ 46 | {Align: simpletable.AlignCenter, Text: "UUID"}, 47 | {Align: simpletable.AlignCenter, Text: "Name"}, 48 | {Align: simpletable.AlignCenter, Text: "Domain Count"}, 49 | }, 50 | } 51 | 52 | // Initialize a variable to store the total domain count. 53 | totalDomainCount := 0 54 | 55 | // Iterate through the campaigns and add rows to the table. 56 | for _, campaign := range campaigns { 57 | row := []*simpletable.Cell{ 58 | {Text: campaign.UUID.String()}, 59 | {Text: campaign.Name}, 60 | {Align: simpletable.AlignRight, Text: fmt.Sprintf("%d", campaign.Count)}, 61 | } 62 | 63 | table.Body.Cells = append(table.Body.Cells, row) 64 | 65 | totalDomainCount += int(campaign.Count) 66 | } 67 | 68 | // Set table footer with the total domain count. 69 | table.Footer = &simpletable.Footer{ 70 | Cells: []*simpletable.Cell{ 71 | {}, 72 | {Align: simpletable.AlignRight, Text: "Total"}, 73 | {Align: simpletable.AlignRight, Text: fmt.Sprintf("%d", totalDomainCount)}, 74 | }, 75 | } 76 | 77 | // Set table style and print it. 78 | table.SetStyle(simpletable.StyleDefault) 79 | fmt.Println(table.String()) 80 | } 81 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/changelog.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | "whynoipv6/internal/core" 9 | 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | // changelogCmd represents the command for displaying the last changelogs 14 | var changelogCmd = &cobra.Command{ 15 | Use: "changelog", 16 | Short: "Displays the last changelogs", 17 | Long: "Displays the last changelogs", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | displayChangelogs() 20 | }, 21 | } 22 | 23 | func init() { 24 | rootCmd.AddCommand(changelogCmd) 25 | } 26 | 27 | // displayChangelogs retrieves and prints the last 50 changelog entries 28 | func displayChangelogs() { 29 | ctx := context.Background() 30 | 31 | // Instantiate the changelog service 32 | changelogService := core.NewChangelogService(db) 33 | 34 | // Retrieve the last 50 changelog entries 35 | changelogEntries, err := changelogService.List(ctx, 50, 0) 36 | if err != nil { 37 | log.Fatal(err.Error()) 38 | } 39 | 40 | // Print the retrieved changelog entries 41 | for _, entry := range changelogEntries { 42 | fmt.Printf( 43 | "[%s] %s - %s\n", 44 | entry.Ts.Format("2006-01-02 15:04:05"), 45 | entry.Site, 46 | entry.Message, 47 | ) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/disabledomain.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | 10 | "whynoipv6/internal/core" 11 | 12 | "github.com/spf13/cobra" 13 | "gopkg.in/yaml.v3" 14 | ) 15 | 16 | // disableCmd represents the command for disabling domains 17 | var disableCmd = &cobra.Command{ 18 | Use: "disable", 19 | Short: "Disables service domains", 20 | Long: "Disables service domains from a file", 21 | Run: func(cmd *cobra.Command, args []string) { 22 | domainService = *core.NewDomainService(db) 23 | disableDomains() 24 | }, 25 | } 26 | 27 | func init() { 28 | rootCmd.AddCommand(disableCmd) 29 | } 30 | 31 | // ServiceDomainYAML represents a service domain in YAML format 32 | type ServiceDomainYAML struct { 33 | Domains []string `yaml:"domains"` 34 | } 35 | 36 | // disableDomains disables domains from a file. 37 | // It reads and processes the YAML files in the campaign folder and disables the domains. 38 | func disableDomains() { 39 | ctx := context.Background() 40 | log.Println("Banning service domains") 41 | 42 | // Read and unmarshal the YAML file 43 | yamlData, err := unmarshalDomainFile("service_domains.yml") 44 | if err != nil { 45 | log.Println(err) 46 | return 47 | } 48 | 49 | // Disable the domains 50 | for _, domain := range yamlData.Domains { 51 | log.Println("Disabling domain:", domain) 52 | err := domainService.DisableDomain(ctx, domain) 53 | if err != nil { 54 | log.Println("Error disabling domain:", domain, err) 55 | } 56 | } 57 | } 58 | 59 | // unmarshalYAMLFile reads and unmarshals the YAML data from the given file and returns the unmarshalled data as CampaignYAML. 60 | func unmarshalDomainFile(serviceFile string) (*ServiceDomainYAML, error) { 61 | data, err := os.ReadFile(serviceFile) 62 | if err != nil { 63 | return nil, fmt.Errorf("error reading file %s: %v", serviceFile, err) 64 | } 65 | log.Println("Processing file:", filepath.Base(serviceFile)) 66 | 67 | // Unmarshal the YAML data 68 | var yamlData ServiceDomainYAML 69 | err = yaml.Unmarshal(data, &yamlData) 70 | if err != nil { 71 | return nil, fmt.Errorf("error unmarshalling YAML data from %s: %v", serviceFile, err) 72 | } 73 | 74 | return &yamlData, nil 75 | } 76 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/dump.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "whynoipv6/internal/core" 10 | 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | // crawlCmd represents the crawl command 15 | var dumpCmd = &cobra.Command{ 16 | Use: "dump", 17 | Short: "Dumps all domains per country to a file", 18 | Long: "Dump all domains per country to a file", 19 | Run: func(cmd *cobra.Command, args []string) { 20 | dumpData() 21 | }, 22 | } 23 | 24 | func init() { 25 | rootCmd.AddCommand(dumpCmd) 26 | } 27 | 28 | func dumpData() { 29 | ctx := context.Background() 30 | 31 | // Spawn core service 32 | // domainService := core.NewDomainService(db) 33 | countryService := core.NewCountryService(db) 34 | 35 | // start time 36 | t := time.Now() 37 | log.Println("Dumping country data...") 38 | 39 | // Get all countries 40 | country, err := countryService.List(ctx) 41 | if err != nil { 42 | log.Fatal(err.Error()) 43 | } 44 | 45 | // Loop through countries 46 | for _, c := range country { 47 | fmt.Println("Country: ", c.Country) 48 | 49 | // Get all domains for country 50 | // domains, err := countryService.GetCountryCode()(ctx, c.Country) 51 | // if err != nil { 52 | // log.Fatal(err.Error()) 53 | // } 54 | 55 | // // Loop through domains 56 | // for _, d := range domains { 57 | // fmt.Println(d.Domain) 58 | // } 59 | } 60 | // Print time it took to import 61 | fmt.Printf("Dumped country data in: %s", time.Since(t)) 62 | } 63 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/geoip.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "whynoipv6/internal/core" 8 | "whynoipv6/internal/geoip" 9 | "whynoipv6/internal/resolver" 10 | 11 | "github.com/jackc/pgx/v4" 12 | ) 13 | 14 | // getNetworkProvider retrieves the network provider for a given domain 15 | func getNetworkProvider(ctx context.Context, domain string) (int64, error) { 16 | logg := logg.With().Str("service", "getNetworkProvider").Logger() 17 | // Get the domain's IP addresses. 18 | ip, err := resolver.IPLookup(domain) 19 | if err != nil { 20 | logg.Debug().Msgf("[%s] GeoLookup Error: %s", domain, err) 21 | } 22 | // logg.Debug().Msgf("[%s] IP: %s", domain, ip) 23 | 24 | // Get the domain's ASN. 25 | asn, err := geoip.AsnLookup(ip) 26 | if err != nil { 27 | logg.Debug().Msgf("[%s] AsnLookup Error: %s", domain, err) 28 | } 29 | logg.Debug().Msgf("[%s] ASN: '%s' (AS%d)", domain, asn.Name, asn.Number) 30 | 31 | // If a valid ASN number is found, check if it exists in the database. 32 | if asn.Number != 0 { 33 | dbAsn, err := asnService.GetASByNumber(ctx, asn.Number) 34 | if err != nil && err != pgx.ErrNoRows { 35 | logg.Debug().Msgf("[%s] GetASByNumber Error: %s\n", domain, err) 36 | } 37 | 38 | // If the ASN is not present in the database, create a new entry. 39 | if err == pgx.ErrNoRows { 40 | if verbose { 41 | logg.Debug().Msgf("[%s] AS%d does not exist in database", domain, asn.Number) 42 | } 43 | 44 | newAsn, err := asnService.CreateAsn(ctx, asn.Number, asn.Name) 45 | if err != nil { 46 | logg.Debug().Msgf("[%s] CreateAsn Error (AS: %s): %s\n", domain, asn.Name, err) 47 | return 1, nil 48 | } 49 | logg.Debug().Msgf("[%s] Added %s(AS%d) to database", domain, newAsn.Name, newAsn.Number) 50 | dbAsn = newAsn 51 | } 52 | 53 | // Return ASN ID for the found or created ASN. 54 | return dbAsn.ID, nil 55 | } 56 | 57 | // If no ASN is found, return a default ASN ID (1 - Unknown) as a fallback. 58 | return 1, nil 59 | } 60 | 61 | func getCountryID(ctx context.Context, domain string) (int64, error) { 62 | logg := logg.With().Str("service", "getCountryID").Logger() 63 | // Extract the TLD from the domain. 64 | tld, err := geoip.ExtractTLDFromDomain(domain) 65 | if err != nil { 66 | logg.Debug().Msgf("[%s] ExtractTLDFromDomain Error: %s\n", domain, err) 67 | return 251, nil // See the fallback explanation below. 68 | } 69 | 70 | // If the TLD is empty, return a default country ID (251 - Unknown). 71 | if tld == "" { 72 | logg.Debug().Msgf("[%s] TLD is empty: %s\n", domain, tld) 73 | return 251, nil // See the fallback explanation below. 74 | } 75 | 76 | // Check if the TLD is country-bound in the database. 77 | // Ignore if no mapping is found. 78 | dbTld, err := countryService.GetCountryTld(ctx, fmt.Sprintf(".%s", tld)) 79 | if err != nil && err != pgx.ErrNoRows { 80 | logg.Debug().Msgf("[%s] GetCountryTld Error: %s\n", domain, err) 81 | } 82 | 83 | // Return the country ID if a mapping is found in the database. 84 | if dbTld != (core.CountryModel{}) { 85 | logg.Debug().Msgf("[%s] Domain is TLD-bound to: %s", domain, dbTld.CountryTld) 86 | return dbTld.ID, nil 87 | } 88 | 89 | // If no TLD mapping is found, check the Geo Database for the country code. 90 | 91 | // Get the domains IP. 92 | ip, err := resolver.IPLookup(domain) 93 | if err != nil { 94 | logg.Debug().Msgf("[%s] IPLookup Error: %s", domain, err) 95 | } 96 | geoCountryCode, err := geoip.CountryLookup(ip) 97 | if err != nil { 98 | logg.Debug().Msgf("[%s] CountryLookup Error: %s", domain, err) 99 | } 100 | logg.Debug().Msgf("[%s] GEO CountryCode is %s", domain, geoCountryCode) 101 | 102 | // Check if the Geo Country Code exists in the database. 103 | dbTld, err = countryService.GetCountryTld(ctx, fmt.Sprintf(".%s", geoCountryCode)) 104 | if err != nil && err != pgx.ErrNoRows { 105 | logg.Debug().Msgf("[%s] GetCountryTld Error: %s\n", domain, err) 106 | } 107 | 108 | // Return the country ID if a mapping is found in the database. 109 | if dbTld != (core.CountryModel{}) { 110 | logg.Debug().Msgf("[%s] Domain is GEO mapped to %s", domain, dbTld.Country) 111 | return dbTld.ID, nil 112 | } 113 | 114 | // Fallback: if no country code is found, return a default country ID (251 - Unknown). 115 | return 251, nil 116 | } 117 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/import.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "whynoipv6/internal/core" 9 | 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | // crawlCmd represents the crawl command 14 | var importCmd = &cobra.Command{ 15 | Use: "import", 16 | Short: "Imports data from sites to scan table", 17 | Long: "Imports data from sites to scan table", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | importData() 20 | }, 21 | } 22 | 23 | func init() { 24 | rootCmd.AddCommand(importCmd) 25 | } 26 | 27 | func importData() { 28 | ctx := context.Background() 29 | 30 | // Spawn core service 31 | siteService := core.NewSiteService(db) 32 | domainService := core.NewDomainService(db) 33 | 34 | // start time 35 | t := time.Now() 36 | log.Println("Importing data...") 37 | 38 | var offset int64 = 0 39 | var limit int64 = 1000 40 | // Main loop 41 | for { 42 | sites, err := siteService.ListSite(ctx, offset, limit) 43 | if err != nil { 44 | log.Fatal(err.Error()) 45 | } 46 | 47 | // Stop if no more data 48 | if len(sites) == 0 { 49 | log.Println("No sites left to import") 50 | break 51 | } 52 | 53 | // Loop through sites 54 | for _, s := range sites { 55 | err := domainService.InsertDomain(ctx, s.Site) 56 | if err != nil { 57 | log.Println(err) 58 | } 59 | } 60 | // Increment offset 61 | offset += limit 62 | 63 | // Print current progress if verbose 64 | if verbose { 65 | log.Println("Imported", offset, "sites") 66 | } 67 | } 68 | 69 | // Print time it took to import 70 | log.Println("Imported sites in", time.Since(t)) 71 | 72 | // Space out domain timestamps 73 | log.Println("Spacing out domain timestamps...") 74 | err := domainService.InitSpaceTimestamps(ctx) 75 | if err != nil { 76 | log.Println("Failed to space out domain timestamps: ", err) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "whynoipv6/internal/config" 9 | "whynoipv6/internal/postgres" 10 | 11 | cc "github.com/ivanpirog/coloredcobra" 12 | 13 | "github.com/jackc/pgx/v4/pgxpool" 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | var ( 18 | cfg *config.Config 19 | db *pgxpool.Pool 20 | verbose bool 21 | err error 22 | ) 23 | 24 | // rootCmd represents the base command when called without any subcommands 25 | var rootCmd = &cobra.Command{ 26 | Use: "v6manage", 27 | Short: "IPv6 Magic!", 28 | Long: `Does all the magic behind https://whynoipv6.com`, 29 | } 30 | 31 | // Execute adds all child commands to the root command and sets flags appropriately. 32 | // This is called by main.main(). It only needs to happen once to the rootCmd. 33 | func Execute() { 34 | cc.Init(&cc.Config{ 35 | RootCmd: rootCmd, 36 | Headings: cc.HiCyan + cc.Bold + cc.Underline, 37 | Commands: cc.HiRed + cc.Bold, 38 | CmdShortDescr: cc.HiCyan, 39 | Example: cc.Italic, 40 | ExecName: cc.Bold, 41 | Flags: cc.Red, 42 | FlagsDescr: cc.HiCyan, 43 | NoExtraNewlines: true, 44 | NoBottomNewline: true, 45 | }) 46 | 47 | rootCmd.CompletionOptions.DisableDefaultCmd = true 48 | err := rootCmd.Execute() 49 | if err != nil { 50 | os.Exit(1) 51 | } 52 | } 53 | 54 | func init() { 55 | rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose output") 56 | 57 | // Read config 58 | cfg, err = config.Read() 59 | if err != nil { 60 | log.Fatal(err.Error()) 61 | } 62 | 63 | // Connect to the database 64 | const maxRetries = 5 65 | const timeout = 10 * time.Second 66 | dbSource := cfg.DatabaseSource + "&application_name=v6manage" 67 | db, err = postgres.NewPostgreSQL(dbSource, maxRetries, timeout) 68 | if err != nil { 69 | log.Fatalln("Error connecting to database:", err) 70 | } 71 | // defer db.Close() 72 | } 73 | -------------------------------------------------------------------------------- /cmd/v6manage/cmd/services.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "whynoipv6/internal/core" 8 | "whynoipv6/internal/logger" 9 | ) 10 | 11 | var ( 12 | // Global services 13 | campaignService core.CampaignService 14 | changelogService core.ChangelogService 15 | domainService core.DomainService 16 | countryService core.CountryService 17 | asnService core.ASNService 18 | metricService core.MetricService 19 | logg = logger.GetLogger() // Global logger 20 | // toolboxService toolbox.Service 21 | // statService core.StatService 22 | // resolver *toolbox.Resolver 23 | ) 24 | 25 | // prettyDuration converts a time.Duration value into a human-readable format 26 | // by rounding it to the nearest second and formatting it as "HH:mm:ss". 27 | // Sorry i dont know where to put this :( 28 | func prettyDuration(d time.Duration) string { 29 | // Round the duration to the nearest second to avoid fractional seconds. 30 | d = d.Round(time.Second) 31 | 32 | // Extract the number of hours, and subtract them from the total duration. 33 | hours := int(d.Hours()) 34 | d -= time.Duration(hours) * time.Hour 35 | 36 | // Extract the number of minutes, and subtract them from the remaining duration. 37 | minutes := int(d.Minutes()) 38 | d -= time.Duration(minutes) * time.Minute 39 | 40 | // Extract the number of seconds from the remaining duration. 41 | seconds := int(d.Seconds()) 42 | 43 | // Format the hours, minutes, and seconds as a string in the "HH:mm:ss" format. 44 | return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) 45 | } 46 | -------------------------------------------------------------------------------- /cmd/v6manage/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2022 Lasse Haugen 3 | */ 4 | package main 5 | 6 | import "whynoipv6/cmd/v6manage/cmd" 7 | 8 | func main() { 9 | cmd.Execute() 10 | } 11 | -------------------------------------------------------------------------------- /db/geoip/GeoLite2-ASN.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/db/geoip/GeoLite2-ASN.mmdb -------------------------------------------------------------------------------- /db/geoip/GeoLite2-Country.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/db/geoip/GeoLite2-Country.mmdb -------------------------------------------------------------------------------- /db/migrations/01_schema.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE "lists" CASCADE; 2 | DROP TABLE "sites" CASCADE; 3 | DROP TABLE "metrics" CASCADE; 4 | DROP TABLE "asn" CASCADE; 5 | DROP TABLE "domain" CASCADE; 6 | DROP TABLE "country" CASCADE; 7 | DROP TABLE "changelog" CASCADE; 8 | DROP TABLE "campaign_changelog" CASCADE; 9 | DROP TABLE "campaign" CASCADE; 10 | DROP TABLE "campaign_domain" CASCADE; 11 | DROP TABLE "top_shame" CASCADE; 12 | DROP FUNCTION update_asn_metrics(); 13 | DROP FUNCTION update_country_metrics(); 14 | -------------------------------------------------------------------------------- /db/migrations/02_data.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lasseh/whynoipv6/6cc9090a629060e973211db72be485b8c02d1747/db/migrations/02_data.down.sql -------------------------------------------------------------------------------- /db/migrations/02_data.up.sql: -------------------------------------------------------------------------------- 1 | -- ---------------------------- 2 | -- Records of country 3 | -- ---------------------------- 4 | INSERT INTO "country" ("id", "country_name", "country_code", "country_tld", "continent") VALUES 5 | (1, 'Georgia', 'GE', '.GE', 'Asia'), 6 | (2, 'Afghanistan', 'AF', '.AF', 'Asia'), 7 | (3, 'Åland Islands', 'AX', '.AX', 'Europe'), 8 | (4, 'Albania', 'AL', '.AL', 'Europe'), 9 | (5, 'Algeria', 'DZ', '.DZ', 'Africa'), 10 | (6, 'American Samoa', 'AS', '.AS', 'Oceania'), 11 | (7, 'Andorra', 'AD', '.AD', 'Europe'), 12 | (8, 'Anguilla', 'AI', '.AI', 'North America'), 13 | (9, 'Antarctica', 'AQ', '.AQ', 'Antarctica'), 14 | (10, 'Antigua and Barbuda', 'AG', '.AG', 'North America'), 15 | (11, 'Armenia', 'AM', '.AM', 'Asia'), 16 | (12, 'Aruba', 'AW', '.AW', 'North America'), 17 | (13, 'Bahamas', 'BS', '.BS', 'North America'), 18 | (14, 'Bahrain', 'BH', '.BH', 'Asia'), 19 | (15, 'Barbados', 'BB', '.BB', 'North America'), 20 | (16, 'Benin', 'BJ', '.BJ', 'Africa'), 21 | (17, 'Bermuda', 'BM', '.BM', 'North America'), 22 | (18, 'Bhutan', 'BT', '.BT', 'Asia'), 23 | (19, 'Bolivia', 'BO', '.BO', 'South America'), 24 | (20, 'Botswana', 'BW', '.BW', 'Africa'), 25 | (21, 'Bouvet Island', 'BV', '.BV', 'Antarctica'), 26 | (22, 'British Indian Ocean Territory', 'IO', '.IO', 'Africa'), 27 | (23, 'Brunei Darussalam', 'BN', '.BN', 'Asia'), 28 | (24, 'Burkina Faso', 'BF', '.BF', 'Africa'), 29 | (25, 'Burundi', 'BI', '.BI', 'Africa'), 30 | (26, 'Cambodia', 'KH', '.KH', 'Asia'), 31 | (27, 'Cameroon', 'CM', '.CM', 'Africa'), 32 | (28, 'Cape Verde', 'CV', '.CV', 'Africa'), 33 | (29, 'Central African Republic', 'CF', '.CF', 'Africa'), 34 | (30, 'Chad', 'TD', '.TD', 'Africa'), 35 | (31, 'Christmas Island', 'CX', '.CX', 'Oceania'), 36 | (32, 'Cocos (Keeling) Islands', 'CC', '.CC', 'Oceania'), 37 | (33, 'Comoros', 'KM', '.KM', 'Africa'), 38 | (34, 'Congo', 'CG', '.CG', 'Africa'), 39 | (35, 'The Democratic Republic Of The Congo', 'CD', '.CD', 'Africa'), 40 | (36, 'Cook Islands', 'CK', '.CK', 'Oceania'), 41 | (37, 'Costa Rica', 'CR', '.CR', 'North America'), 42 | (38, 'Cote Divoire', 'CI', '.CI', 'Africa'), 43 | (39, 'Cuba', 'CU', '.CU', 'North America'), 44 | (40, 'Djibouti', 'DJ', '.DJ', 'Africa'), 45 | (41, 'Dominica', 'DM', '.DM', 'North America'), 46 | (42, 'Dominican Republic', 'DO', '.DO', 'North America'), 47 | (43, 'El Salvador', 'SV', '.SV', 'North America'), 48 | (44, 'Equatorial Guinea', 'GQ', '.GQ', 'Africa'), 49 | (45, 'Eritrea', 'ER', '.ER', 'Africa'), 50 | (46, 'Ethiopia', 'ET', '.ET', 'Africa'), 51 | (47, 'Falkland Islands (Malvinas)', 'FK', '.FK', 'South America'), 52 | (48, 'Faroe Islands', 'FO', '.FO', 'Europe'), 53 | (49, 'Fiji', 'FJ', '.FJ', 'Oceania'), 54 | (50, 'French Guiana', 'GF', '.GF', 'South America'), 55 | (51, 'French Polynesia', 'PF', '.PF', 'Oceania'), 56 | (52, 'French Southern Territories', 'TF', '.TF', 'Antarctica'), 57 | (53, 'Gabon', 'GA', '.GA', 'Africa'), 58 | (54, 'Gambia', 'GM', '.GM', 'Africa'), 59 | (55, 'Ghana', 'GH', '.GH', 'Africa'), 60 | (56, 'Gibraltar', 'GI', '.GI', 'Europe'), 61 | (57, 'Greenland', 'GL', '.GL', 'North America'), 62 | (58, 'Grenada', 'GD', '.GD', 'North America'), 63 | (59, 'Guadeloupe', 'GP', '.GP', 'North America'), 64 | (60, 'Guam', 'GU', '.GU', 'Oceania'), 65 | (61, 'Guatemala', 'GT', '.GT', 'North America'), 66 | (62, 'Guernsey', 'GG', '.GG', NULL), 67 | (63, 'Guinea', 'GN', '.GN', 'Africa'), 68 | (64, 'Guinea-Bissau', 'GW', '.GW', 'Africa'), 69 | (65, 'Guyana', 'GY', '.GY', 'South America'), 70 | (66, 'Haiti', 'HT', '.HT', 'North America'), 71 | (67, 'Heard and McDonald Islands', 'HM', '.HM', 'Antarctica'), 72 | (68, 'Holy See (Vatican City State)', 'VA', '.VA', 'Europe'), 73 | (69, 'Honduras', 'HN', '.HN', 'North America'), 74 | (70, 'Iceland', 'IS', '.IS', 'Europe'), 75 | (71, 'Iraq', 'IQ', '.IQ', 'Asia'), 76 | (72, 'Isle of Man', 'IM', '.IM', NULL), 77 | (73, 'Jamaica', 'JM', '.JM', 'North America'), 78 | (74, 'Jersey', 'JE', '.JE', NULL), 79 | (75, 'Jordan', 'JO', '.JO', 'Asia'), 80 | (76, 'Argentina', 'AR', '.AR', 'South America'), 81 | (77, 'Austria', 'AT', '.AT', 'Europe'), 82 | (78, 'Australia', 'AU', '.AU', 'Oceania'), 83 | (79, 'Azerbaijan', 'AZ', '.AZ', 'Asia'), 84 | (80, 'Bosnia and Herzegovina', 'BA', '.BA', 'Europe'), 85 | (81, 'Bangladesh', 'BD', '.BD', 'Asia'), 86 | (82, 'Belgium', 'BE', '.BE', 'Europe'), 87 | (83, 'Bulgaria', 'BG', '.BG', 'Europe'), 88 | (84, 'Brazil', 'BR', '.BR', 'South America'), 89 | (85, 'Belarus', 'BY', '.BY', 'Europe'), 90 | (86, 'Belize', 'BZ', '.BZ', 'North America'), 91 | (87, 'Canada', 'CA', '.CA', 'North America'), 92 | (88, 'Chile', 'CL', '.CL', 'South America'), 93 | (89, 'China', 'CN', '.CN', 'Asia'), 94 | (90, 'Colombia', 'CO', '.CO', 'South America'), 95 | (91, 'Cyprus', 'CY', '.CY', 'Asia'), 96 | (92, 'Czech Republic', 'CZ', '.CZ', 'Europe'), 97 | (93, 'Germany', 'DE', '.DE', 'Europe'), 98 | (94, 'Denmark', 'DK', '.DK', 'Europe'), 99 | (95, 'Ecuador', 'EC', '.EC', 'South America'), 100 | (96, 'Estonia', 'EE', '.EE', 'Europe'), 101 | (97, 'Egypt', 'EG', '.EG', 'Africa'), 102 | (98, 'Finland', 'FI', '.FI', 'Europe'), 103 | (99, 'France', 'FR', '.FR', 'Europe'), 104 | (100, 'Greece', 'GR', '.GR', 'Europe'), 105 | (101, 'Hong Kong', 'HK', '.HK', 'Asia'), 106 | (102, 'Croatia', 'HR', '.HR', 'Europe'), 107 | (103, 'Hungary', 'HU', '.HU', 'Europe'), 108 | (104, 'Indonesia', 'ID', '.ID', 'Asia'), 109 | (105, 'Ireland', 'IE', '.IE', 'Europe'), 110 | (106, 'Israel', 'IL', '.IL', 'Asia'), 111 | (107, 'India', 'IN', '.IN', 'Asia'), 112 | (108, 'Iran', 'IR', '.IR', 'Asia'), 113 | (109, 'Italy', 'IT', '.IT', 'Europe'), 114 | (110, 'Japan', 'JP', '.JP', 'Asia'), 115 | (111, 'Cayman Islands', 'KY', '.KY', 'North America'), 116 | (112, 'Kazakhstan', 'KZ', '.KZ', 'Asia'), 117 | (113, 'Kiribati', 'KI', '.KI', 'Oceania'), 118 | (114, 'North Korea', 'KP', '.KP', 'Asia'), 119 | (115, 'Kuwait', 'KW', '.KW', 'Asia'), 120 | (116, 'Kyrgyzstan', 'KG', '.KG', 'Asia'), 121 | (117, 'Lao People', 'LA', '.LA', 'Asia'), 122 | (118, 'Lebanon', 'LB', '.LB', 'Asia'), 123 | (119, 'Lesotho', 'LS', '.LS', 'Africa'), 124 | (120, 'Liberia', 'LR', '.LR', 'Africa'), 125 | (121, 'Libya', 'LY', '.LY', 'Africa'), 126 | (122, 'Liechtenstein', 'LI', '.LI', 'Europe'), 127 | (123, 'Macao', 'MO', '.MO', 'Asia'), 128 | (124, 'Madagascar', 'MG', '.MG', 'Africa'), 129 | (125, 'Malawi', 'MW', '.MW', 'Africa'), 130 | (126, 'Maldives', 'MV', '.MV', 'Asia'), 131 | (127, 'Mali', 'ML', '.ML', 'Africa'), 132 | (128, 'Marshall Islands', 'MH', '.MH', 'Oceania'), 133 | (129, 'Martinique', 'MQ', '.MQ', 'North America'), 134 | (130, 'Mauritania', 'MR', '.MR', 'Africa'), 135 | (131, 'Mauritius', 'MU', '.MU', 'Africa'), 136 | (132, 'Mayotte', 'YT', '.YT', 'Africa'), 137 | (133, 'Micronesia', 'FM', '.FM', 'Oceania'), 138 | (134, 'Monaco', 'MC', '.MC', 'Europe'), 139 | (135, 'Mongolia', 'MN', '.MN', 'Asia'), 140 | (136, 'Montenegro', 'ME', '.ME', 'Europe'), 141 | (137, 'Montserrat', 'MS', '.MS', 'North America'), 142 | (138, 'Morocco', 'MA', '.MA', 'Africa'), 143 | (139, 'Mozambique', 'MZ', '.MZ', 'Africa'), 144 | (140, 'Myanmar', 'MM', '.MM', 'Asia'), 145 | (141, 'Namibia', 'NA', '.NA', 'Africa'), 146 | (142, 'Nauru', 'NR', '.NR', 'Oceania'), 147 | (143, 'Netherlands Antilles', 'AN', '.AN', NULL), 148 | (144, 'New Caledonia', 'NC', '.NC', 'Oceania'), 149 | (145, 'Nicaragua', 'NI', '.NI', 'North America'), 150 | (146, 'Niger', 'NE', '.NE', 'Africa'), 151 | (147, 'Nigeria', 'NG', '.NG', 'Africa'), 152 | (148, 'Niue', 'NU', '.NU', 'Oceania'), 153 | (149, 'Norfolk Island', 'NF', '.NF', 'Oceania'), 154 | (150, 'Northern Mariana Islands', 'MP', '.MP', 'Oceania'), 155 | (151, 'Oman', 'OM', '.OM', 'Asia'), 156 | (152, 'Palau', 'PW', '.PW', 'Oceania'), 157 | (153, 'Palestine', 'PS', '.PS', 'Asia'), 158 | (154, 'Papua New Guinea', 'PG', '.PG', 'Oceania'), 159 | (155, 'Paraguay', 'PY', '.PY', 'South America'), 160 | (156, 'Peru', 'PE', '.PE', 'South America'), 161 | (157, 'Philippines', 'PH', '.PH', 'Asia'), 162 | (158, 'Saint Pierre And Miquelon', 'PM', '.PM', 'North America'), 163 | (159, 'Pitcairn', 'PN', '.PN', 'Oceania'), 164 | (160, 'Puerto Rico', 'PR', '.PR', 'North America'), 165 | (161, 'Réunion', 'RE', '.RE', 'Africa'), 166 | (162, 'Rwanda', 'RW', '.RW', 'Africa'), 167 | (163, 'Saint Helena', 'SH', '.SH', 'Africa'), 168 | (164, 'Saint Kitts And Nevis', 'KN', '.KN', 'North America'), 169 | (165, 'Saint Lucia', 'LC', '.LC', 'North America'), 170 | (166, 'Saint Vincent And The Grenedines', 'VC', '.VC', 'North America'), 171 | (167, 'Samoa', 'WS', '.WS', 'Oceania'), 172 | (168, 'San Marino', 'SM', '.SM', 'Europe'), 173 | (169, 'Sao Tome and Principe', 'ST', '.ST', 'Africa'), 174 | (170, 'Senegal', 'SN', '.SN', 'Africa'), 175 | (171, 'Sierra Leone', 'SL', '.SL', 'Africa'), 176 | (172, 'Slovenia', 'SI', '.SI', 'Europe'), 177 | (173, 'Solomon Islands', 'SB', '.SB', 'Oceania'), 178 | (174, 'Somalia', 'SO', '.SO', 'Africa'), 179 | (175, 'South Georgia', 'GS', '.GS', 'Antarctica'), 180 | (176, 'Sri Lanka', 'LK', '.LK', 'Asia'), 181 | (177, 'Sudan', 'SD', '.SD', 'Africa'), 182 | (178, 'Suriname', 'SR', '.SR', 'South America'), 183 | (179, 'Svalbard And Jan Mayen', 'SJ', '.SJ', 'Europe'), 184 | (180, 'Swaziland', 'SZ', '.SZ', 'Africa'), 185 | (181, 'Syrian Arab Republic', 'SY', '.SY', 'Asia'), 186 | (182, 'Tajikistan', 'TJ', '.TJ', 'Asia'), 187 | (183, 'Tanzania', 'TZ', '.TZ', 'Africa'), 188 | (184, 'Timor-Leste', 'TL', '.TL', 'Asia'), 189 | (185, 'Togo', 'TG', '.TG', 'Africa'), 190 | (186, 'Tokelau', 'TK', '.TK', 'Oceania'), 191 | (187, 'Tonga', 'TO', '.TO', 'Oceania'), 192 | (188, 'Trinidad and Tobago', 'TT', '.TT', 'North America'), 193 | (189, 'Spain', 'ES', '.ES', 'Europe'), 194 | (190, 'Kenya', 'KE', '.KE', 'Africa'), 195 | (191, 'South Korea', 'KR', '.KR', 'Asia'), 196 | (192, 'Lithuania', 'LT', '.LT', 'Europe'), 197 | (193, 'Luxembourg', 'LU', '.LU', 'Europe'), 198 | (194, 'Latvia', 'LV', '.LV', 'Europe'), 199 | (195, 'Moldova', 'MD', '.MD', 'Europe'), 200 | (196, 'Macedonia', 'MK', '.MK', 'Europe'), 201 | (197, 'Malta', 'MT', '.MT', 'Europe'), 202 | (198, 'Mexico', 'MX', '.MX', 'North America'), 203 | (199, 'Malaysia', 'MY', '.MY', 'Asia'), 204 | (200, 'Netherlands', 'NL', '.NL', 'Europe'), 205 | (201, 'Norway', 'NO', '.NO', 'Europe'), 206 | (202, 'Nepal', 'NP', '.NP', 'Asia'), 207 | (203, 'New Zealand', 'NZ', '.NZ', 'Oceania'), 208 | (204, 'Panama', 'PA', '.PA', 'North America'), 209 | (205, 'Pakistan', 'PK', '.PK', 'Asia'), 210 | (206, 'Poland', 'PL', '.PL', 'Europe'), 211 | (207, 'Portugal', 'PT', '.PT', 'Europe'), 212 | (208, 'Qatar', 'QA', '.QA', 'Asia'), 213 | (209, 'Romania', 'RO', '.RO', 'Europe'), 214 | (210, 'Serbia', 'RS', '.RS', 'Europe'), 215 | (211, 'Russian Federation', 'RU', '.RU', 'Europe'), 216 | (212, 'Saudi Arabia', 'SA', '.SA', 'Asia'), 217 | (213, 'Seychelles', 'SC', '.SC', 'Africa'), 218 | (214, 'Sweden', 'SE', '.SE', 'Europe'), 219 | (215, 'Singapore', 'SG', '.SG', 'Asia'), 220 | (216, 'Slovakia', 'SK', '.SK', 'Europe'), 221 | (217, 'Thailand', 'TH', '.TH', 'Asia'), 222 | (218, 'Tunisia', 'TN', '.TN', 'Africa'), 223 | (219, 'Taiwan', 'TW', '.TW', 'Asia'), 224 | (220, 'South Africa', 'ZA', '.ZA', 'Africa'), 225 | (221, 'Turkmenistan', 'TM', '.TM', 'Asia'), 226 | (222, 'Turks and Caicos Islands', 'TC', '.TC', 'North America'), 227 | (223, 'Tuvalu', 'TV', '.TV', 'Oceania'), 228 | (224, 'Uganda', 'UG', '.UG', 'Africa'), 229 | (225, 'United Arab Emirates', 'AE', '.AE', 'Asia'), 230 | (226, 'United States Minor Outlying Islands', 'UM', '.UM', 'Oceania'), 231 | (227, 'Uruguay', 'UY', '.UY', 'South America'), 232 | (228, 'Vanuatu', 'VU', '.VU', 'Oceania'), 233 | (229, 'Virgin Islands', 'VI', '.VI', 'North America'), 234 | (230, 'Wallis and Futuna', 'WF', '.WF', 'Oceania'), 235 | (231, 'Western Sahara', 'EH', '.EH', 'Africa'), 236 | (232, 'Yemen', 'YE', '.YE', 'Asia'), 237 | (233, 'Zambia', 'ZM', '.ZM', 'Africa'), 238 | (234, 'Zimbabwe', 'ZW', '.ZW', 'Africa'), 239 | (235, 'Saint Barthélemy', 'BL', '.BL', 'North America'), 240 | (236, 'Saint Martin', 'MF', '.MF', NULL), 241 | (237, 'Bonaire', 'BQ', '.BQ', NULL), 242 | (238, 'Curaçao', 'CW', '.CW', NULL), 243 | (239, 'Sint Maarten', 'SX', '.SX', NULL), 244 | (240, 'South Sudan', 'SS', '.SS', 'Africa'), 245 | (241, 'Angola', 'AO', '.AO', 'Africa'), 246 | (242, 'Switzerland', 'CH', '.CH', 'Europe'), 247 | (243, 'United Kingdom', 'GB', '.UK', 'Europe'), 248 | (244, 'Turkey', 'TR', '.TR', 'Asia'), 249 | (245, 'Ukraine', 'UA', '.UA', 'Europe'), 250 | (246, 'United States', 'US', '.US', 'North America'), 251 | (247, 'Uzbekistan', 'UZ', '.UZ', 'Asia'), 252 | (248, 'Venezuela', 'VE', '.VE', 'South America'), 253 | (249, 'Virgin Islands', 'VG', '.VG', 'North America'), 254 | (250, 'Vietnam', 'VN', '.VN', 'Asia'), 255 | (251, 'Unknown', 'UN', '.UN', NULL); 256 | 257 | -- ASN 258 | INSERT INTO "asn" ("id", "number", "name") VALUES (1, '0', 'Unknown'); 259 | 260 | -- Top Shame Domains 261 | INSERT INTO "top_shame" (id, site) VALUES 262 | (1, 'twitter.com'), 263 | (2, 'twitch.tv'), 264 | (3, 'ebay.com'), 265 | (4, 'imgur.com'), 266 | (5, 'imdb.com'), 267 | (6, 'wordpress.com'), 268 | (7, 'github.com'), 269 | (8, 'paypal.com'), 270 | (9, 'stackoverflow.com'), 271 | (10, 'soundcloud.com'), 272 | (11, 'nytimes.com'), 273 | (12, 'w3schools.com'); 274 | -------------------------------------------------------------------------------- /db/query/asn.sql: -------------------------------------------------------------------------------- 1 | -- name: CreateASN :one 2 | -- The ON CONFLICT DO NOTHING clause prevents errors in case a record with the same ASN number already exists. 3 | INSERT INTO asn(number, name) 4 | VALUES ($1, $2) 5 | ON CONFLICT DO NOTHING 6 | RETURNING *; 7 | 8 | -- name: GetASByNumber :one 9 | SELECT * 10 | FROM asn 11 | WHERE number = $1 12 | LIMIT 1; 13 | 14 | -- name: AsnByIPv4 :many 15 | SELECT * 16 | FROM asn 17 | WHERE count_v4 IS NOT NULL AND id != 1 18 | ORDER BY count_v4 DESC 19 | LIMIT $1 OFFSET $2; 20 | 21 | -- name: AsnByIPv6 :many 22 | SELECT * 23 | FROM asn 24 | WHERE count_v4 IS NOT NULL AND id != 1 25 | ORDER BY count_v6 DESC 26 | LIMIT $1 OFFSET $2; 27 | 28 | -- name: SearchAsNumber :many 29 | SELECT * 30 | FROM asn 31 | WHERE number = $1 32 | ORDER BY count_v4 DESC 33 | LIMIT 100; 34 | 35 | -- name: SearchAsName :many 36 | SELECT * 37 | FROM asn 38 | WHERE name ILIKE '%' || $1 || '%' 39 | ORDER BY count_v4 DESC 40 | LIMIT 100; 41 | -------------------------------------------------------------------------------- /db/query/campaign.sql: -------------------------------------------------------------------------------- 1 | -- name: InsertCampaignDomain :exec 2 | -- The ON CONFLICT DO NOTHING clause prevents errors in case a record with the same campaign_id and site already exists. 3 | INSERT INTO campaign_domain(campaign_id, site) 4 | VALUES ($1, $2) 5 | ON CONFLICT DO NOTHING; 6 | 7 | -- name: ListCampaignDomain :many 8 | -- Description: Retrieves a list of campaign domains with additional information from 'asn' and 'country' tables. 9 | SELECT campaign_domain.*, 10 | asn.name as asname, 11 | country.country_name 12 | FROM campaign_domain 13 | LEFT JOIN asn ON campaign_domain.asn_id = asn.id 14 | LEFT JOIN country ON campaign_domain.country_id = country.id 15 | WHERE campaign_domain.campaign_id = $1 16 | ORDER BY campaign_domain.id 17 | LIMIT $2 OFFSET $3; 18 | 19 | -- name: ViewCampaignDomain :one 20 | SELECT campaign_domain.*, 21 | asn.name as asname, 22 | country.country_name 23 | FROM campaign_domain 24 | LEFT JOIN asn ON campaign_domain.asn_id = asn.id 25 | LEFT JOIN country ON campaign_domain.country_id = country.id 26 | WHERE site = $1 27 | AND campaign_id = $2 28 | LIMIT 1; 29 | 30 | -- name: CrawlCampaignDomain :many 31 | SELECT * 32 | FROM campaign_domain 33 | ORDER BY id 34 | LIMIT $1 OFFSET $2; 35 | 36 | -- name: UpdateCampaignDomain :exec 37 | UPDATE 38 | campaign_domain 39 | SET base_domain = $3, 40 | www_domain = $4, 41 | nameserver = $5, 42 | mx_record = $6, 43 | v6_only = $7, 44 | ts_base_domain = $8, 45 | ts_www_domain = $9, 46 | ts_nameserver = $10, 47 | ts_mx_record = $11, 48 | ts_v6_only = $12, 49 | ts_check = $13, 50 | ts_updated = $14, 51 | asn_id = $15, 52 | country_id = $16 53 | WHERE site = $1 54 | AND campaign_id = $2; 55 | 56 | -- name: DisableCampaignDomain :exec 57 | UPDATE 58 | campaign_domain 59 | SET disabled = TRUE 60 | WHERE site = $1; 61 | 62 | -- name: ListCampaign :many 63 | -- Description: Retrieves a list of campaigns along with their associated domain count. 64 | SELECT campaign.*, 65 | COUNT(campaign_domain.id) AS domain_count, 66 | COUNT( 67 | CASE 68 | WHEN campaign_domain.base_domain = 'supported' AND 69 | campaign_domain.www_domain = 'supported' AND 70 | campaign_domain.nameserver = 'supported' 71 | THEN 1 72 | ELSE NULL 73 | END 74 | ) AS v6_ready_count 75 | FROM campaign 76 | LEFT JOIN campaign_domain ON campaign.uuid = campaign_domain.campaign_id AND campaign.disabled = FALSE 77 | GROUP BY campaign.id 78 | ORDER BY campaign.id; 79 | 80 | -- name: GetCampaignByUUID :one 81 | SELECT campaign.*, 82 | COUNT(campaign_domain.id) AS domain_count, 83 | COUNT( 84 | CASE 85 | WHEN campaign_domain.base_domain = 'supported' AND 86 | campaign_domain.www_domain = 'supported' AND 87 | campaign_domain.nameserver = 'supported' 88 | THEN 1 89 | ELSE NULL 90 | END 91 | ) AS v6_ready_count 92 | FROM campaign 93 | LEFT JOIN campaign_domain ON campaign.uuid = campaign_domain.campaign_id 94 | WHERE campaign.uuid = $1 95 | GROUP BY campaign.id 96 | LIMIT 1; 97 | 98 | -- name: CreateCampaign :one 99 | INSERT INTO campaign(name, description) 100 | VALUES ($1, $2) 101 | RETURNING *; 102 | 103 | -- name: CreateOrUpdateCampaign :one 104 | INSERT INTO campaign(uuid, name, description) 105 | VALUES ($1, $2, $3) 106 | ON CONFLICT (uuid) DO UPDATE 107 | SET name = EXCLUDED.name, 108 | description = EXCLUDED.description 109 | RETURNING *; 110 | 111 | -- name: DeleteCampaignDomain :exec 112 | DELETE 113 | FROM campaign_domain 114 | WHERE campaign_id = $1 115 | AND site = $2; 116 | 117 | -- name: GetCampaignDomainsByName :many 118 | -- Used for searching campaign domains by site name. 119 | SELECT * 120 | FROM campaign_domain 121 | WHERE site LIKE '%' || $1 || '%' 122 | LIMIT $2 OFFSET $3; 123 | 124 | -- name: StoreCampaignDomainLog :exec 125 | INSERT INTO campaign_domain_log(domain_id, data) 126 | VALUES ($1, $2) 127 | RETURNING *; 128 | 129 | -- name: GetCampaignDomainLog :many 130 | SELECT id, 131 | time, 132 | data 133 | FROM campaign_domain_log 134 | WHERE domain_id = $1 135 | ORDER BY time DESC 136 | LIMIT 90; 137 | -------------------------------------------------------------------------------- /db/query/changelog.sql: -------------------------------------------------------------------------------- 1 | -- name: ListChangelog :many 2 | SELECT * 3 | FROM changelog_view 4 | LIMIT $1 OFFSET $2; 5 | 6 | -- name: ListCampaignChangelog :many 7 | SELECT * 8 | FROM changelog_campaign_view 9 | LIMIT $1 OFFSET $2; 10 | 11 | -- name: GetChangelogByDomain :many 12 | SELECT * 13 | FROM changelog_view 14 | WHERE site = $1 15 | LIMIT $2 OFFSET $3; 16 | 17 | -- name: GetChangelogByCampaign :many 18 | SELECT * 19 | FROM changelog_campaign_view 20 | WHERE campaign_id = $1 21 | LIMIT $2 OFFSET $3; 22 | 23 | -- name: GetChangelogByCampaignDomain :many 24 | SELECT * 25 | FROM changelog_campaign_view 26 | WHERE campaign_id = $1 27 | AND site = $2 28 | LIMIT $3 OFFSET $4; 29 | 30 | -- name: CreateChangelog :one 31 | INSERT INTO changelog (domain_id, message, ipv6_status) 32 | VALUES ($1, $2, $3) 33 | RETURNING *; 34 | 35 | -- name: CreateCampaignChangelog :one 36 | INSERT INTO campaign_changelog (domain_id, campaign_id, message, ipv6_status) 37 | VALUES ($1, $2, $3, $4) 38 | RETURNING *; 39 | -------------------------------------------------------------------------------- /db/query/country.sql: -------------------------------------------------------------------------------- 1 | -- name: GetCountry :one 2 | SELECT * 3 | FROM country 4 | WHERE country_code = $1 5 | LIMIT 1; 6 | 7 | -- name: GetCountryTld :one 8 | SELECT * 9 | FROM country 10 | WHERE country_tld = $1 11 | LIMIT 1; 12 | 13 | -- name: ListDomainsByCountry :many 14 | SELECT * 15 | FROM domain_view_list 16 | WHERE domain_view_list.country_id = $1 17 | AND ( 18 | domain_view_list.base_domain = 'unsupported' 19 | OR domain_view_list.www_domain = 'unsupported' 20 | ) 21 | ORDER BY domain_view_list.id 22 | LIMIT $2 OFFSET $3; 23 | 24 | -- name: ListDomainHeroesByCountry :many 25 | SELECT * 26 | FROM domain_view_list 27 | WHERE country_id = $1 28 | AND base_domain = 'supported' 29 | AND www_domain = 'supported' 30 | AND nameserver = 'supported' 31 | AND mx_record != 'unsupported' 32 | ORDER BY rank 33 | LIMIT $2 OFFSET $3; 34 | 35 | -- name: AllDomainsByCountry :many 36 | SELECT * 37 | FROM domain_view_list 38 | WHERE domain_view_list.country_id = $1 39 | ORDER BY domain_view_list.id 40 | LIMIT $2 OFFSET $3; 41 | 42 | -- name: ListCountry :many 43 | SELECT * 44 | FROM country 45 | ORDER BY sites DESC; 46 | -------------------------------------------------------------------------------- /db/query/domain.sql: -------------------------------------------------------------------------------- 1 | -- name: InsertDomain :exec 2 | INSERT INTO domain(site) 3 | VALUES ($1) 4 | ON CONFLICT DO NOTHING; 5 | 6 | -- name: ListDomain :many 7 | SELECT * 8 | FROM domain_view_list 9 | WHERE base_domain = 'unsupported' 10 | OR www_domain = 'unsupported' 11 | ORDER BY rank 12 | LIMIT $1 OFFSET $2; 13 | 14 | -- name: ListDomainHeroes :many 15 | SELECT * 16 | FROM domain_view_list 17 | WHERE base_domain = 'supported' 18 | AND www_domain = 'supported' 19 | AND nameserver = 'supported' 20 | AND mx_record != 'unsupported' 21 | ORDER BY rank 22 | LIMIT $1 OFFSET $2; 23 | 24 | -- name: CrawlDomain :many 25 | SELECT * 26 | FROM domain_crawl_list 27 | WHERE id > $1 28 | ORDER BY id 29 | LIMIT $2; 30 | 31 | -- name: ViewDomain :one 32 | SELECT * 33 | FROM domain_view_list 34 | WHERE site = $1 35 | LIMIT 1; 36 | 37 | -- name: UpdateDomain :exec 38 | UPDATE 39 | domain 40 | SET base_domain = $2, 41 | www_domain = $3, 42 | nameserver = $4, 43 | mx_record = $5, 44 | v6_only = $6, 45 | ts_base_domain = $7, 46 | ts_www_domain = $8, 47 | ts_nameserver = $9, 48 | ts_mx_record = $10, 49 | ts_v6_only = $11, 50 | ts_check = $12, 51 | ts_updated = $13, 52 | asn_id = $14, 53 | country_id = $15 54 | WHERE site = $1; 55 | 56 | -- name: DisableDomain :exec 57 | UPDATE 58 | domain 59 | SET disabled = TRUE 60 | WHERE site = $1; 61 | 62 | -- name: GetDomainsByName :many 63 | SELECT * 64 | FROM domain_view_list 65 | WHERE site LIKE '%' || $1 || '%' 66 | ORDER BY rank 67 | LIMIT $2 OFFSET $3; 68 | 69 | -- name: ListDomainShamers :many 70 | SELECT * 71 | FROM domain_shame_view; 72 | 73 | -- name: InitSpaceTimestamps :exec 74 | WITH DomainCount AS (SELECT count(*)::DECIMAL AS total_records 75 | FROM domain), 76 | IntervalCalculation AS (SELECT (NOW() - '1 days'::INTERVAL) AS calculatedStartTime, 77 | ('1 days'::INTERVAL) / total_records AS calculatedIntervalStep 78 | FROM DomainCount), 79 | SpacedTimestampUpdates AS (SELECT d.id, 80 | ic.calculatedStartTime + ic.calculatedIntervalStep * 81 | ROW_NUMBER() OVER (ORDER BY d.id) AS newSpacedTimestamp 82 | FROM domain d, 83 | IntervalCalculation ic) 84 | UPDATE domain 85 | SET ts_check = stu.newSpacedTimestamp 86 | FROM SpacedTimestampUpdates stu 87 | WHERE domain.id = stu.id; 88 | 89 | -- name: StoreDomainLog :exec 90 | INSERT INTO domain_log(domain_id, data) 91 | VALUES ($1, $2) 92 | RETURNING *; 93 | 94 | -- name: GetDomainLog :many 95 | SELECT id, 96 | time, 97 | data 98 | FROM domain_log 99 | WHERE domain_id = $1 100 | ORDER BY time DESC 101 | LIMIT 90; 102 | -------------------------------------------------------------------------------- /db/query/metrics.sql: -------------------------------------------------------------------------------- 1 | -- name: StoreMetric :exec 2 | INSERT INTO metrics(measurement, data) 3 | VALUES ($1, $2) 4 | RETURNING *; 5 | 6 | -- name: GetMetric :many 7 | SELECT time, 8 | data 9 | FROM metrics 10 | WHERE measurement = $1 11 | ORDER BY time DESC; 12 | 13 | -- name: DomainStats :many 14 | SELECT time, 15 | data 16 | FROM metrics 17 | WHERE measurement = 'domains' 18 | ORDER BY time DESC 19 | LIMIT 1; 20 | 21 | -- name: DomainStats :one 22 | -- SELECT 23 | -- time, 24 | -- ((data->>'total_sites')::NUMERIC) as TotalSites, 25 | -- ((data->>'total_ns')::NUMERIC) as TotalNs, 26 | -- ((data->>'total_aaaa')::NUMERIC) as TotalAaaa, 27 | -- ((data->>'total_www')::NUMERIC) as TotalWww, 28 | -- ((data->>'total_both')::NUMERIC) as TotalBoth, 29 | -- ((data->>'top_1k')::NUMERIC) as Top1k, 30 | -- ((data->>'top_ns')::NUMERIC) as Topns 31 | -- FROM 32 | -- metrics 33 | -- WHERE measurement = 'domains' LIMIT 1; 34 | -------------------------------------------------------------------------------- /db/query/sites.sql: -------------------------------------------------------------------------------- 1 | -- name: ListSites :many 2 | SELECT * 3 | FROM sites 4 | ORDER BY rank 5 | LIMIT $1 OFFSET $2; 6 | -------------------------------------------------------------------------------- /db/query/stats.sql: -------------------------------------------------------------------------------- 1 | -- name: CrawlerStats :one 2 | -- Used by the crawler to store total stats in the metric table 3 | SELECT 4 | count(1) AS "domains", 5 | count(1) filter (WHERE base_domain = 'supported') AS "base_domain", 6 | count(1) filter (WHERE www_domain = 'supported') AS "www_domain", 7 | count(1) filter (WHERE nameserver = 'supported') AS "nameserver", 8 | count(1) filter (WHERE mx_record = 'supported') AS "mx_record", 9 | count(1) filter (WHERE base_domain = 'supported' AND www_domain = 'supported') AS "heroes", 10 | count(1) filter (WHERE base_domain != 'unsupported' AND www_domain != 'unsupported' AND rank < 1000) AS "top_heroes", 11 | count(1) filter (WHERE nameserver = 'supported' AND rank < 1000) AS "top_nameserver" 12 | FROM domain_view_list; 13 | 14 | -- name: CalculateCountryStats :exec 15 | SELECT update_country_metrics(); 16 | 17 | -- name: CalculateASNStats :exec 18 | SELECT update_asn_metrics(); 19 | -------------------------------------------------------------------------------- /extra/logrotate/whynoipv6-nginx.conf: -------------------------------------------------------------------------------- 1 | /var/log/nginx/whynoipv6.com.log { 2 | daily # Rotate the log files every day 3 | missingok # It's okay if the log file doesn't exist 4 | rotate 14 # Keep 14 days worth of backlogs 5 | compress # Compress (gzip) the log files on rotation 6 | delaycompress # Delay compression until the next log rotation cycle 7 | notifempty # Do not rotate the log if it's empty 8 | create 0644 nginx nginx # Create a new log file with set permissions/owner/group 9 | sharedscripts # Run post-rotate script only after all logs are rotated 10 | postrotate # Script to run after rotating is done (typically to reload nginx) 11 | if [ -f /var/run/nginx.pid ]; then 12 | kill -USR1 `cat /var/run/nginx.pid` 13 | fi 14 | endscript 15 | } 16 | # TODO: NEEDS IMPROVEMENTS 17 | -------------------------------------------------------------------------------- /extra/nginx/ipv6.fail.conf: -------------------------------------------------------------------------------- 1 | # This is part of my nginx repo: https://github.com/lasseh/nginx-conf 2 | # Redirect HTTP to HTTPS 3 | server { 4 | listen 80; 5 | listen [::]:80; 6 | server_name ipv6.fail www.ipv6.fail api.ipv6.fail; 7 | 8 | # Allow ACME challenge 9 | include prefabs.d/letsencrypt.conf; 10 | 11 | location / { 12 | return 301 https://$server_name$request_uri; 13 | } 14 | } 15 | # Frontend 16 | server { 17 | listen 443 ssl http2; 18 | listen [::]:443 ssl http2; 19 | server_name ipv6.fail www.ipv6.fail; 20 | 21 | # SSL 22 | ssl_certificate /etc/letsencrypt/live/ipv6.fail/fullchain.pem; 23 | ssl_certificate_key /etc/letsencrypt/live/ipv6.fail/privkey.pem; 24 | ssl_trusted_certificate /etc/letsencrypt/live/ipv6.fail/chain.pem; 25 | 26 | # Security Headers 27 | include sites-security/ipv6.fail.conf; 28 | 29 | # Logging 30 | access_log /var/log/nginx/access.log cloudflare; 31 | error_log /var/log/nginx/error.log warn; 32 | 33 | # Additional config 34 | include nginx.d/general.conf; 35 | 36 | # Root 37 | root /usr/share/nginx/html/ipv6; 38 | index index.html; 39 | 40 | # Frontend 41 | location / { 42 | add_header Cache-Control no-cache; 43 | try_files $uri $uri/ /index.html; 44 | } 45 | } 46 | 47 | # HTTPS Reverse Proxy 48 | server { 49 | listen 443 ssl http2; 50 | listen [::]:443 ssl http2; 51 | server_name api.ipv6.fail; 52 | 53 | # SSL 54 | ssl_certificate /etc/letsencrypt/live/ipv6.fail/fullchain.pem; 55 | ssl_certificate_key /etc/letsencrypt/live/ipv6.fail/privkey.pem; 56 | ssl_trusted_certificate /etc/letsencrypt/live/ipv6.fail/chain.pem; 57 | 58 | # Security Headers 59 | include sites-security/ipv6.fail.conf; 60 | 61 | # Logging 62 | access_log /var/log/nginx/access.log cloudflare; 63 | error_log /var/log/nginx/error.log warn; 64 | 65 | # Additional config 66 | include nginx.d/general.conf; 67 | 68 | # Returns the client's IP 69 | location /ip { 70 | add_header 'Access-Control-Allow-Origin' '*' always; 71 | add_header Content-Type "application/json"; 72 | return 200 '{"ip":"$remote_addr"}\n'; 73 | } 74 | 75 | # Backend 76 | location / { 77 | include nginx.d/proxy.conf; 78 | proxy_pass http://localhost:9001; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /extra/nginx/whynoipv6.com.conf: -------------------------------------------------------------------------------- 1 | # This is part of my nginx repo: https://github.com/lasseh/nginx-conf 2 | # Redirect HTTP to HTTPS 3 | server { 4 | listen 80; 5 | listen [::]:80; 6 | server_name whynoipv6.com www.whynoipv6.com api.whynoipv6.com; 7 | 8 | # Allow ACME challenge 9 | include prefabs.d/letsencrypt.conf; 10 | 11 | location / { 12 | return 301 https://$server_name$request_uri; 13 | } 14 | } 15 | # Frontend 16 | server { 17 | listen 443 ssl http2; 18 | listen [::]:443 ssl http2; 19 | server_name whynoipv6.com www.whynoipv6.com; 20 | 21 | # SSL 22 | ssl_certificate /etc/letsencrypt/live/whynoipv6.com/fullchain.pem; 23 | ssl_certificate_key /etc/letsencrypt/live/whynoipv6.com/privkey.pem; 24 | ssl_trusted_certificate /etc/letsencrypt/live/whynoipv6.com/chain.pem; 25 | 26 | # Security Headers 27 | #include sites-security/whynoipv6.com.conf; 28 | 29 | # Logging 30 | access_log /var/log/nginx/access.log cloudflare; 31 | error_log /var/log/nginx/error.log warn; 32 | 33 | # Additional config 34 | include nginx.d/general.conf; 35 | 36 | # Root 37 | root /usr/share/nginx/html/ipv6; 38 | index index.html; 39 | 40 | # Frontend 41 | location / { 42 | try_files $uri $uri/ /index.html; 43 | } 44 | } 45 | 46 | # HTTPS Reverse Proxy 47 | server { 48 | listen 443 ssl http2; 49 | listen [::]:443 ssl http2; 50 | server_name api.whynoipv6.com; 51 | 52 | # SSL 53 | ssl_certificate /etc/letsencrypt/live/whynoipv6.com/fullchain.pem; 54 | ssl_certificate_key /etc/letsencrypt/live/whynoipv6.com/privkey.pem; 55 | ssl_trusted_certificate /etc/letsencrypt/live/whynoipv6.com/chain.pem; 56 | 57 | # Security Headers 58 | #include sites-security/ipv6.fail.conf; 59 | 60 | # Logging 61 | access_log /var/log/nginx/access.log cloudflare; 62 | error_log /var/log/nginx/error.log warn; 63 | 64 | # Additional config 65 | include nginx.d/general.conf; 66 | 67 | # Returns the client's IP 68 | location /ip { 69 | add_header 'Access-Control-Allow-Origin' '*' always; 70 | add_header Content-Type "application/json"; 71 | return 200 '{"ip":"$remote_addr"}\n'; 72 | } 73 | 74 | # Backend 75 | location / { 76 | include nginx.d/proxy.conf; 77 | proxy_pass http://localhost:9001; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /extra/systemd/whynoipv6-api.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=WhynoIPv6: API 3 | After=network-online.target postgresql.service 4 | 5 | [Service] 6 | Type=simple 7 | WorkingDirectory=/opt/whynoipv6/whynoipv6 8 | ExecStart=/opt/whynoipv6/go/bin/v6-api 9 | ExecReload=/usr/bin/kill -HUP $MAINPID 10 | Restart=on-failure 11 | RestartSec=30 12 | User=ipv6 13 | Group=ipv6 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /extra/systemd/whynoipv6-campaign-crawler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=WhynoIPv6: Campaign Crawler 3 | After=network-online.target postgresql.service 4 | 5 | [Service] 6 | Type=simple 7 | WorkingDirectory=/opt/whynoipv6/whynoipv6 8 | ExecStart=/opt/whynoipv6/go/bin/v6manage campaign crawl 9 | ExecReload=/usr/bin/kill -HUP $MAINPID 10 | Restart=on-failure 11 | RestartSec=30 12 | User=ipv6 13 | Group=ipv6 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /extra/systemd/whynoipv6-crawler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=WhynoIPv6: Crawler 3 | After=network-online.target postgresql.service 4 | 5 | [Service] 6 | Type=simple 7 | WorkingDirectory=/opt/whynoipv6/whynoipv6 8 | ExecStart=/opt/whynoipv6/go/bin/v6manage crawl 9 | ExecReload=/usr/bin/kill -HUP $MAINPID 10 | Restart=on-failure 11 | RestartSec=30 12 | User=ipv6 13 | Group=ipv6 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /extra/whynoipv6-campaign-import.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=WhynoIPv6: Campaign Import 3 | After=network-online.target postgresql-16.service 4 | 5 | [Service] 6 | Type=simple 7 | WorkingDirectory={{ project_path }} 8 | Environment="CAMPAIGN_PATH={{ ansible_env.GOPATH | default(default_gopath) }}/src/github.com/lasseh/whynoipv6-campaign/" 9 | ExecStart={{ bin_path }}/v6manage campaign import 10 | ExecReload=/usr/bin/kill -HUP $MAINPID 11 | Restart=on-failure 12 | RestartSec=30 13 | User={{ user }} 14 | Group={{ group}} 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /extra/whynoipv6-campaign-import.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=WhyNoIPv6: Import Campaigns 3 | Requires=v6-campaign-import.service 4 | 5 | [Timer] 6 | OnBootSec=15min 7 | OnCalendar=daily 8 | Persistent=true 9 | 10 | [Install] 11 | WantedBy=timers.target 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module whynoipv6 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.24.0 6 | 7 | require ( 8 | github.com/IncSW/geoip2 v0.1.3 9 | github.com/alexeyco/simpletable v1.0.0 10 | github.com/ggicci/httpin v0.19.0 11 | github.com/go-chi/chi/v5 v5.2.1 12 | github.com/go-chi/render v1.0.3 13 | github.com/google/uuid v1.6.0 14 | github.com/ivanpirog/coloredcobra v1.0.1 15 | github.com/jackc/pgconn v1.14.3 16 | github.com/jackc/pgtype v1.14.4 17 | github.com/jackc/pgx/v4 v4.18.3 18 | github.com/lithammer/shortuuid/v4 v4.2.0 19 | github.com/miekg/dns v1.1.64 20 | github.com/rs/cors v1.11.1 21 | github.com/rs/zerolog v1.34.0 22 | github.com/spf13/cobra v1.9.1 23 | github.com/spf13/viper v1.20.0 24 | golang.org/x/net v0.37.0 25 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 26 | gopkg.in/yaml.v3 v3.0.1 27 | ) 28 | 29 | require ( 30 | github.com/ajg/form v1.5.1 // indirect 31 | github.com/fatih/color v1.18.0 // indirect 32 | github.com/fsnotify/fsnotify v1.8.0 // indirect 33 | github.com/ggicci/owl v0.8.2 // indirect 34 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 35 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 36 | github.com/jackc/chunkreader/v2 v2.0.1 // indirect 37 | github.com/jackc/pgio v1.0.0 // indirect 38 | github.com/jackc/pgpassfile v1.0.0 // indirect 39 | github.com/jackc/pgproto3/v2 v2.3.3 // indirect 40 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 41 | github.com/jackc/puddle v1.3.0 // indirect 42 | github.com/mattn/go-colorable v0.1.14 // indirect 43 | github.com/mattn/go-isatty v0.0.20 // indirect 44 | github.com/mattn/go-runewidth v0.0.16 // indirect 45 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/rivo/uniseg v0.4.7 // indirect 48 | github.com/sagikazarmark/locafero v0.8.0 // indirect 49 | github.com/sourcegraph/conc v0.3.0 // indirect 50 | github.com/spf13/afero v1.14.0 // indirect 51 | github.com/spf13/cast v1.7.1 // indirect 52 | github.com/spf13/pflag v1.0.6 // indirect 53 | github.com/subosito/gotenv v1.6.0 // indirect 54 | go.uber.org/multierr v1.11.0 // indirect 55 | golang.org/x/crypto v0.36.0 // indirect 56 | golang.org/x/mod v0.24.0 // indirect 57 | golang.org/x/sync v0.12.0 // indirect 58 | golang.org/x/sys v0.31.0 // indirect 59 | golang.org/x/text v0.23.0 // indirect 60 | golang.org/x/tools v0.31.0 // indirect 61 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | // Config represents the configuration of the application read from app.env. 11 | type Config struct { 12 | DatabaseSource string `mapstructure:"DB_SOURCE"` 13 | APIPort string `mapstructure:"API_PORT"` 14 | IRCToken string `mapstructure:"IRC_TOKEN"` 15 | GeoIPPath string `mapstructure:"GEOIP_PATH"` 16 | CampaignPath string `mapstructure:"CAMPAIGN_PATH"` 17 | Nameserver string `mapstructure:"NAMESERVER"` 18 | HealthcheckCrawler string `mapstructure:"HEALTHCHECK_CRAWLER"` 19 | HealthcheckCampaign string `mapstructure:"HEALTHCHECK_CAMPAIGN"` 20 | } 21 | 22 | // Read reads the configuration from the app.env file. 23 | func Read() (*Config, error) { 24 | // Set up Viper to read environment variables. 25 | viper.AutomaticEnv() 26 | 27 | // Configure Viper to read the app.env file. 28 | viper.SetConfigName("app") 29 | viper.SetConfigType("env") 30 | viper.AddConfigPath(".") 31 | viper.AddConfigPath("$HOME") 32 | 33 | // Read the app.env file. 34 | if err := viper.ReadInConfig(); err != nil { 35 | log.Printf("Error reading config file: %v", err) 36 | return nil, errors.New("error reading config file") 37 | 38 | } 39 | 40 | // Unmarshal the configuration into a Config struct. 41 | var config Config 42 | if err := viper.Unmarshal(&config); err != nil { 43 | log.Printf("Error unmarshalling config: %v", err) 44 | return nil, errors.New("error unmarshalling config") 45 | } 46 | 47 | return &config, nil 48 | } 49 | -------------------------------------------------------------------------------- /internal/core/asn.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | 6 | "whynoipv6/internal/postgres/db" 7 | 8 | "github.com/jackc/pgx/v4" 9 | ) 10 | 11 | // ASNService provides methods to interact with BGP Autonomous System Numbers (ASNs). 12 | type ASNService struct { 13 | q *db.Queries 14 | } 15 | 16 | // NewASNService creates a new ASNService instance. 17 | func NewASNService(d db.DBTX) *ASNService { 18 | return &ASNService{ 19 | q: db.New(d), 20 | } 21 | } 22 | 23 | // ASNModel represents a BGP Autonomous System Number (ASN) and its associated information. 24 | type ASNModel struct { 25 | ID int64 `json:"id"` 26 | Number int32 `json:"asn"` 27 | Name string `json:"name"` 28 | CountV4 int32 `json:"count_v4,omitempty"` 29 | CountV6 int32 `json:"count_v6,omitempty"` 30 | PercentV4 float64 `json:"percent_v4,omitempty"` 31 | PercentV6 float64 `json:"percent_v6,omitempty"` 32 | } 33 | 34 | // CreateAsn creates a new BGP ASN record with the specified number and name. 35 | func (s *ASNService) CreateAsn(ctx context.Context, number int32, name string) (ASNModel, error) { 36 | asn, err := s.q.CreateASN(ctx, db.CreateASNParams{ 37 | Number: number, 38 | Name: name, 39 | }) 40 | if err != nil { 41 | return ASNModel{}, err 42 | } 43 | return ASNModel{ 44 | ID: asn.ID, 45 | Number: asn.Number, 46 | Name: asn.Name, 47 | }, nil 48 | } 49 | 50 | // GetASByNumber retrieves the BGP ASN record with the specified AS number. 51 | func (s *ASNService) GetASByNumber(ctx context.Context, number int32) (ASNModel, error) { 52 | asnRecord, err := s.q.GetASByNumber(ctx, number) 53 | if err == pgx.ErrNoRows { 54 | return ASNModel{}, pgx.ErrNoRows 55 | } 56 | if err != nil { 57 | return ASNModel{}, err 58 | } 59 | return ASNModel{ 60 | ID: asnRecord.ID, 61 | Number: asnRecord.Number, 62 | Name: asnRecord.Name, 63 | }, nil 64 | } 65 | 66 | // ListASN retrieves all BGP ASN records. 67 | // func (s *ASNService) ListASN(ctx context.Context, offset, limit int32) ([]ASNModel, error) { 68 | // asnRecords, err := s.q.ListASN(ctx, db.ListASNParams{ 69 | // Offset: offset, 70 | // Limit: limit, 71 | // }) 72 | // if err != nil { 73 | // return nil, err 74 | // } 75 | // asns := make([]ASNModel, len(asnRecords)) 76 | // for i, asnRecord := range asnRecords { 77 | // asns[i] = ASNModel{ 78 | // ID: asnRecord.ID, 79 | // Number: asnRecord.Number, 80 | // Name: asnRecord.Name, 81 | // CountV4: asnRecord.CountV4.Int32, 82 | // CountV6: asnRecord.CountV6.Int32, 83 | // } 84 | // } 85 | // return asns, nil 86 | // } 87 | 88 | // CalculateASNStats calculates the statistics for an ASN. 89 | func (s *ASNService) CalculateASNStats(ctx context.Context) error { 90 | return s.q.CalculateASNStats(ctx) 91 | } 92 | -------------------------------------------------------------------------------- /internal/core/campaign.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "whynoipv6/internal/postgres/db" 9 | 10 | "github.com/google/uuid" 11 | "github.com/jackc/pgtype" 12 | ) 13 | 14 | // CampaignService is a service for managing scans. 15 | type CampaignService struct { 16 | q *db.Queries 17 | } 18 | 19 | // NewCampaignService creates a new ScanService. 20 | func NewCampaignService(d db.DBTX) *CampaignService { 21 | return &CampaignService{ 22 | q: db.New(d), 23 | } 24 | } 25 | 26 | // CampaignModel represents a campaign. 27 | type CampaignModel struct { 28 | ID int64 `json:"id"` 29 | CreatedAt time.Time `json:"created_at"` 30 | UUID uuid.UUID `json:"uuid"` 31 | Name string `json:"name"` 32 | Description string `json:"description"` 33 | Count int64 `json:"count"` 34 | V6Ready int64 `json:"v6_ready"` 35 | } 36 | 37 | // CampaignDomainModel represents a scan. 38 | type CampaignDomainModel struct { 39 | ID int64 `json:"id"` 40 | Site string `json:"site"` 41 | CampaignID uuid.UUID `json:"campaign_id"` 42 | BaseDomain string `json:"check_aaaa"` 43 | WwwDomain string `json:"check_www"` 44 | Nameserver string `json:"check_ns"` 45 | MXRecord string `json:"check_mx"` 46 | V6Only string `json:"check_curl"` 47 | AsnID int64 `json:"asn_id"` 48 | AsName string `json:"asn"` 49 | CountryID int64 `json:"country_id"` 50 | Country string `json:"country"` 51 | TsBaseDomain time.Time `json:"ts_aaaa"` 52 | TsWwwDomain time.Time `json:"ts_www"` 53 | TsNameserver time.Time `json:"ts_ns"` 54 | TsMXRecord time.Time `json:"ts_mx"` 55 | TsV6Only time.Time `json:"ts_curl"` 56 | TsCheck time.Time `json:"ts_check"` 57 | TsUpdated time.Time `json:"ts_updated"` 58 | } 59 | 60 | // InsertCampaignDomain inserts a domain into a campaign. 61 | func (s *CampaignService) InsertCampaignDomain( 62 | ctx context.Context, 63 | campaignID uuid.UUID, 64 | domain string, 65 | ) error { 66 | err := s.q.InsertCampaignDomain(ctx, db.InsertCampaignDomainParams{ 67 | CampaignID: campaignID, 68 | Site: domain, 69 | }) 70 | if err != nil { 71 | return err 72 | } 73 | return nil 74 | } 75 | 76 | // CrawlCampaignDomain lists all domains available for crawling 77 | func (s *CampaignService) CrawlCampaignDomain( 78 | ctx context.Context, 79 | offset, limit int64, 80 | ) ([]CampaignDomainModel, error) { 81 | domains, err := s.q.CrawlCampaignDomain(ctx, db.CrawlCampaignDomainParams{ 82 | Offset: offset, 83 | Limit: limit, 84 | }) 85 | if err != nil { 86 | return nil, err 87 | } 88 | var list []CampaignDomainModel 89 | for _, d := range domains { 90 | list = append(list, CampaignDomainModel{ 91 | ID: d.ID, 92 | Site: d.Site, 93 | CampaignID: d.CampaignID, 94 | BaseDomain: d.BaseDomain, 95 | WwwDomain: d.WwwDomain, 96 | Nameserver: d.Nameserver, 97 | MXRecord: d.MxRecord, 98 | V6Only: d.V6Only, 99 | TsBaseDomain: TimeNull(d.TsBaseDomain), 100 | TsWwwDomain: TimeNull(d.TsWwwDomain), 101 | TsNameserver: TimeNull(d.TsNameserver), 102 | TsMXRecord: TimeNull(d.TsMxRecord), 103 | TsV6Only: TimeNull(d.TsV6Only), 104 | TsCheck: TimeNull(d.TsCheck), 105 | TsUpdated: TimeNull(d.TsUpdated), 106 | }) 107 | } 108 | return list, nil 109 | } 110 | 111 | // UpdateCampaignDomain updates a domain. 112 | func (s *CampaignService) UpdateCampaignDomain( 113 | ctx context.Context, 114 | domain CampaignDomainModel, 115 | ) error { 116 | err := s.q.UpdateCampaignDomain(ctx, db.UpdateCampaignDomainParams{ 117 | Site: domain.Site, 118 | CampaignID: domain.CampaignID, 119 | BaseDomain: domain.BaseDomain, 120 | WwwDomain: domain.WwwDomain, 121 | Nameserver: domain.Nameserver, 122 | MxRecord: domain.MXRecord, 123 | V6Only: domain.V6Only, 124 | AsnID: NullInt(domain.AsnID), 125 | CountryID: NullInt(domain.CountryID), 126 | TsBaseDomain: NullTime(domain.TsBaseDomain), 127 | TsWwwDomain: NullTime(domain.TsWwwDomain), 128 | TsNameserver: NullTime(domain.TsNameserver), 129 | TsMxRecord: NullTime(domain.TsMXRecord), 130 | TsV6Only: NullTime(domain.TsV6Only), 131 | TsCheck: NullTime(domain.TsCheck), 132 | TsUpdated: NullTime(domain.TsUpdated), 133 | }) 134 | if err != nil { 135 | return err 136 | } 137 | return nil 138 | } 139 | 140 | // ViewCampaignDomain list a domain. 141 | func (s *CampaignService) ViewCampaignDomain( 142 | ctx context.Context, 143 | uuid uuid.UUID, 144 | domain string, 145 | ) (CampaignDomainModel, error) { 146 | d, err := s.q.ViewCampaignDomain(ctx, db.ViewCampaignDomainParams{ 147 | Site: domain, 148 | CampaignID: uuid, 149 | }) 150 | if err != nil { 151 | return CampaignDomainModel{}, err 152 | } 153 | return CampaignDomainModel{ 154 | ID: d.ID, 155 | Site: d.Site, 156 | BaseDomain: d.BaseDomain, 157 | WwwDomain: d.WwwDomain, 158 | Nameserver: d.Nameserver, 159 | MXRecord: d.MxRecord, 160 | V6Only: d.V6Only, 161 | AsName: StringNull(d.Asname), 162 | Country: StringNull(d.CountryName), 163 | TsBaseDomain: TimeNull(d.TsBaseDomain), 164 | TsWwwDomain: TimeNull(d.TsWwwDomain), 165 | TsNameserver: TimeNull(d.TsNameserver), 166 | TsMXRecord: TimeNull(d.TsMxRecord), 167 | TsV6Only: TimeNull(d.TsV6Only), 168 | TsCheck: TimeNull(d.TsCheck), 169 | TsUpdated: TimeNull(d.TsUpdated), 170 | }, nil 171 | } 172 | 173 | // DisableCampaignDomain disables a domain. 174 | func (s *CampaignService) DisableCampaignDomain(ctx context.Context, domain string) error { 175 | err := s.q.DisableCampaignDomain(ctx, domain) 176 | if err != nil { 177 | return err 178 | } 179 | return nil 180 | } 181 | 182 | // ListCampaign list all campaigns. 183 | func (s *CampaignService) ListCampaign(ctx context.Context) ([]CampaignModel, error) { 184 | campaigns, err := s.q.ListCampaign(ctx) 185 | if err != nil { 186 | return nil, err 187 | } 188 | var list []CampaignModel 189 | for _, c := range campaigns { 190 | list = append(list, CampaignModel{ 191 | ID: c.ID, 192 | CreatedAt: c.CreatedAt, 193 | UUID: c.Uuid, 194 | Name: c.Name, 195 | Description: c.Description, 196 | Count: c.DomainCount, 197 | V6Ready: c.V6ReadyCount, 198 | }) 199 | } 200 | return list, nil 201 | } 202 | 203 | // GetCampaign returns a campaign. 204 | func (s *CampaignService) GetCampaign(ctx context.Context, id uuid.UUID) (CampaignModel, error) { 205 | c, err := s.q.GetCampaignByUUID(ctx, id) 206 | if err != nil { 207 | return CampaignModel{}, err 208 | } 209 | return CampaignModel{ 210 | ID: c.ID, 211 | CreatedAt: c.CreatedAt, 212 | UUID: c.Uuid, 213 | Name: c.Name, 214 | Description: c.Description, 215 | Count: c.DomainCount, 216 | V6Ready: c.V6ReadyCount, 217 | }, nil 218 | } 219 | 220 | // CreateCampaign creates a new campaign and returns the new CampaignModel. 221 | func (s *CampaignService) CreateCampaign( 222 | ctx context.Context, 223 | name, description string, 224 | ) (CampaignModel, error) { 225 | c, err := s.q.CreateCampaign(ctx, db.CreateCampaignParams{ 226 | Name: name, 227 | Description: description, 228 | }) 229 | if err != nil { 230 | return CampaignModel{}, err 231 | } 232 | return CampaignModel{ 233 | ID: c.ID, 234 | UUID: c.Uuid, 235 | Name: c.Name, 236 | Description: c.Description, 237 | }, nil 238 | } 239 | 240 | // CreateOrUpdateCampaign creates or updates a campaign. 241 | func (s *CampaignService) CreateOrUpdateCampaign( 242 | ctx context.Context, 243 | campaign CampaignModel, 244 | ) (CampaignModel, error) { 245 | c, err := s.q.CreateOrUpdateCampaign(ctx, db.CreateOrUpdateCampaignParams{ 246 | Uuid: campaign.UUID, 247 | Name: campaign.Name, 248 | Description: campaign.Description, 249 | }) 250 | if err != nil { 251 | return CampaignModel{}, err 252 | } 253 | return CampaignModel{ 254 | ID: c.ID, 255 | UUID: c.Uuid, 256 | Name: c.Name, 257 | Description: c.Description, 258 | }, nil 259 | } 260 | 261 | // ListCampaignDomain lists all domains for a campaign. 262 | func (s *CampaignService) ListCampaignDomain( 263 | ctx context.Context, 264 | campaignID uuid.UUID, 265 | offset, limit int64, 266 | ) ([]CampaignDomainModel, error) { 267 | domains, err := s.q.ListCampaignDomain(ctx, db.ListCampaignDomainParams{ 268 | CampaignID: campaignID, 269 | Offset: offset, 270 | Limit: limit, 271 | }) 272 | if err != nil { 273 | return nil, err 274 | } 275 | var list []CampaignDomainModel 276 | for _, d := range domains { 277 | list = append(list, CampaignDomainModel{ 278 | ID: d.ID, 279 | Site: d.Site, 280 | CampaignID: d.CampaignID, 281 | BaseDomain: d.BaseDomain, 282 | WwwDomain: d.WwwDomain, 283 | Nameserver: d.Nameserver, 284 | MXRecord: d.MxRecord, 285 | V6Only: d.V6Only, 286 | TsBaseDomain: TimeNull(d.TsBaseDomain), 287 | TsWwwDomain: TimeNull(d.TsWwwDomain), 288 | TsNameserver: TimeNull(d.TsNameserver), 289 | TsMXRecord: TimeNull(d.TsMxRecord), 290 | TsV6Only: TimeNull(d.TsV6Only), 291 | TsCheck: TimeNull(d.TsCheck), 292 | TsUpdated: TimeNull(d.TsUpdated), 293 | AsName: StringNull(d.Asname), 294 | Country: StringNull(d.CountryName), 295 | }) 296 | } 297 | return list, nil 298 | } 299 | 300 | // DeleteCampaignDomain deletes a domain from a campaign. 301 | func (s *CampaignService) DeleteCampaignDomain( 302 | ctx context.Context, 303 | campaignID uuid.UUID, 304 | domain string, 305 | ) error { 306 | err := s.q.DeleteCampaignDomain(ctx, db.DeleteCampaignDomainParams{ 307 | CampaignID: campaignID, 308 | Site: domain, 309 | }) 310 | if err != nil { 311 | return err 312 | } 313 | return nil 314 | } 315 | 316 | // GetCampaignDomainsByName returns a list of domains from a campaign by name. 317 | func (s *CampaignService) GetCampaignDomainsByName( 318 | ctx context.Context, 319 | searchString string, 320 | offset, limit int64, 321 | ) ([]CampaignDomainModel, error) { 322 | domains, err := s.q.GetCampaignDomainsByName(ctx, db.GetCampaignDomainsByNameParams{ 323 | Column1: NullString(searchString), 324 | Offset: offset, 325 | Limit: limit, 326 | }) 327 | if err != nil { 328 | return nil, err 329 | } 330 | 331 | var list []CampaignDomainModel 332 | for _, d := range domains { 333 | list = append(list, CampaignDomainModel{ 334 | ID: d.ID, 335 | Site: d.Site, 336 | BaseDomain: d.BaseDomain, 337 | WwwDomain: d.WwwDomain, 338 | Nameserver: d.Nameserver, 339 | MXRecord: d.MxRecord, 340 | V6Only: d.V6Only, 341 | CampaignID: d.CampaignID, 342 | }) 343 | } 344 | return list, nil 345 | } 346 | 347 | // CampaignDomainLog represents a crawler data point. 348 | type CampaignDomainLog struct { 349 | ID int64 350 | Time time.Time 351 | Data pgtype.JSONB 352 | } 353 | 354 | // StoreCampaignDomainLog saves a crawl log for a campaign domain. 355 | func (s *CampaignService) StoreCampaignDomainLog( 356 | ctx context.Context, 357 | domain int64, 358 | data any, 359 | ) error { 360 | // Encode the data to a []byte 361 | dataBytes, err := json.Marshal(data) 362 | if err != nil { 363 | return err 364 | } 365 | 366 | // Create a new pgtype.JSONB struct 367 | jsonb := &pgtype.JSONB{} 368 | 369 | // Set the data on the pgtype.JSONB struct 370 | if err := jsonb.Set(dataBytes); err != nil { 371 | return err 372 | } 373 | 374 | return s.q.StoreCampaignDomainLog(ctx, db.StoreCampaignDomainLogParams{ 375 | DomainID: domain, 376 | Data: *jsonb, 377 | }) 378 | } 379 | 380 | // GetCampaignDomainLog retrieves all the logs for a specified domain. 381 | func (s *CampaignService) GetCampaignDomainLog( 382 | ctx context.Context, 383 | uuid uuid.UUID, 384 | domain string, 385 | ) ([]CampaignDomainLog, error) { 386 | // Get the domain ID from the database 387 | d, err := s.q.ViewCampaignDomain(ctx, db.ViewCampaignDomainParams{ 388 | CampaignID: uuid, 389 | Site: domain, 390 | }) 391 | if err != nil { 392 | return []CampaignDomainLog{}, err 393 | } 394 | 395 | logs, err := s.q.GetCampaignDomainLog(ctx, d.ID) 396 | if err != nil { 397 | return nil, err 398 | } 399 | 400 | var logList []CampaignDomainLog 401 | for _, log := range logs { 402 | logList = append(logList, CampaignDomainLog{ 403 | ID: log.ID, 404 | Time: log.Time, 405 | Data: log.Data, 406 | }) 407 | } 408 | return logList, nil 409 | } 410 | -------------------------------------------------------------------------------- /internal/core/changelog.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "whynoipv6/internal/postgres/db" 9 | 10 | "github.com/google/uuid" 11 | "github.com/jackc/pgx/v4" 12 | ) 13 | 14 | // ChangelogService is a service for managing changelogs. 15 | type ChangelogService struct { 16 | q *db.Queries 17 | } 18 | 19 | // NewChangelogService creates a new ChangelogService. 20 | func NewChangelogService(d db.DBTX) *ChangelogService { 21 | return &ChangelogService{ 22 | q: db.New(d), 23 | } 24 | } 25 | 26 | // ChangelogModel represents a changelog entry. 27 | type ChangelogModel struct { 28 | ID int64 `json:"id"` 29 | Ts time.Time `json:"timestamp"` 30 | DomainID int64 `json:"domain_id,omitempty"` 31 | CampaignID uuid.UUID `json:"campaign_id,omitempty"` 32 | Site string `json:"site"` 33 | Message string `json:"message"` 34 | IPv6Status string `json:"ipv6_status"` 35 | } 36 | 37 | // Create creates a new changelog entry. 38 | func (s *ChangelogService) Create( 39 | ctx context.Context, 40 | params ChangelogModel, 41 | ) (ChangelogModel, error) { 42 | changelog, err := s.q.CreateChangelog(ctx, db.CreateChangelogParams{ 43 | DomainID: params.DomainID, 44 | Message: params.Message, 45 | Ipv6Status: params.IPv6Status, 46 | }) 47 | if err != nil { 48 | return ChangelogModel{}, err 49 | } 50 | 51 | return ChangelogModel{ 52 | ID: changelog.ID, 53 | Ts: changelog.Ts, 54 | DomainID: changelog.DomainID, 55 | Message: changelog.Message, 56 | IPv6Status: changelog.Ipv6Status, 57 | }, nil 58 | } 59 | 60 | // CampaignCreate creates a new changelog entry for campaign table. 61 | func (s *ChangelogService) CampaignCreate( 62 | ctx context.Context, 63 | params ChangelogModel, 64 | ) (ChangelogModel, error) { 65 | changelog, err := s.q.CreateCampaignChangelog(ctx, db.CreateCampaignChangelogParams{ 66 | DomainID: params.DomainID, 67 | CampaignID: params.CampaignID, 68 | Message: params.Message, 69 | Ipv6Status: params.IPv6Status, 70 | }) 71 | if err != nil { 72 | return ChangelogModel{}, err 73 | } 74 | 75 | return ChangelogModel{ 76 | ID: changelog.ID, 77 | Ts: changelog.Ts, 78 | DomainID: changelog.DomainID, 79 | CampaignID: changelog.CampaignID, 80 | Message: changelog.Message, 81 | IPv6Status: changelog.Ipv6Status, 82 | }, nil 83 | } 84 | 85 | // List lists all changelog entries. 86 | func (s *ChangelogService) List( 87 | ctx context.Context, 88 | offset, limit int64, 89 | ) ([]ChangelogModel, error) { 90 | changelogs, err := s.q.ListChangelog(ctx, db.ListChangelogParams{ 91 | Offset: offset, 92 | Limit: limit, 93 | }) 94 | if err != nil { 95 | return nil, err 96 | } 97 | var models []ChangelogModel 98 | for _, changelog := range changelogs { 99 | models = append(models, ChangelogModel{ 100 | ID: changelog.ID, 101 | Ts: changelog.Ts, 102 | Site: changelog.Site, 103 | Message: changelog.Message, 104 | IPv6Status: changelog.Ipv6Status, 105 | }) 106 | } 107 | return models, nil 108 | } 109 | 110 | // CampaignList lists all changelog entries for campaign table. 111 | func (s *ChangelogService) CampaignList( 112 | ctx context.Context, 113 | offset, limit int64, 114 | ) ([]ChangelogModel, error) { 115 | changelogs, err := s.q.ListCampaignChangelog(ctx, db.ListCampaignChangelogParams{ 116 | Offset: offset, 117 | Limit: limit, 118 | }) 119 | if err != nil { 120 | return nil, err 121 | } 122 | var models []ChangelogModel 123 | for _, changelog := range changelogs { 124 | models = append(models, ChangelogModel{ 125 | ID: changelog.ID, 126 | Ts: changelog.Ts, 127 | Site: changelog.Site, 128 | CampaignID: changelog.CampaignID, 129 | Message: changelog.Message, 130 | IPv6Status: changelog.Ipv6Status, 131 | }) 132 | } 133 | return models, nil 134 | } 135 | 136 | // GetChangelogByDomain gets all changelog entries for a domain name. 137 | func (s *ChangelogService) GetChangelogByDomain( 138 | ctx context.Context, 139 | site string, 140 | offset, limit int64, 141 | ) ([]ChangelogModel, error) { 142 | // Get all changelog entries for site id 143 | changelogs, err := s.q.GetChangelogByDomain(ctx, db.GetChangelogByDomainParams{ 144 | Site: site, 145 | Offset: offset, 146 | Limit: limit, 147 | }) 148 | if err == pgx.ErrNoRows { 149 | return []ChangelogModel{}, errors.New("domain not found") 150 | } 151 | if err != nil { 152 | return []ChangelogModel{}, err 153 | } 154 | var models []ChangelogModel 155 | for _, changelog := range changelogs { 156 | models = append(models, ChangelogModel{ 157 | ID: changelog.ID, 158 | Ts: changelog.Ts, 159 | DomainID: changelog.DomainID, 160 | Site: changelog.Site, 161 | Message: changelog.Message, 162 | IPv6Status: changelog.Ipv6Status, 163 | }) 164 | } 165 | return models, nil 166 | } 167 | 168 | // GetChangelogByCampaign gets all changelog entries for a campaign. 169 | func (s *ChangelogService) GetChangelogByCampaign( 170 | ctx context.Context, 171 | campaignID uuid.UUID, 172 | offset, limit int64, 173 | ) ([]ChangelogModel, error) { 174 | // Get all changelog entries for site id 175 | changelogs, err := s.q.GetChangelogByCampaign(ctx, db.GetChangelogByCampaignParams{ 176 | CampaignID: campaignID, 177 | Offset: offset, 178 | Limit: limit, 179 | }) 180 | if err == pgx.ErrNoRows { 181 | return []ChangelogModel{}, errors.New("campaign not found") 182 | } 183 | if err != nil { 184 | return []ChangelogModel{}, err 185 | } 186 | var models []ChangelogModel 187 | for _, changelog := range changelogs { 188 | models = append(models, ChangelogModel{ 189 | ID: changelog.ID, 190 | Ts: changelog.Ts, 191 | DomainID: changelog.DomainID, 192 | CampaignID: changelog.CampaignID, 193 | Site: changelog.Site, 194 | Message: changelog.Message, 195 | IPv6Status: changelog.Ipv6Status, 196 | }) 197 | } 198 | return models, nil 199 | } 200 | 201 | // GetChangelogByCampaignDomain gets all changelog entries for a campaign and domain. 202 | func (s *ChangelogService) GetChangelogByCampaignDomain( 203 | ctx context.Context, 204 | campaignID uuid.UUID, 205 | site string, 206 | offset, limit int64, 207 | ) ([]ChangelogModel, error) { 208 | // Get all changelog entries for site id 209 | changelogs, err := s.q.GetChangelogByCampaignDomain(ctx, db.GetChangelogByCampaignDomainParams{ 210 | CampaignID: campaignID, 211 | Site: site, 212 | Offset: offset, 213 | Limit: limit, 214 | }) 215 | if err == pgx.ErrNoRows { 216 | return []ChangelogModel{}, errors.New("campaign not found") 217 | } 218 | if err != nil { 219 | return []ChangelogModel{}, err 220 | } 221 | var models []ChangelogModel 222 | for _, changelog := range changelogs { 223 | models = append(models, ChangelogModel{ 224 | ID: changelog.ID, 225 | Ts: changelog.Ts, 226 | DomainID: changelog.DomainID, 227 | CampaignID: changelog.CampaignID, 228 | Site: changelog.Site, 229 | Message: changelog.Message, 230 | IPv6Status: changelog.Ipv6Status, 231 | }) 232 | } 233 | return models, nil 234 | } 235 | -------------------------------------------------------------------------------- /internal/core/country.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "whynoipv6/internal/postgres/db" 8 | 9 | "github.com/jackc/pgtype" 10 | ) 11 | 12 | // CountryService is a service for managing countries. 13 | type CountryService struct { 14 | q *db.Queries 15 | } 16 | 17 | // NewCountryService creates a new CountryService instance. 18 | func NewCountryService(d db.DBTX) *CountryService { 19 | return &CountryService{ 20 | q: db.New(d), 21 | } 22 | } 23 | 24 | // CountryModel represents a country. 25 | type CountryModel struct { 26 | ID int64 `json:"id"` 27 | Country string `json:"country"` 28 | CountryCode string `json:"country_code"` 29 | CountryTld string `json:"country_tld"` 30 | Sites int32 `json:"sites"` 31 | V6sites int32 `json:"v6sites"` 32 | Percent pgtype.Numeric `json:"percent"` 33 | } 34 | 35 | // GetCountryCode gets a country by CountryCode. 36 | func (s *CountryService) GetCountryCode(ctx context.Context, code string) (CountryModel, error) { 37 | country, err := s.q.GetCountry(ctx, code) 38 | if err != nil { 39 | return CountryModel{}, err 40 | } 41 | return CountryModel{ 42 | ID: country.ID, 43 | Country: country.CountryName, 44 | CountryCode: country.CountryCode, 45 | CountryTld: country.CountryTld, 46 | Sites: country.Sites, 47 | V6sites: country.V6sites, 48 | Percent: country.Percent, 49 | }, nil 50 | } 51 | 52 | // GetCountryTld gets a country by CountryTLD. 53 | func (s *CountryService) GetCountryTld(ctx context.Context, tld string) (CountryModel, error) { 54 | country, err := s.q.GetCountryTld(ctx, strings.ToUpper(tld)) 55 | if err != nil { 56 | return CountryModel{}, err 57 | } 58 | return CountryModel{ 59 | ID: country.ID, 60 | Country: country.CountryName, 61 | CountryCode: country.CountryCode, 62 | CountryTld: country.CountryTld, 63 | Sites: country.Sites, 64 | V6sites: country.V6sites, 65 | Percent: country.Percent, 66 | }, nil 67 | } 68 | 69 | // List all countries. 70 | func (s *CountryService) List(ctx context.Context) ([]CountryModel, error) { 71 | countries, err := s.q.ListCountry(ctx) 72 | if err != nil { 73 | return nil, err 74 | } 75 | var models []CountryModel 76 | for _, country := range countries { 77 | models = append(models, CountryModel{ 78 | ID: country.ID, 79 | Country: country.CountryName, 80 | CountryCode: country.CountryCode, 81 | CountryTld: country.CountryTld, 82 | Sites: country.Sites, 83 | V6sites: country.V6sites, 84 | Percent: country.Percent, 85 | }) 86 | } 87 | return models, nil 88 | } 89 | 90 | // ListDomainsByCountry gets a list of all country TLDs. 91 | func (s *CountryService) ListDomainsByCountry( 92 | ctx context.Context, 93 | countryID int64, 94 | offset, limit int64, 95 | ) ([]DomainModel, error) { 96 | domains, err := s.q.ListDomainsByCountry(ctx, db.ListDomainsByCountryParams{ 97 | CountryID: NullInt(countryID), 98 | Offset: offset, 99 | Limit: limit, 100 | }) 101 | if err != nil { 102 | return nil, err 103 | } 104 | var list []DomainModel 105 | for _, d := range domains { 106 | list = append(list, DomainModel{ 107 | ID: IntNull(d.ID), 108 | Site: StringNull(d.Site), 109 | BaseDomain: StringNull(d.BaseDomain), 110 | WwwDomain: StringNull(d.WwwDomain), 111 | Nameserver: StringNull(d.Nameserver), 112 | MXRecord: StringNull(d.MxRecord), 113 | V6Only: StringNull(d.V6Only), 114 | AsName: StringNull(d.Asname), 115 | Country: StringNull(d.CountryName), 116 | TsBaseDomain: TimeNull(d.TsBaseDomain), 117 | TsWwwDomain: TimeNull(d.TsWwwDomain), 118 | TsNameserver: TimeNull(d.TsNameserver), 119 | TsMXRecord: TimeNull(d.TsMxRecord), 120 | TsV6Only: TimeNull(d.TsV6Only), 121 | TsCheck: TimeNull(d.TsCheck), 122 | TsUpdated: TimeNull(d.TsUpdated), 123 | Rank: d.Rank, 124 | }) 125 | } 126 | return list, nil 127 | } 128 | 129 | // ListDomainHeroesByCountry gets a list of all country TLDs. 130 | func (s *CountryService) ListDomainHeroesByCountry( 131 | ctx context.Context, 132 | countryID int64, 133 | offset, limit int64, 134 | ) ([]DomainModel, error) { 135 | domains, err := s.q.ListDomainHeroesByCountry(ctx, db.ListDomainHeroesByCountryParams{ 136 | CountryID: NullInt(countryID), 137 | Offset: offset, 138 | Limit: limit, 139 | }) 140 | if err != nil { 141 | return nil, err 142 | } 143 | var list []DomainModel 144 | for _, d := range domains { 145 | list = append(list, DomainModel{ 146 | ID: IntNull(d.ID), 147 | Site: StringNull(d.Site), 148 | BaseDomain: StringNull(d.BaseDomain), 149 | WwwDomain: StringNull(d.WwwDomain), 150 | Nameserver: StringNull(d.Nameserver), 151 | MXRecord: StringNull(d.MxRecord), 152 | V6Only: StringNull(d.V6Only), 153 | AsName: StringNull(d.Asname), 154 | Country: StringNull(d.CountryName), 155 | TsBaseDomain: TimeNull(d.TsBaseDomain), 156 | TsWwwDomain: TimeNull(d.TsWwwDomain), 157 | TsNameserver: TimeNull(d.TsNameserver), 158 | TsMXRecord: TimeNull(d.TsMxRecord), 159 | TsV6Only: TimeNull(d.TsV6Only), 160 | TsCheck: TimeNull(d.TsCheck), 161 | TsUpdated: TimeNull(d.TsUpdated), 162 | Rank: d.Rank, 163 | }) 164 | } 165 | return list, nil 166 | } 167 | 168 | // CalculateCountryStats calculates the statistics for a country. 169 | func (s *CountryService) CalculateCountryStats(ctx context.Context) error { 170 | return s.q.CalculateCountryStats(ctx) 171 | } 172 | -------------------------------------------------------------------------------- /internal/core/metric.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "whynoipv6/internal/postgres/db" 12 | 13 | "github.com/jackc/pgtype" 14 | ) 15 | 16 | // MetricService is a service for handling metrics. 17 | type MetricService struct { 18 | q *db.Queries 19 | } 20 | 21 | // NewMetricService creates a new MetricService instance. 22 | func NewMetricService(d db.DBTX) *MetricService { 23 | return &MetricService{ 24 | q: db.New(d), 25 | } 26 | } 27 | 28 | // Metric represents a metric data point. 29 | type Metric struct { 30 | Time time.Time 31 | Data pgtype.JSONB 32 | } 33 | 34 | // StoreMetric stores a metric data point with the given measurement name and data. 35 | func (s *MetricService) StoreMetric(ctx context.Context, measurement string, data any) error { 36 | // Encode the data to a []byte 37 | dataBytes, err := json.Marshal(data) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | // Create a new pgtype.JSONB struct 43 | jsonb := &pgtype.JSONB{} 44 | 45 | // Set the data on the pgtype.JSONB struct 46 | if err := jsonb.Set(dataBytes); err != nil { 47 | return err 48 | } 49 | 50 | return s.q.StoreMetric(ctx, db.StoreMetricParams{ 51 | Measurement: measurement, 52 | Data: *jsonb, 53 | }) 54 | } 55 | 56 | // GetMetrics retrieves all the metrics for a specified measurement. 57 | func (s *MetricService) GetMetrics(ctx context.Context, measurement string) ([]Metric, error) { 58 | metrics, err := s.q.GetMetric(ctx, measurement) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | var metricList []Metric 64 | for _, metric := range metrics { 65 | metricList = append(metricList, Metric{ 66 | Time: metric.Time, 67 | Data: metric.Data, 68 | }) 69 | } 70 | return metricList, nil 71 | } 72 | 73 | // AsnList retrieves all BGP ASN records. 74 | func (s *MetricService) AsnList( 75 | ctx context.Context, 76 | offset, limit int64, 77 | order string, 78 | ) ([]ASNModel, error) { 79 | var asnRecords []db.Asn 80 | var err error 81 | 82 | switch order { 83 | case "ipv6": 84 | asnRecords, err = s.q.AsnByIPv6(ctx, db.AsnByIPv6Params{Offset: offset, Limit: limit}) 85 | default: // default to ipv4 86 | asnRecords, err = s.q.AsnByIPv4(ctx, db.AsnByIPv4Params{Offset: offset, Limit: limit}) 87 | } 88 | 89 | if err != nil { 90 | return nil, fmt.Errorf("error fetching ASN records: %w", err) 91 | } 92 | 93 | asns := make([]ASNModel, 0, len(asnRecords)) 94 | for _, asn := range asnRecords { 95 | asns = append(asns, ASNModel{ 96 | ID: asn.ID, 97 | Number: asn.Number, 98 | Name: asn.Name, 99 | CountV4: asn.CountV4.Int32 - asn.CountV6.Int32, // Since all domains has IPv4, we subtract IPv6 from IPv4 to get the IPv4 only domains 100 | CountV6: asn.CountV6.Int32, 101 | }) 102 | } 103 | 104 | return asns, nil 105 | } 106 | 107 | // SearchAsn retrieves all BGP ASN records for the given ASN number. 108 | func (s *MetricService) SearchAsn(ctx context.Context, searchQuery string) ([]ASNModel, error) { 109 | // Normalize search query by trimming "AS" prefix if present 110 | searchQuery = strings.TrimPrefix(strings.ToUpper(searchQuery), "AS") 111 | 112 | var asnRecords []db.Asn 113 | asnNumber, err := strconv.Atoi(searchQuery) 114 | if err == nil { 115 | // Search query is a valid ASN number 116 | asnRecords, err = s.q.SearchAsNumber(ctx, int32(asnNumber)) 117 | if err != nil { 118 | return nil, fmt.Errorf("error fetching ASN record by number: %w", err) 119 | } 120 | } else { 121 | // Search query is an ASN name 122 | asnRecords, err = s.q.SearchAsName(ctx, NullString(searchQuery)) 123 | if err != nil { 124 | return nil, fmt.Errorf("error fetching ASN record by name: %w", err) 125 | } 126 | } 127 | 128 | asns := make([]ASNModel, 0, len(asnRecords)) 129 | for _, asn := range asnRecords { 130 | asns = append(asns, ASNModel{ 131 | ID: asn.ID, 132 | Number: asn.Number, 133 | Name: asn.Name, 134 | CountV4: asn.CountV4.Int32 - asn.CountV6.Int32, // Since all domains has IPv4, we subtract IPv6 from IPv4 to get the IPv4 only domains 135 | CountV6: asn.CountV6.Int32, 136 | }) 137 | } 138 | 139 | return asns, nil 140 | } 141 | 142 | // type DomainStatsModel struct { 143 | // TotalSites int64 `json:"total_sites"` 144 | // TotalAaaa int64 `json:"total_aaaa"` 145 | // TotalWww int64 `json:"total_www"` 146 | // TotalBoth int64 `json:"total_both"` 147 | // TotalNs int64 `json:"total_ns"` 148 | // Top1k int64 `json:"top_1k"` 149 | // TopNs int64 `json:"top_ns"` 150 | // } 151 | 152 | // DomainStatsModel represents a domain statistic. 153 | type DomainStatsModel struct { 154 | Time time.Time 155 | Totalsites pgtype.Numeric 156 | Totalns pgtype.Numeric 157 | Totalaaaa pgtype.Numeric 158 | Totalwww pgtype.Numeric 159 | Totalboth pgtype.Numeric 160 | Top1k pgtype.Numeric 161 | Topns pgtype.Numeric 162 | } 163 | 164 | // DomainStats retrieves the aggregated metrics for all crawled domains. 165 | func (s *MetricService) DomainStats(ctx context.Context) ([]Metric, error) { 166 | metrics, err := s.q.DomainStats(ctx) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | var metricList []Metric 172 | for _, metric := range metrics { 173 | metricList = append(metricList, Metric{ 174 | Time: metric.Time, 175 | Data: metric.Data, 176 | }) 177 | } 178 | return metricList, nil 179 | } 180 | -------------------------------------------------------------------------------- /internal/core/null.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "database/sql" 5 | "time" 6 | ) 7 | 8 | // TimeNull converts a sql.NullTime value to a time.Time value. 9 | // If the sql.NullTime value is not valid, it returns the zero value of time.Time. 10 | func TimeNull(t sql.NullTime) time.Time { 11 | if t.Valid { 12 | return t.Time 13 | } 14 | return time.Time{} 15 | } 16 | 17 | // NullTime converts a time.Time value to a sql.NullTime value. 18 | // If the time.Time value is zero, it returns an invalid sql.NullTime value. 19 | func NullTime(t time.Time) sql.NullTime { 20 | if t.IsZero() { 21 | return sql.NullTime{} 22 | } 23 | return sql.NullTime{ 24 | Time: t, 25 | Valid: true, 26 | } 27 | } 28 | 29 | // NullString converts a string value to a sql.NullString value. 30 | // It always returns a valid sql.NullString value. 31 | func NullString(s string) sql.NullString { 32 | return sql.NullString{ 33 | String: s, 34 | Valid: true, 35 | } 36 | } 37 | 38 | // StringNull converts a sql.NullString value to a string value. 39 | // If the sql.NullString value is not valid, it returns an empty string. 40 | func StringNull(s sql.NullString) string { 41 | if s.Valid { 42 | return s.String 43 | } 44 | return "" 45 | } 46 | 47 | // IntNull converts a sql.NullInt64 value to an int64 value. 48 | // If the sql.NullInt64 value is not valid, it returns 0. 49 | func IntNull(i sql.NullInt64) int64 { 50 | if i.Valid { 51 | return i.Int64 52 | } 53 | return 0 54 | } 55 | 56 | // NullInt32 converts an int32 value to a sql.NullInt64 value. 57 | // It always returns a valid sql.NullInt64 value. 58 | func NullInt32(i int32) sql.NullInt32 { 59 | return sql.NullInt32{ 60 | Int32: int32(i), 61 | Valid: true, 62 | } 63 | } 64 | 65 | // NullInt converts an int64 value to a sql.NullInt64 value. 66 | // It always returns a valid sql.NullInt64 value. 67 | func NullInt(i int64) sql.NullInt64 { 68 | return sql.NullInt64{ 69 | Int64: i, 70 | Valid: true, 71 | } 72 | } 73 | 74 | // BoolNull converts a sql.NullBool value to a bool value. 75 | // If the sql.NullBool value is not valid, it returns false. 76 | func BoolNull(b sql.NullBool) bool { 77 | if b.Valid { 78 | return b.Bool 79 | } 80 | return false 81 | } 82 | 83 | // NullBool converts a bool value to a sql.NullBool value. 84 | // It always returns a valid sql.NullBool value. 85 | func NullBool(b bool) sql.NullBool { 86 | return sql.NullBool{ 87 | Bool: b, 88 | Valid: true, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /internal/core/site.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | 6 | "whynoipv6/internal/postgres/db" 7 | ) 8 | 9 | // SiteService is a service for managing sites. 10 | type SiteService struct { 11 | q *db.Queries 12 | } 13 | 14 | // NewSiteService creates a new instance of SiteService. 15 | func NewSiteService(d db.DBTX) *SiteService { 16 | return &SiteService{ 17 | q: db.New(d), 18 | } 19 | } 20 | 21 | // SiteModel represents a site entity. 22 | type SiteModel struct { 23 | ID int64 `json:"id"` // Unique identifier for the site. 24 | Site string `json:"site"` // URL or name of the site. 25 | } 26 | 27 | // ListSite retrieves a list of sites with pagination support. 28 | // It accepts a context, offset, and limit as parameters. 29 | // Returns a slice of SiteModel and an error if any. 30 | func (s *SiteService) ListSite(ctx context.Context, offset, limit int64) ([]SiteModel, error) { 31 | sites, err := s.q.ListSites(ctx, db.ListSitesParams{ 32 | Offset: offset, 33 | Limit: limit, 34 | }) 35 | if err != nil { 36 | return []SiteModel{}, err 37 | } 38 | 39 | // Convert the list of sites into a slice of SiteModel. 40 | var siteModels []SiteModel 41 | for _, site := range sites { 42 | siteModels = append(siteModels, SiteModel{ 43 | ID: site.ID, 44 | Site: site.Site, 45 | }) 46 | } 47 | return siteModels, nil 48 | } 49 | -------------------------------------------------------------------------------- /internal/core/stats.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "whynoipv6/internal/postgres/db" 5 | ) 6 | 7 | // StatService is a service for managing domain statistics. 8 | type StatService struct { 9 | q *db.Queries 10 | } 11 | 12 | // NewStatService creates a new StatService instance. 13 | func NewStatService(d db.DBTX) *StatService { 14 | return &StatService{ 15 | q: db.New(d), 16 | } 17 | } 18 | 19 | // DomainStat represents a domain statistic. 20 | // type DomainStat struct { 21 | // TotalSites int64 `json:"total_sites"` 22 | // TotalAaaa int64 `json:"total_aaaa"` 23 | // TotalWww int64 `json:"total_www"` 24 | // TotalBoth int64 `json:"total_both"` 25 | // TotalNs int64 `json:"total_ns"` 26 | // Top1k int64 `json:"top_1k"` 27 | // TopNs int64 `json:"top_ns"` 28 | // } 29 | 30 | // DomainStats retrieves the statistics for all crawled domains. 31 | // func (s *StatService) DomainStats(ctx context.Context) (DomainStat, error) { 32 | // stats, err := s.q.DomainStats(ctx) 33 | // if err != nil { 34 | // return DomainStat{}, err 35 | // } 36 | // return DomainStat{ 37 | // TotalSites: stats.TotalSites, 38 | // TotalAaaa: stats.TotalAaaa, 39 | // TotalWww: stats.TotalWww, 40 | // TotalBoth: stats.TotalBoth, 41 | // TotalNs: stats.TotalNs, 42 | // Top1k: stats.Top1k, 43 | // TopNs: stats.TopNs, 44 | // }, nil 45 | // } 46 | 47 | // CalculateCountryStats calculates the statistics for a country. 48 | // func (s *StatService) CalculateCountryStats(ctx context.Context) error { 49 | // return s.q.CalculateCountryStats(ctx) 50 | // } 51 | 52 | // CalculateASNStats calculates the statistics for an ASN. 53 | // func (s *StatService) CalculateASNStats(ctx context.Context) error { 54 | // return s.q.CalculateASNStats(ctx) 55 | // } 56 | -------------------------------------------------------------------------------- /internal/geoip/geoip.go: -------------------------------------------------------------------------------- 1 | package geoip 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | "regexp" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/IncSW/geoip2" 11 | ) 12 | 13 | var ( 14 | // Global database readers. 15 | asnDB *geoip2.ASNReader 16 | countryDB *geoip2.CountryReader 17 | 18 | // Mutex for thread-safe initialization of the database readers. 19 | dbInitOnce sync.Once 20 | 21 | // Predefined error messages. 22 | errInvalidIP = errors.New("invalid IP address") 23 | errNoInfoFound = errors.New("no information found") 24 | // errDBInitFailed = errors.New("database initialization failed") 25 | 26 | // Compiled regular expression for TLD extraction. 27 | tldRegex = regexp.MustCompile(`(?i)([a-z0-9-]+)\.([a-z]{2,})$`) 28 | ) 29 | 30 | // Initialize initializes the database readers for ASN and country lookup. 31 | // asnDBPath and countryDBPath are file paths to the respective database files. 32 | // It should be called before using the package for bulk operations. 33 | func Initialize(dbPath string) error { 34 | var initErr error // Variable to hold initialization error 35 | 36 | dbInitOnce.Do(func() { 37 | asnDB, initErr = geoip2.NewASNReaderFromFile(dbPath + "GeoLite2-ASN.mmdb") 38 | if initErr != nil { 39 | return 40 | } 41 | 42 | countryDB, initErr = geoip2.NewCountryReaderFromFile(dbPath + "GeoLite2-Country.mmdb") 43 | if initErr != nil { 44 | return 45 | } 46 | }) 47 | 48 | return initErr // Return the error encountered during initialization, if any 49 | } 50 | 51 | // CloseDBs closes the database readers. 52 | // It should be called after the bulk operations are done to free up resources. 53 | func CloseDBs() { 54 | if asnDB != nil { 55 | // The geoip2 library does not provide a Close() method for the readers. 56 | // asnDB.Close() 57 | asnDB = nil 58 | } 59 | if countryDB != nil { 60 | // The geoip2 library does not provide a Close() method for the readers. 61 | // countryDB.Close() 62 | countryDB = nil 63 | } 64 | } 65 | 66 | // validateIP checks if the given IP address is valid. 67 | func validateIP(ip string) error { 68 | if net.ParseIP(ip) == nil { 69 | return errInvalidIP 70 | } 71 | return nil 72 | } 73 | 74 | // Asn represents an Autonomous System Number (ASN) record. 75 | type Asn struct { 76 | Number int32 77 | Name string 78 | } 79 | 80 | // AsnLookup retrieves the ASN information for a given IP address. 81 | // It returns an Asn struct and an error, if any. 82 | func AsnLookup(ip string) (Asn, error) { 83 | if err := validateIP(ip); err != nil { 84 | return Asn{}, err 85 | } 86 | 87 | record, err := asnDB.Lookup(net.ParseIP(ip)) 88 | if err != nil || record == nil { 89 | return Asn{}, errNoInfoFound 90 | } 91 | 92 | return Asn{ 93 | Number: int32(record.AutonomousSystemNumber), 94 | Name: record.AutonomousSystemOrganization, 95 | }, nil 96 | } 97 | 98 | // CountryLookup retrieves the country information for a given IP address. 99 | // It returns the country ISO code and an error, if any. 100 | func CountryLookup(ip string) (string, error) { 101 | if err := validateIP(ip); err != nil { 102 | return "", err 103 | } 104 | 105 | record, err := countryDB.Lookup(net.ParseIP(ip)) 106 | if err != nil || record == nil { 107 | return "", errNoInfoFound 108 | } 109 | 110 | return record.Country.ISOCode, nil 111 | } 112 | 113 | // ExtractTLDFromDomain extracts the Top-Level Domain (TLD) from a given domain. 114 | // It uses a regular expression to match the TLD pattern. 115 | // If a match is found, it returns the TLD in uppercase. 116 | // If no match is found, it returns an error. 117 | func ExtractTLDFromDomain(domain string) (string, error) { 118 | match := tldRegex.FindStringSubmatch(domain) 119 | if len(match) < 3 { 120 | return "", errors.New("no match found") 121 | } 122 | return strings.ToUpper(match[2]), nil 123 | } 124 | -------------------------------------------------------------------------------- /internal/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "github.com/rs/zerolog" 11 | "github.com/rs/zerolog/pkgerrors" 12 | "gopkg.in/natefinch/lumberjack.v2" 13 | ) 14 | 15 | // Singleton pattern is used to ensure only one instance of zerolog.Logger 16 | var singleton sync.Once 17 | 18 | // Instance of zerolog.Logger 19 | var loggerInstance zerolog.Logger 20 | 21 | // GetLogger initializes a zerolog.Logger instance if it has not been initialized 22 | // already and returns the same instance for subsequent calls. 23 | func GetLogger() zerolog.Logger { 24 | singleton.Do(func() { 25 | // Set zerolog to use pkgerrors for marshaling errors with stack trace 26 | zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack 27 | // Set the time format and field name for timestamps 28 | zerolog.TimeFieldFormat = time.RFC3339Nano 29 | zerolog.TimestampFieldName = "dt" // Custom time key for BetterStack 30 | 31 | // Default log level is Info 32 | logLevel := zerolog.InfoLevel 33 | // Check if log level is set in environment variable 34 | levelEnv := os.Getenv("LOG_LEVEL") 35 | if levelEnv != "" { 36 | levelFromEnv, err := zerolog.ParseLevel(levelEnv) 37 | if err != nil { 38 | log.Println(fmt.Errorf("defaulting to Info: %w", err)) 39 | } 40 | 41 | logLevel = levelFromEnv 42 | } 43 | // Configure an auto rotating file for storing JSON-formatted records 44 | fileLogger := &lumberjack.Logger{ 45 | Filename: "logs/crawler.log", 46 | MaxSize: 10, // Max size in megabytes before the file is rotated 47 | MaxBackups: 2, // Max number of old log files to keep 48 | MaxAge: 14, // Max number of days to retain the log files 49 | } 50 | 51 | // Configure console logging in a human-friendly and colorized format 52 | consoleLogger := zerolog.ConsoleWriter{ 53 | Out: os.Stdout, 54 | TimeFormat: "15:04:05", // 24-hour time format for console 55 | NoColor: false, // Enable color 56 | FieldsExclude: []string{ 57 | // "service", // Exclude service from console logs 58 | // "nameserver", // Exclude nameserver from console logs 59 | }, 60 | } 61 | 62 | // Allows logging to multiple destinations at once 63 | multiLevelOutput := zerolog.MultiLevelWriter(consoleLogger, fileLogger) 64 | 65 | // Create a global logger instance 66 | loggerInstance = zerolog.New(multiLevelOutput). 67 | Level(zerolog.Level(logLevel)). 68 | With(). 69 | Timestamp(). 70 | Logger() 71 | 72 | // Set the default logger context 73 | zerolog.DefaultContextLogger = &loggerInstance 74 | }) 75 | 76 | return loggerInstance 77 | } 78 | -------------------------------------------------------------------------------- /internal/postgres/db/asn.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: asn.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | "database/sql" 11 | ) 12 | 13 | const AsnByIPv4 = `-- name: AsnByIPv4 :many 14 | SELECT id, number, name, count_v4, count_v6, percent_v4, percent_v6 15 | FROM asn 16 | WHERE count_v4 IS NOT NULL AND id != 1 17 | ORDER BY count_v4 DESC 18 | LIMIT $1 OFFSET $2 19 | ` 20 | 21 | type AsnByIPv4Params struct { 22 | Limit int64 23 | Offset int64 24 | } 25 | 26 | func (q *Queries) AsnByIPv4(ctx context.Context, arg AsnByIPv4Params) ([]Asn, error) { 27 | rows, err := q.db.Query(ctx, AsnByIPv4, arg.Limit, arg.Offset) 28 | if err != nil { 29 | return nil, err 30 | } 31 | defer rows.Close() 32 | items := []Asn{} 33 | for rows.Next() { 34 | var i Asn 35 | if err := rows.Scan( 36 | &i.ID, 37 | &i.Number, 38 | &i.Name, 39 | &i.CountV4, 40 | &i.CountV6, 41 | &i.PercentV4, 42 | &i.PercentV6, 43 | ); err != nil { 44 | return nil, err 45 | } 46 | items = append(items, i) 47 | } 48 | if err := rows.Err(); err != nil { 49 | return nil, err 50 | } 51 | return items, nil 52 | } 53 | 54 | const AsnByIPv6 = `-- name: AsnByIPv6 :many 55 | SELECT id, number, name, count_v4, count_v6, percent_v4, percent_v6 56 | FROM asn 57 | WHERE count_v4 IS NOT NULL AND id != 1 58 | ORDER BY count_v6 DESC 59 | LIMIT $1 OFFSET $2 60 | ` 61 | 62 | type AsnByIPv6Params struct { 63 | Limit int64 64 | Offset int64 65 | } 66 | 67 | func (q *Queries) AsnByIPv6(ctx context.Context, arg AsnByIPv6Params) ([]Asn, error) { 68 | rows, err := q.db.Query(ctx, AsnByIPv6, arg.Limit, arg.Offset) 69 | if err != nil { 70 | return nil, err 71 | } 72 | defer rows.Close() 73 | items := []Asn{} 74 | for rows.Next() { 75 | var i Asn 76 | if err := rows.Scan( 77 | &i.ID, 78 | &i.Number, 79 | &i.Name, 80 | &i.CountV4, 81 | &i.CountV6, 82 | &i.PercentV4, 83 | &i.PercentV6, 84 | ); err != nil { 85 | return nil, err 86 | } 87 | items = append(items, i) 88 | } 89 | if err := rows.Err(); err != nil { 90 | return nil, err 91 | } 92 | return items, nil 93 | } 94 | 95 | const CreateASN = `-- name: CreateASN :one 96 | INSERT INTO asn(number, name) 97 | VALUES ($1, $2) 98 | ON CONFLICT DO NOTHING 99 | RETURNING id, number, name, count_v4, count_v6, percent_v4, percent_v6 100 | ` 101 | 102 | type CreateASNParams struct { 103 | Number int32 104 | Name string 105 | } 106 | 107 | // The ON CONFLICT DO NOTHING clause prevents errors in case a record with the same ASN number already exists. 108 | func (q *Queries) CreateASN(ctx context.Context, arg CreateASNParams) (Asn, error) { 109 | row := q.db.QueryRow(ctx, CreateASN, arg.Number, arg.Name) 110 | var i Asn 111 | err := row.Scan( 112 | &i.ID, 113 | &i.Number, 114 | &i.Name, 115 | &i.CountV4, 116 | &i.CountV6, 117 | &i.PercentV4, 118 | &i.PercentV6, 119 | ) 120 | return i, err 121 | } 122 | 123 | const GetASByNumber = `-- name: GetASByNumber :one 124 | SELECT id, number, name, count_v4, count_v6, percent_v4, percent_v6 125 | FROM asn 126 | WHERE number = $1 127 | LIMIT 1 128 | ` 129 | 130 | func (q *Queries) GetASByNumber(ctx context.Context, number int32) (Asn, error) { 131 | row := q.db.QueryRow(ctx, GetASByNumber, number) 132 | var i Asn 133 | err := row.Scan( 134 | &i.ID, 135 | &i.Number, 136 | &i.Name, 137 | &i.CountV4, 138 | &i.CountV6, 139 | &i.PercentV4, 140 | &i.PercentV6, 141 | ) 142 | return i, err 143 | } 144 | 145 | const SearchAsName = `-- name: SearchAsName :many 146 | SELECT id, number, name, count_v4, count_v6, percent_v4, percent_v6 147 | FROM asn 148 | WHERE name ILIKE '%' || $1 || '%' 149 | ORDER BY count_v4 DESC 150 | LIMIT 100 151 | ` 152 | 153 | func (q *Queries) SearchAsName(ctx context.Context, dollar_1 sql.NullString) ([]Asn, error) { 154 | rows, err := q.db.Query(ctx, SearchAsName, dollar_1) 155 | if err != nil { 156 | return nil, err 157 | } 158 | defer rows.Close() 159 | items := []Asn{} 160 | for rows.Next() { 161 | var i Asn 162 | if err := rows.Scan( 163 | &i.ID, 164 | &i.Number, 165 | &i.Name, 166 | &i.CountV4, 167 | &i.CountV6, 168 | &i.PercentV4, 169 | &i.PercentV6, 170 | ); err != nil { 171 | return nil, err 172 | } 173 | items = append(items, i) 174 | } 175 | if err := rows.Err(); err != nil { 176 | return nil, err 177 | } 178 | return items, nil 179 | } 180 | 181 | const SearchAsNumber = `-- name: SearchAsNumber :many 182 | SELECT id, number, name, count_v4, count_v6, percent_v4, percent_v6 183 | FROM asn 184 | WHERE number = $1 185 | ORDER BY count_v4 DESC 186 | LIMIT 100 187 | ` 188 | 189 | func (q *Queries) SearchAsNumber(ctx context.Context, number int32) ([]Asn, error) { 190 | rows, err := q.db.Query(ctx, SearchAsNumber, number) 191 | if err != nil { 192 | return nil, err 193 | } 194 | defer rows.Close() 195 | items := []Asn{} 196 | for rows.Next() { 197 | var i Asn 198 | if err := rows.Scan( 199 | &i.ID, 200 | &i.Number, 201 | &i.Name, 202 | &i.CountV4, 203 | &i.CountV6, 204 | &i.PercentV4, 205 | &i.PercentV6, 206 | ); err != nil { 207 | return nil, err 208 | } 209 | items = append(items, i) 210 | } 211 | if err := rows.Err(); err != nil { 212 | return nil, err 213 | } 214 | return items, nil 215 | } 216 | -------------------------------------------------------------------------------- /internal/postgres/db/changelog.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: changelog.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | 11 | "github.com/google/uuid" 12 | ) 13 | 14 | const CreateCampaignChangelog = `-- name: CreateCampaignChangelog :one 15 | INSERT INTO campaign_changelog (domain_id, campaign_id, message, ipv6_status) 16 | VALUES ($1, $2, $3, $4) 17 | RETURNING id, ts, domain_id, campaign_id, message, ipv6_status 18 | ` 19 | 20 | type CreateCampaignChangelogParams struct { 21 | DomainID int64 22 | CampaignID uuid.UUID 23 | Message string 24 | Ipv6Status string 25 | } 26 | 27 | func (q *Queries) CreateCampaignChangelog(ctx context.Context, arg CreateCampaignChangelogParams) (CampaignChangelog, error) { 28 | row := q.db.QueryRow(ctx, CreateCampaignChangelog, 29 | arg.DomainID, 30 | arg.CampaignID, 31 | arg.Message, 32 | arg.Ipv6Status, 33 | ) 34 | var i CampaignChangelog 35 | err := row.Scan( 36 | &i.ID, 37 | &i.Ts, 38 | &i.DomainID, 39 | &i.CampaignID, 40 | &i.Message, 41 | &i.Ipv6Status, 42 | ) 43 | return i, err 44 | } 45 | 46 | const CreateChangelog = `-- name: CreateChangelog :one 47 | INSERT INTO changelog (domain_id, message, ipv6_status) 48 | VALUES ($1, $2, $3) 49 | RETURNING id, ts, domain_id, message, ipv6_status 50 | ` 51 | 52 | type CreateChangelogParams struct { 53 | DomainID int64 54 | Message string 55 | Ipv6Status string 56 | } 57 | 58 | func (q *Queries) CreateChangelog(ctx context.Context, arg CreateChangelogParams) (Changelog, error) { 59 | row := q.db.QueryRow(ctx, CreateChangelog, arg.DomainID, arg.Message, arg.Ipv6Status) 60 | var i Changelog 61 | err := row.Scan( 62 | &i.ID, 63 | &i.Ts, 64 | &i.DomainID, 65 | &i.Message, 66 | &i.Ipv6Status, 67 | ) 68 | return i, err 69 | } 70 | 71 | const GetChangelogByCampaign = `-- name: GetChangelogByCampaign :many 72 | SELECT id, ts, domain_id, campaign_id, message, ipv6_status, site 73 | FROM changelog_campaign_view 74 | WHERE campaign_id = $1 75 | LIMIT $2 OFFSET $3 76 | ` 77 | 78 | type GetChangelogByCampaignParams struct { 79 | CampaignID uuid.UUID 80 | Limit int64 81 | Offset int64 82 | } 83 | 84 | func (q *Queries) GetChangelogByCampaign(ctx context.Context, arg GetChangelogByCampaignParams) ([]ChangelogCampaignView, error) { 85 | rows, err := q.db.Query(ctx, GetChangelogByCampaign, arg.CampaignID, arg.Limit, arg.Offset) 86 | if err != nil { 87 | return nil, err 88 | } 89 | defer rows.Close() 90 | items := []ChangelogCampaignView{} 91 | for rows.Next() { 92 | var i ChangelogCampaignView 93 | if err := rows.Scan( 94 | &i.ID, 95 | &i.Ts, 96 | &i.DomainID, 97 | &i.CampaignID, 98 | &i.Message, 99 | &i.Ipv6Status, 100 | &i.Site, 101 | ); err != nil { 102 | return nil, err 103 | } 104 | items = append(items, i) 105 | } 106 | if err := rows.Err(); err != nil { 107 | return nil, err 108 | } 109 | return items, nil 110 | } 111 | 112 | const GetChangelogByCampaignDomain = `-- name: GetChangelogByCampaignDomain :many 113 | SELECT id, ts, domain_id, campaign_id, message, ipv6_status, site 114 | FROM changelog_campaign_view 115 | WHERE campaign_id = $1 116 | AND site = $2 117 | LIMIT $3 OFFSET $4 118 | ` 119 | 120 | type GetChangelogByCampaignDomainParams struct { 121 | CampaignID uuid.UUID 122 | Site string 123 | Limit int64 124 | Offset int64 125 | } 126 | 127 | func (q *Queries) GetChangelogByCampaignDomain(ctx context.Context, arg GetChangelogByCampaignDomainParams) ([]ChangelogCampaignView, error) { 128 | rows, err := q.db.Query(ctx, GetChangelogByCampaignDomain, 129 | arg.CampaignID, 130 | arg.Site, 131 | arg.Limit, 132 | arg.Offset, 133 | ) 134 | if err != nil { 135 | return nil, err 136 | } 137 | defer rows.Close() 138 | items := []ChangelogCampaignView{} 139 | for rows.Next() { 140 | var i ChangelogCampaignView 141 | if err := rows.Scan( 142 | &i.ID, 143 | &i.Ts, 144 | &i.DomainID, 145 | &i.CampaignID, 146 | &i.Message, 147 | &i.Ipv6Status, 148 | &i.Site, 149 | ); err != nil { 150 | return nil, err 151 | } 152 | items = append(items, i) 153 | } 154 | if err := rows.Err(); err != nil { 155 | return nil, err 156 | } 157 | return items, nil 158 | } 159 | 160 | const GetChangelogByDomain = `-- name: GetChangelogByDomain :many 161 | SELECT id, ts, domain_id, message, ipv6_status, site 162 | FROM changelog_view 163 | WHERE site = $1 164 | LIMIT $2 OFFSET $3 165 | ` 166 | 167 | type GetChangelogByDomainParams struct { 168 | Site string 169 | Limit int64 170 | Offset int64 171 | } 172 | 173 | func (q *Queries) GetChangelogByDomain(ctx context.Context, arg GetChangelogByDomainParams) ([]ChangelogView, error) { 174 | rows, err := q.db.Query(ctx, GetChangelogByDomain, arg.Site, arg.Limit, arg.Offset) 175 | if err != nil { 176 | return nil, err 177 | } 178 | defer rows.Close() 179 | items := []ChangelogView{} 180 | for rows.Next() { 181 | var i ChangelogView 182 | if err := rows.Scan( 183 | &i.ID, 184 | &i.Ts, 185 | &i.DomainID, 186 | &i.Message, 187 | &i.Ipv6Status, 188 | &i.Site, 189 | ); err != nil { 190 | return nil, err 191 | } 192 | items = append(items, i) 193 | } 194 | if err := rows.Err(); err != nil { 195 | return nil, err 196 | } 197 | return items, nil 198 | } 199 | 200 | const ListCampaignChangelog = `-- name: ListCampaignChangelog :many 201 | SELECT id, ts, domain_id, campaign_id, message, ipv6_status, site 202 | FROM changelog_campaign_view 203 | LIMIT $1 OFFSET $2 204 | ` 205 | 206 | type ListCampaignChangelogParams struct { 207 | Limit int64 208 | Offset int64 209 | } 210 | 211 | func (q *Queries) ListCampaignChangelog(ctx context.Context, arg ListCampaignChangelogParams) ([]ChangelogCampaignView, error) { 212 | rows, err := q.db.Query(ctx, ListCampaignChangelog, arg.Limit, arg.Offset) 213 | if err != nil { 214 | return nil, err 215 | } 216 | defer rows.Close() 217 | items := []ChangelogCampaignView{} 218 | for rows.Next() { 219 | var i ChangelogCampaignView 220 | if err := rows.Scan( 221 | &i.ID, 222 | &i.Ts, 223 | &i.DomainID, 224 | &i.CampaignID, 225 | &i.Message, 226 | &i.Ipv6Status, 227 | &i.Site, 228 | ); err != nil { 229 | return nil, err 230 | } 231 | items = append(items, i) 232 | } 233 | if err := rows.Err(); err != nil { 234 | return nil, err 235 | } 236 | return items, nil 237 | } 238 | 239 | const ListChangelog = `-- name: ListChangelog :many 240 | SELECT id, ts, domain_id, message, ipv6_status, site 241 | FROM changelog_view 242 | LIMIT $1 OFFSET $2 243 | ` 244 | 245 | type ListChangelogParams struct { 246 | Limit int64 247 | Offset int64 248 | } 249 | 250 | func (q *Queries) ListChangelog(ctx context.Context, arg ListChangelogParams) ([]ChangelogView, error) { 251 | rows, err := q.db.Query(ctx, ListChangelog, arg.Limit, arg.Offset) 252 | if err != nil { 253 | return nil, err 254 | } 255 | defer rows.Close() 256 | items := []ChangelogView{} 257 | for rows.Next() { 258 | var i ChangelogView 259 | if err := rows.Scan( 260 | &i.ID, 261 | &i.Ts, 262 | &i.DomainID, 263 | &i.Message, 264 | &i.Ipv6Status, 265 | &i.Site, 266 | ); err != nil { 267 | return nil, err 268 | } 269 | items = append(items, i) 270 | } 271 | if err := rows.Err(); err != nil { 272 | return nil, err 273 | } 274 | return items, nil 275 | } 276 | -------------------------------------------------------------------------------- /internal/postgres/db/country.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: country.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | "database/sql" 11 | ) 12 | 13 | const AllDomainsByCountry = `-- name: AllDomainsByCountry :many 14 | SELECT id, site, base_domain, www_domain, nameserver, mx_record, v6_only, asn_id, country_id, disabled, ts_base_domain, ts_www_domain, ts_nameserver, ts_mx_record, ts_v6_only, ts_check, ts_updated, rank, asname, country_name 15 | FROM domain_view_list 16 | WHERE domain_view_list.country_id = $1 17 | ORDER BY domain_view_list.id 18 | LIMIT $2 OFFSET $3 19 | ` 20 | 21 | type AllDomainsByCountryParams struct { 22 | CountryID sql.NullInt64 23 | Limit int64 24 | Offset int64 25 | } 26 | 27 | func (q *Queries) AllDomainsByCountry(ctx context.Context, arg AllDomainsByCountryParams) ([]DomainViewList, error) { 28 | rows, err := q.db.Query(ctx, AllDomainsByCountry, arg.CountryID, arg.Limit, arg.Offset) 29 | if err != nil { 30 | return nil, err 31 | } 32 | defer rows.Close() 33 | items := []DomainViewList{} 34 | for rows.Next() { 35 | var i DomainViewList 36 | if err := rows.Scan( 37 | &i.ID, 38 | &i.Site, 39 | &i.BaseDomain, 40 | &i.WwwDomain, 41 | &i.Nameserver, 42 | &i.MxRecord, 43 | &i.V6Only, 44 | &i.AsnID, 45 | &i.CountryID, 46 | &i.Disabled, 47 | &i.TsBaseDomain, 48 | &i.TsWwwDomain, 49 | &i.TsNameserver, 50 | &i.TsMxRecord, 51 | &i.TsV6Only, 52 | &i.TsCheck, 53 | &i.TsUpdated, 54 | &i.Rank, 55 | &i.Asname, 56 | &i.CountryName, 57 | ); err != nil { 58 | return nil, err 59 | } 60 | items = append(items, i) 61 | } 62 | if err := rows.Err(); err != nil { 63 | return nil, err 64 | } 65 | return items, nil 66 | } 67 | 68 | const GetCountry = `-- name: GetCountry :one 69 | SELECT id, country_name, country_code, country_tld, continent, sites, v6sites, percent 70 | FROM country 71 | WHERE country_code = $1 72 | LIMIT 1 73 | ` 74 | 75 | func (q *Queries) GetCountry(ctx context.Context, countryCode string) (Country, error) { 76 | row := q.db.QueryRow(ctx, GetCountry, countryCode) 77 | var i Country 78 | err := row.Scan( 79 | &i.ID, 80 | &i.CountryName, 81 | &i.CountryCode, 82 | &i.CountryTld, 83 | &i.Continent, 84 | &i.Sites, 85 | &i.V6sites, 86 | &i.Percent, 87 | ) 88 | return i, err 89 | } 90 | 91 | const GetCountryTld = `-- name: GetCountryTld :one 92 | SELECT id, country_name, country_code, country_tld, continent, sites, v6sites, percent 93 | FROM country 94 | WHERE country_tld = $1 95 | LIMIT 1 96 | ` 97 | 98 | func (q *Queries) GetCountryTld(ctx context.Context, countryTld string) (Country, error) { 99 | row := q.db.QueryRow(ctx, GetCountryTld, countryTld) 100 | var i Country 101 | err := row.Scan( 102 | &i.ID, 103 | &i.CountryName, 104 | &i.CountryCode, 105 | &i.CountryTld, 106 | &i.Continent, 107 | &i.Sites, 108 | &i.V6sites, 109 | &i.Percent, 110 | ) 111 | return i, err 112 | } 113 | 114 | const ListCountry = `-- name: ListCountry :many 115 | SELECT id, country_name, country_code, country_tld, continent, sites, v6sites, percent 116 | FROM country 117 | ORDER BY sites DESC 118 | ` 119 | 120 | func (q *Queries) ListCountry(ctx context.Context) ([]Country, error) { 121 | rows, err := q.db.Query(ctx, ListCountry) 122 | if err != nil { 123 | return nil, err 124 | } 125 | defer rows.Close() 126 | items := []Country{} 127 | for rows.Next() { 128 | var i Country 129 | if err := rows.Scan( 130 | &i.ID, 131 | &i.CountryName, 132 | &i.CountryCode, 133 | &i.CountryTld, 134 | &i.Continent, 135 | &i.Sites, 136 | &i.V6sites, 137 | &i.Percent, 138 | ); err != nil { 139 | return nil, err 140 | } 141 | items = append(items, i) 142 | } 143 | if err := rows.Err(); err != nil { 144 | return nil, err 145 | } 146 | return items, nil 147 | } 148 | 149 | const ListDomainHeroesByCountry = `-- name: ListDomainHeroesByCountry :many 150 | SELECT id, site, base_domain, www_domain, nameserver, mx_record, v6_only, asn_id, country_id, disabled, ts_base_domain, ts_www_domain, ts_nameserver, ts_mx_record, ts_v6_only, ts_check, ts_updated, rank, asname, country_name 151 | FROM domain_view_list 152 | WHERE country_id = $1 153 | AND base_domain = 'supported' 154 | AND www_domain = 'supported' 155 | AND nameserver = 'supported' 156 | AND mx_record != 'unsupported' 157 | ORDER BY rank 158 | LIMIT $2 OFFSET $3 159 | ` 160 | 161 | type ListDomainHeroesByCountryParams struct { 162 | CountryID sql.NullInt64 163 | Limit int64 164 | Offset int64 165 | } 166 | 167 | func (q *Queries) ListDomainHeroesByCountry(ctx context.Context, arg ListDomainHeroesByCountryParams) ([]DomainViewList, error) { 168 | rows, err := q.db.Query(ctx, ListDomainHeroesByCountry, arg.CountryID, arg.Limit, arg.Offset) 169 | if err != nil { 170 | return nil, err 171 | } 172 | defer rows.Close() 173 | items := []DomainViewList{} 174 | for rows.Next() { 175 | var i DomainViewList 176 | if err := rows.Scan( 177 | &i.ID, 178 | &i.Site, 179 | &i.BaseDomain, 180 | &i.WwwDomain, 181 | &i.Nameserver, 182 | &i.MxRecord, 183 | &i.V6Only, 184 | &i.AsnID, 185 | &i.CountryID, 186 | &i.Disabled, 187 | &i.TsBaseDomain, 188 | &i.TsWwwDomain, 189 | &i.TsNameserver, 190 | &i.TsMxRecord, 191 | &i.TsV6Only, 192 | &i.TsCheck, 193 | &i.TsUpdated, 194 | &i.Rank, 195 | &i.Asname, 196 | &i.CountryName, 197 | ); err != nil { 198 | return nil, err 199 | } 200 | items = append(items, i) 201 | } 202 | if err := rows.Err(); err != nil { 203 | return nil, err 204 | } 205 | return items, nil 206 | } 207 | 208 | const ListDomainsByCountry = `-- name: ListDomainsByCountry :many 209 | SELECT id, site, base_domain, www_domain, nameserver, mx_record, v6_only, asn_id, country_id, disabled, ts_base_domain, ts_www_domain, ts_nameserver, ts_mx_record, ts_v6_only, ts_check, ts_updated, rank, asname, country_name 210 | FROM domain_view_list 211 | WHERE domain_view_list.country_id = $1 212 | AND ( 213 | domain_view_list.base_domain = 'unsupported' 214 | OR domain_view_list.www_domain = 'unsupported' 215 | ) 216 | ORDER BY domain_view_list.id 217 | LIMIT $2 OFFSET $3 218 | ` 219 | 220 | type ListDomainsByCountryParams struct { 221 | CountryID sql.NullInt64 222 | Limit int64 223 | Offset int64 224 | } 225 | 226 | func (q *Queries) ListDomainsByCountry(ctx context.Context, arg ListDomainsByCountryParams) ([]DomainViewList, error) { 227 | rows, err := q.db.Query(ctx, ListDomainsByCountry, arg.CountryID, arg.Limit, arg.Offset) 228 | if err != nil { 229 | return nil, err 230 | } 231 | defer rows.Close() 232 | items := []DomainViewList{} 233 | for rows.Next() { 234 | var i DomainViewList 235 | if err := rows.Scan( 236 | &i.ID, 237 | &i.Site, 238 | &i.BaseDomain, 239 | &i.WwwDomain, 240 | &i.Nameserver, 241 | &i.MxRecord, 242 | &i.V6Only, 243 | &i.AsnID, 244 | &i.CountryID, 245 | &i.Disabled, 246 | &i.TsBaseDomain, 247 | &i.TsWwwDomain, 248 | &i.TsNameserver, 249 | &i.TsMxRecord, 250 | &i.TsV6Only, 251 | &i.TsCheck, 252 | &i.TsUpdated, 253 | &i.Rank, 254 | &i.Asname, 255 | &i.CountryName, 256 | ); err != nil { 257 | return nil, err 258 | } 259 | items = append(items, i) 260 | } 261 | if err := rows.Err(); err != nil { 262 | return nil, err 263 | } 264 | return items, nil 265 | } 266 | -------------------------------------------------------------------------------- /internal/postgres/db/db.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | 5 | package db 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/jackc/pgconn" 11 | "github.com/jackc/pgx/v4" 12 | ) 13 | 14 | type DBTX interface { 15 | Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) 16 | Query(context.Context, string, ...interface{}) (pgx.Rows, error) 17 | QueryRow(context.Context, string, ...interface{}) pgx.Row 18 | } 19 | 20 | func New(db DBTX) *Queries { 21 | return &Queries{db: db} 22 | } 23 | 24 | type Queries struct { 25 | db DBTX 26 | } 27 | 28 | func (q *Queries) WithTx(tx pgx.Tx) *Queries { 29 | return &Queries{ 30 | db: tx, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/postgres/db/metrics.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: metrics.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | "time" 11 | 12 | "github.com/jackc/pgtype" 13 | ) 14 | 15 | const DomainStats = `-- name: DomainStats :many 16 | SELECT time, 17 | data 18 | FROM metrics 19 | WHERE measurement = 'domains' 20 | ORDER BY time DESC 21 | LIMIT 1 22 | ` 23 | 24 | type DomainStatsRow struct { 25 | Time time.Time 26 | Data pgtype.JSONB 27 | } 28 | 29 | func (q *Queries) DomainStats(ctx context.Context) ([]DomainStatsRow, error) { 30 | rows, err := q.db.Query(ctx, DomainStats) 31 | if err != nil { 32 | return nil, err 33 | } 34 | defer rows.Close() 35 | items := []DomainStatsRow{} 36 | for rows.Next() { 37 | var i DomainStatsRow 38 | if err := rows.Scan(&i.Time, &i.Data); err != nil { 39 | return nil, err 40 | } 41 | items = append(items, i) 42 | } 43 | if err := rows.Err(); err != nil { 44 | return nil, err 45 | } 46 | return items, nil 47 | } 48 | 49 | const GetMetric = `-- name: GetMetric :many 50 | SELECT time, 51 | data 52 | FROM metrics 53 | WHERE measurement = $1 54 | ORDER BY time DESC 55 | ` 56 | 57 | type GetMetricRow struct { 58 | Time time.Time 59 | Data pgtype.JSONB 60 | } 61 | 62 | func (q *Queries) GetMetric(ctx context.Context, measurement string) ([]GetMetricRow, error) { 63 | rows, err := q.db.Query(ctx, GetMetric, measurement) 64 | if err != nil { 65 | return nil, err 66 | } 67 | defer rows.Close() 68 | items := []GetMetricRow{} 69 | for rows.Next() { 70 | var i GetMetricRow 71 | if err := rows.Scan(&i.Time, &i.Data); err != nil { 72 | return nil, err 73 | } 74 | items = append(items, i) 75 | } 76 | if err := rows.Err(); err != nil { 77 | return nil, err 78 | } 79 | return items, nil 80 | } 81 | 82 | const StoreMetric = `-- name: StoreMetric :exec 83 | INSERT INTO metrics(measurement, data) 84 | VALUES ($1, $2) 85 | RETURNING id, measurement, time, data 86 | ` 87 | 88 | type StoreMetricParams struct { 89 | Measurement string 90 | Data pgtype.JSONB 91 | } 92 | 93 | func (q *Queries) StoreMetric(ctx context.Context, arg StoreMetricParams) error { 94 | _, err := q.db.Exec(ctx, StoreMetric, arg.Measurement, arg.Data) 95 | return err 96 | } 97 | -------------------------------------------------------------------------------- /internal/postgres/db/models.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | 5 | package db 6 | 7 | import ( 8 | "database/sql" 9 | "database/sql/driver" 10 | "fmt" 11 | "time" 12 | 13 | "github.com/google/uuid" 14 | "github.com/jackc/pgtype" 15 | ) 16 | 17 | type Continents string 18 | 19 | const ( 20 | ContinentsAfrica Continents = "Africa" 21 | ContinentsAntarctica Continents = "Antarctica" 22 | ContinentsAsia Continents = "Asia" 23 | ContinentsEurope Continents = "Europe" 24 | ContinentsOceania Continents = "Oceania" 25 | ContinentsNorthAmerica Continents = "North America" 26 | ContinentsSouthAmerica Continents = "South America" 27 | ) 28 | 29 | func (e *Continents) Scan(src interface{}) error { 30 | switch s := src.(type) { 31 | case []byte: 32 | *e = Continents(s) 33 | case string: 34 | *e = Continents(s) 35 | default: 36 | return fmt.Errorf("unsupported scan type for Continents: %T", src) 37 | } 38 | return nil 39 | } 40 | 41 | type NullContinents struct { 42 | Continents Continents 43 | Valid bool // Valid is true if Continents is not NULL 44 | } 45 | 46 | // Scan implements the Scanner interface. 47 | func (ns *NullContinents) Scan(value interface{}) error { 48 | if value == nil { 49 | ns.Continents, ns.Valid = "", false 50 | return nil 51 | } 52 | ns.Valid = true 53 | return ns.Continents.Scan(value) 54 | } 55 | 56 | // Value implements the driver Valuer interface. 57 | func (ns NullContinents) Value() (driver.Value, error) { 58 | if !ns.Valid { 59 | return nil, nil 60 | } 61 | return string(ns.Continents), nil 62 | } 63 | 64 | type Asn struct { 65 | ID int64 66 | Number int32 67 | Name string 68 | CountV4 sql.NullInt32 69 | CountV6 sql.NullInt32 70 | PercentV4 sql.NullFloat64 71 | PercentV6 sql.NullFloat64 72 | } 73 | 74 | type Campaign struct { 75 | ID int64 76 | CreatedAt time.Time 77 | Uuid uuid.UUID 78 | Name string 79 | Description string 80 | Disabled bool 81 | } 82 | 83 | type CampaignChangelog struct { 84 | ID int64 85 | Ts time.Time 86 | DomainID int64 87 | CampaignID uuid.UUID 88 | Message string 89 | Ipv6Status string 90 | } 91 | 92 | type CampaignDomain struct { 93 | ID int64 94 | CampaignID uuid.UUID 95 | Site string 96 | BaseDomain string 97 | WwwDomain string 98 | Nameserver string 99 | MxRecord string 100 | V6Only string 101 | AsnID sql.NullInt64 102 | CountryID sql.NullInt64 103 | Disabled bool 104 | TsBaseDomain sql.NullTime 105 | TsWwwDomain sql.NullTime 106 | TsNameserver sql.NullTime 107 | TsMxRecord sql.NullTime 108 | TsV6Only sql.NullTime 109 | TsCheck sql.NullTime 110 | TsUpdated sql.NullTime 111 | } 112 | 113 | type CampaignDomainLog struct { 114 | ID int64 115 | DomainID int64 116 | Time time.Time 117 | Data pgtype.JSONB 118 | } 119 | 120 | type Changelog struct { 121 | ID int64 122 | Ts time.Time 123 | DomainID int64 124 | Message string 125 | Ipv6Status string 126 | } 127 | 128 | type ChangelogCampaignView struct { 129 | ID int64 130 | Ts time.Time 131 | DomainID int64 132 | CampaignID uuid.UUID 133 | Message string 134 | Ipv6Status string 135 | Site string 136 | } 137 | 138 | type ChangelogView struct { 139 | ID int64 140 | Ts time.Time 141 | DomainID int64 142 | Message string 143 | Ipv6Status string 144 | Site string 145 | } 146 | 147 | type Country struct { 148 | ID int64 149 | CountryName string 150 | CountryCode string 151 | CountryTld string 152 | Continent NullContinents 153 | Sites int32 154 | V6sites int32 155 | Percent pgtype.Numeric 156 | } 157 | 158 | type Domain struct { 159 | ID int64 160 | Site string 161 | BaseDomain string 162 | WwwDomain string 163 | Nameserver string 164 | MxRecord string 165 | V6Only string 166 | AsnID sql.NullInt64 167 | CountryID sql.NullInt64 168 | Disabled bool 169 | TsBaseDomain sql.NullTime 170 | TsWwwDomain sql.NullTime 171 | TsNameserver sql.NullTime 172 | TsMxRecord sql.NullTime 173 | TsV6Only sql.NullTime 174 | TsCheck sql.NullTime 175 | TsUpdated sql.NullTime 176 | } 177 | 178 | type DomainCrawlList struct { 179 | ID int64 180 | Site string 181 | BaseDomain string 182 | WwwDomain string 183 | Nameserver string 184 | MxRecord string 185 | V6Only string 186 | AsnID sql.NullInt64 187 | CountryID sql.NullInt64 188 | Disabled bool 189 | TsBaseDomain sql.NullTime 190 | TsWwwDomain sql.NullTime 191 | TsNameserver sql.NullTime 192 | TsMxRecord sql.NullTime 193 | TsV6Only sql.NullTime 194 | TsCheck sql.NullTime 195 | TsUpdated sql.NullTime 196 | } 197 | 198 | type DomainLog struct { 199 | ID int64 200 | DomainID int64 201 | Time time.Time 202 | Data pgtype.JSONB 203 | } 204 | 205 | type DomainShameView struct { 206 | ID int64 207 | Site string 208 | BaseDomain string 209 | WwwDomain string 210 | Nameserver string 211 | MxRecord string 212 | V6Only string 213 | AsnID sql.NullInt64 214 | CountryID sql.NullInt64 215 | Disabled bool 216 | TsBaseDomain sql.NullTime 217 | TsWwwDomain sql.NullTime 218 | TsNameserver sql.NullTime 219 | TsMxRecord sql.NullTime 220 | TsV6Only sql.NullTime 221 | TsCheck sql.NullTime 222 | TsUpdated sql.NullTime 223 | ShameID int64 224 | ShameSite string 225 | } 226 | 227 | type DomainViewList struct { 228 | ID sql.NullInt64 229 | Site sql.NullString 230 | BaseDomain sql.NullString 231 | WwwDomain sql.NullString 232 | Nameserver sql.NullString 233 | MxRecord sql.NullString 234 | V6Only sql.NullString 235 | AsnID sql.NullInt64 236 | CountryID sql.NullInt64 237 | Disabled sql.NullBool 238 | TsBaseDomain sql.NullTime 239 | TsWwwDomain sql.NullTime 240 | TsNameserver sql.NullTime 241 | TsMxRecord sql.NullTime 242 | TsV6Only sql.NullTime 243 | TsCheck sql.NullTime 244 | TsUpdated sql.NullTime 245 | Rank int64 246 | Asname sql.NullString 247 | CountryName sql.NullString 248 | } 249 | 250 | type Lists struct { 251 | ID int64 252 | Name string 253 | Ts time.Time 254 | } 255 | 256 | type Metrics struct { 257 | ID int64 258 | Measurement string 259 | Time time.Time 260 | Data pgtype.JSONB 261 | } 262 | 263 | type Sites struct { 264 | ID int64 265 | ListID int64 266 | Rank int64 267 | Site string 268 | } 269 | 270 | type TopShame struct { 271 | ID int64 272 | Site string 273 | } 274 | -------------------------------------------------------------------------------- /internal/postgres/db/sites.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: sites.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | ) 11 | 12 | const ListSites = `-- name: ListSites :many 13 | SELECT id, list_id, rank, site 14 | FROM sites 15 | ORDER BY rank 16 | LIMIT $1 OFFSET $2 17 | ` 18 | 19 | type ListSitesParams struct { 20 | Limit int64 21 | Offset int64 22 | } 23 | 24 | func (q *Queries) ListSites(ctx context.Context, arg ListSitesParams) ([]Sites, error) { 25 | rows, err := q.db.Query(ctx, ListSites, arg.Limit, arg.Offset) 26 | if err != nil { 27 | return nil, err 28 | } 29 | defer rows.Close() 30 | items := []Sites{} 31 | for rows.Next() { 32 | var i Sites 33 | if err := rows.Scan( 34 | &i.ID, 35 | &i.ListID, 36 | &i.Rank, 37 | &i.Site, 38 | ); err != nil { 39 | return nil, err 40 | } 41 | items = append(items, i) 42 | } 43 | if err := rows.Err(); err != nil { 44 | return nil, err 45 | } 46 | return items, nil 47 | } 48 | -------------------------------------------------------------------------------- /internal/postgres/db/stats.sql.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.27.0 4 | // source: stats.sql 5 | 6 | package db 7 | 8 | import ( 9 | "context" 10 | ) 11 | 12 | const CalculateASNStats = `-- name: CalculateASNStats :exec 13 | SELECT update_asn_metrics() 14 | ` 15 | 16 | func (q *Queries) CalculateASNStats(ctx context.Context) error { 17 | _, err := q.db.Exec(ctx, CalculateASNStats) 18 | return err 19 | } 20 | 21 | const CalculateCountryStats = `-- name: CalculateCountryStats :exec 22 | SELECT update_country_metrics() 23 | ` 24 | 25 | func (q *Queries) CalculateCountryStats(ctx context.Context) error { 26 | _, err := q.db.Exec(ctx, CalculateCountryStats) 27 | return err 28 | } 29 | 30 | const CrawlerStats = `-- name: CrawlerStats :one 31 | SELECT 32 | count(1) AS "domains", 33 | count(1) filter (WHERE base_domain = 'supported') AS "base_domain", 34 | count(1) filter (WHERE www_domain = 'supported') AS "www_domain", 35 | count(1) filter (WHERE nameserver = 'supported') AS "nameserver", 36 | count(1) filter (WHERE mx_record = 'supported') AS "mx_record", 37 | count(1) filter (WHERE base_domain = 'supported' AND www_domain = 'supported') AS "heroes", 38 | count(1) filter (WHERE base_domain != 'unsupported' AND www_domain != 'unsupported' AND rank < 1000) AS "top_heroes", 39 | count(1) filter (WHERE nameserver = 'supported' AND rank < 1000) AS "top_nameserver" 40 | FROM domain_view_list 41 | ` 42 | 43 | type CrawlerStatsRow struct { 44 | Domains int64 45 | BaseDomain int64 46 | WwwDomain int64 47 | Nameserver int64 48 | MxRecord int64 49 | Heroes int64 50 | TopHeroes int64 51 | TopNameserver int64 52 | } 53 | 54 | // Used by the crawler to store total stats in the metric table 55 | func (q *Queries) CrawlerStats(ctx context.Context) (CrawlerStatsRow, error) { 56 | row := q.db.QueryRow(ctx, CrawlerStats) 57 | var i CrawlerStatsRow 58 | err := row.Scan( 59 | &i.Domains, 60 | &i.BaseDomain, 61 | &i.WwwDomain, 62 | &i.Nameserver, 63 | &i.MxRecord, 64 | &i.Heroes, 65 | &i.TopHeroes, 66 | &i.TopNameserver, 67 | ) 68 | return i, err 69 | } 70 | -------------------------------------------------------------------------------- /internal/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log/slog" 7 | "time" 8 | 9 | "github.com/jackc/pgx/v4/pgxpool" 10 | ) 11 | 12 | // NewPostgreSQL creates a new PostgreSQL connection pool with retries and timeout. 13 | // It takes the connection string and returns a pool or an error. 14 | func NewPostgreSQL(conf string, maxRetries int, timeout time.Duration) (*pgxpool.Pool, error) { 15 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 16 | defer cancel() 17 | 18 | var pool *pgxpool.Pool 19 | var err error 20 | 21 | for i := range maxRetries { 22 | slog.Info("Attempting to connect to PostgreSQL", "attempt", i+1) 23 | pool, err = pgxpool.Connect(ctx, conf) 24 | if err != nil { 25 | slog.Warn("Failed to connect to PostgreSQL", "error", err, "attempt", i+1) 26 | if i == maxRetries-1 { 27 | break 28 | } 29 | time.Sleep(2 * time.Second) 30 | continue 31 | } 32 | 33 | // Test the connection 34 | if pingErr := pool.Ping(ctx); pingErr != nil { 35 | slog.Warn("Failed to ping PostgreSQL", "error", pingErr, "attempt", i+1) 36 | if i == maxRetries-1 { 37 | err = pingErr 38 | break 39 | } 40 | pool.Close() 41 | time.Sleep(2 * time.Second) 42 | continue 43 | } 44 | 45 | slog.Info("Successfully connected to PostgreSQL") 46 | return pool, nil 47 | } 48 | 49 | slog.Error("Exhausted retries for PostgreSQL connection", "error", err) 50 | return nil, errors.New("failed to connect to PostgreSQL after retries") 51 | } 52 | -------------------------------------------------------------------------------- /internal/rest/campaign.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | "strings" 7 | "time" 8 | 9 | "whynoipv6/internal/core" 10 | 11 | "github.com/ggicci/httpin" 12 | "github.com/go-chi/chi/v5" 13 | "github.com/go-chi/render" 14 | "github.com/google/uuid" 15 | ) 16 | 17 | // CampaignHandler is a handler for domain endpoints. 18 | type CampaignHandler struct { 19 | Repo *core.CampaignService 20 | } 21 | 22 | // CampaignResponse is the response for a domain. 23 | type CampaignResponse struct { 24 | Domain string `json:"domain"` 25 | BaseDomain string `json:"base_domain"` 26 | WwwDomain string `json:"www_domain"` 27 | Nameserver string `json:"nameserver"` 28 | MXRecord string `json:"mx_record"` 29 | V6Only string `json:"v6_only"` 30 | AsName string `json:"asn"` 31 | Country string `json:"country"` 32 | TsBaseDomain time.Time `json:"ts_aaaa"` 33 | TsWwwDomain time.Time `json:"ts_www"` 34 | TsNameserver time.Time `json:"ts_ns"` 35 | TsMXRecord time.Time `json:"ts_mx"` 36 | TsV6Only time.Time `json:"ts_curl"` 37 | TsCheck time.Time `json:"ts_check"` 38 | TsUpdated time.Time `json:"ts_updated"` 39 | } 40 | 41 | // CampaignListResponse represents a campaign. 42 | type CampaignListResponse struct { 43 | ID int64 `json:"id"` 44 | UUID string `json:"uuid"` 45 | Name string `json:"name"` 46 | Description string `json:"description"` 47 | Count int64 `json:"count"` 48 | V6Ready int64 `json:"v6_ready"` 49 | } 50 | 51 | // CampaignDomainLogResponse is the response structure for a domain log. 52 | type CampaignDomainLogResponse struct { 53 | ID int64 `json:"id"` 54 | Time time.Time `json:"time"` 55 | BaseDomain string `json:"base_domain"` 56 | WwwDomain string `json:"www_domain"` 57 | Nameserver string `json:"nameserver"` 58 | MXRecord string `json:"mx_record"` 59 | } 60 | 61 | // Routes returns a router with all campaign endpoints mounted. 62 | func (rs CampaignHandler) Routes() chi.Router { 63 | r := chi.NewRouter() 64 | 65 | // GET /campaign - List all campaigns 66 | r.Get("/", rs.CampaignList) 67 | // GET /campaign/{uuid} - List all domains for a given campaign UUID 68 | r.With(httpin.NewInput(PaginationInput{})).Get("/{uuid}", rs.CampaignDomains) 69 | // GET /campaign/{campaign}/{domain} - View details of a single domain in a campaign 70 | r.Get("/{uuid}/{domain}", rs.ViewCampaignDomain) 71 | // GET /campaign/{campaign}/{domain}/log - View crawler of a single domain in a campaign 72 | r.Get("/{uuid}/{domain}/log", rs.GetCampaignDomainLog) 73 | // GET /campaign/search/{domain} - search for a domain by its name 74 | r.With(httpin.NewInput(PaginationInput{})).Get("/search/{domain}", rs.SearchDomain) 75 | 76 | return r 77 | } 78 | 79 | // CampaignList retrieves and lists all campaigns. 80 | func (rs CampaignHandler) CampaignList(w http.ResponseWriter, r *http.Request) { 81 | // Retrieve all campaigns from the repository 82 | allCampaigns, err := rs.Repo.ListCampaign(r.Context()) 83 | if err != nil { 84 | render.Status(r, http.StatusInternalServerError) 85 | render.JSON(w, r, render.M{"error": "Internal server error"}) 86 | return 87 | } 88 | 89 | // Prepare the response with campaign details 90 | var campaignList []CampaignListResponse 91 | for _, campaign := range allCampaigns { 92 | campaignList = append(campaignList, CampaignListResponse{ 93 | ID: campaign.ID, 94 | UUID: encodeUUID(campaign.UUID), 95 | Name: campaign.Name, 96 | Description: campaign.Description, 97 | Count: campaign.Count, 98 | V6Ready: campaign.V6Ready, 99 | }) 100 | } 101 | 102 | // Send campaign list as JSON response 103 | render.JSON(w, r, campaignList) 104 | } 105 | 106 | // CampaignDomains lists all domains in a campaign. 107 | func (rs CampaignHandler) CampaignDomains(w http.ResponseWriter, r *http.Request) { 108 | // Handle query params 109 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 110 | if paginationInput.Limit > 100 { 111 | paginationInput.Limit = 100 112 | } 113 | 114 | // Get campaign UUID from path 115 | campaignUUID := chi.URLParam(r, "uuid") 116 | // Decode uuid from shortuuid to google uuid 117 | decodeID, err := decodeUUID(campaignUUID) 118 | if err != nil { 119 | render.Status(r, http.StatusNotFound) 120 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 121 | return 122 | } 123 | // Convert campaignUUID to uuid.UUID 124 | parsedUUID, err := uuid.Parse(decodeID.String()) 125 | if err != nil { 126 | render.Status(r, http.StatusNotFound) 127 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 128 | return 129 | } 130 | 131 | // Retrieve campaign details from the repository 132 | campaignDetails, err := rs.Repo.GetCampaign(r.Context(), parsedUUID) 133 | if err != nil { 134 | render.Status(r, http.StatusNotFound) 135 | render.JSON(w, r, render.M{"error": "Campaign not found"}) 136 | return 137 | } 138 | 139 | // Retrieve domains associated with the campaign 140 | domains, err := rs.Repo.ListCampaignDomain( 141 | r.Context(), 142 | parsedUUID, 143 | paginationInput.Offset, 144 | paginationInput.Limit, 145 | ) 146 | if err != nil { 147 | render.Status(r, http.StatusNotFound) 148 | render.JSON(w, r, render.M{"error": "Campaign not found"}) 149 | return 150 | } 151 | if len(domains) == 0 { 152 | render.Status(r, http.StatusNotFound) 153 | render.JSON(w, r, render.M{"error": "Campaign not found"}) 154 | return 155 | } 156 | 157 | var domainList []CampaignResponse 158 | for _, domain := range domains { 159 | domainList = append(domainList, CampaignResponse{ 160 | Domain: domain.Site, 161 | BaseDomain: domain.BaseDomain, 162 | WwwDomain: domain.WwwDomain, 163 | Nameserver: domain.Nameserver, 164 | MXRecord: domain.MXRecord, 165 | V6Only: domain.V6Only, 166 | AsName: domain.AsName, 167 | Country: domain.Country, 168 | TsBaseDomain: domain.TsBaseDomain, 169 | TsWwwDomain: domain.TsWwwDomain, 170 | TsNameserver: domain.TsNameserver, 171 | TsMXRecord: domain.TsMXRecord, 172 | TsV6Only: domain.TsV6Only, 173 | TsCheck: domain.TsCheck, 174 | TsUpdated: domain.TsUpdated, 175 | }) 176 | } 177 | 178 | // Connect campaign details with domain list 179 | campaignList := struct { 180 | Campaign CampaignListResponse `json:"campaign"` 181 | Domains []CampaignResponse `json:"domains"` 182 | }{ 183 | Campaign: CampaignListResponse{ 184 | ID: campaignDetails.ID, 185 | UUID: encodeUUID(campaignDetails.UUID), 186 | Name: campaignDetails.Name, 187 | Description: campaignDetails.Description, 188 | Count: campaignDetails.Count, 189 | V6Ready: campaignDetails.V6Ready, 190 | }, 191 | Domains: domainList, 192 | } 193 | 194 | render.JSON(w, r, campaignList) 195 | } 196 | 197 | // ViewCampaignDomain retrives a single domain in a campaign. 198 | func (rs CampaignHandler) ViewCampaignDomain(w http.ResponseWriter, r *http.Request) { 199 | // Get campaign UUID and domain from path 200 | campaignUUID := chi.URLParam(r, "uuid") 201 | site := chi.URLParam(r, "domain") 202 | 203 | // Decode uuid from shortuuid to google uuid 204 | decodeID, err := decodeUUID(campaignUUID) 205 | if err != nil { 206 | render.Status(r, http.StatusBadRequest) 207 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 208 | return 209 | } 210 | // Validate and parse the UUID 211 | uuid, err := uuid.Parse(decodeID.String()) 212 | if err != nil { 213 | render.Status(r, http.StatusBadRequest) 214 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 215 | return 216 | } 217 | 218 | // Validate the domain 219 | // TODO: Move this to core package 220 | if !regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`).MatchString(site) { 221 | render.Status(r, http.StatusBadRequest) 222 | render.JSON(w, r, render.M{"error": "Invalid domain"}) 223 | return 224 | } 225 | 226 | // Retrieve domain details from the repository 227 | domainDetails, err := rs.Repo.ViewCampaignDomain(r.Context(), uuid, site) 228 | if err != nil { 229 | render.Status(r, http.StatusNotFound) 230 | render.JSON(w, r, render.M{"error": "Domain not found"}) 231 | return 232 | } 233 | 234 | // If no changelogs are found, return 404 235 | if len(domainDetails.Site) == 0 { 236 | render.Status(r, http.StatusNotFound) 237 | render.JSON( 238 | w, 239 | r, 240 | render.M{ 241 | "error": "No changelog entries found for campaign " + campaignUUID + " and domain " + site, 242 | }, 243 | ) 244 | return 245 | } 246 | 247 | // Send domain details as JSON response 248 | render.JSON(w, r, CampaignResponse{ 249 | Domain: domainDetails.Site, 250 | BaseDomain: domainDetails.BaseDomain, 251 | WwwDomain: domainDetails.WwwDomain, 252 | Nameserver: domainDetails.Nameserver, 253 | MXRecord: domainDetails.MXRecord, 254 | V6Only: domainDetails.V6Only, 255 | AsName: domainDetails.AsName, 256 | Country: domainDetails.Country, 257 | TsBaseDomain: domainDetails.TsBaseDomain, 258 | TsWwwDomain: domainDetails.TsWwwDomain, 259 | TsNameserver: domainDetails.TsNameserver, 260 | TsMXRecord: domainDetails.TsMXRecord, 261 | TsV6Only: domainDetails.TsV6Only, 262 | TsCheck: domainDetails.TsCheck, 263 | TsUpdated: domainDetails.TsUpdated, 264 | }) 265 | } 266 | 267 | // SearchDomain returns a domain based on the provided domain name. 268 | func (rs CampaignHandler) SearchDomain(w http.ResponseWriter, r *http.Request) { 269 | // Handle query params 270 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 271 | if paginationInput.Limit > 100 { 272 | paginationInput.Limit = 100 273 | } 274 | 275 | domain := chi.URLParam(r, "domain") 276 | 277 | // Search for campaign domains 278 | campaignDomains, err := rs.Repo.GetCampaignDomainsByName( 279 | r.Context(), 280 | strings.ToLower(domain), 281 | paginationInput.Offset, 282 | paginationInput.Limit, 283 | ) 284 | if err != nil { 285 | render.Status(r, http.StatusNotFound) 286 | render.JSON(w, r, render.M{"error": "No domains found"}) 287 | return 288 | } 289 | 290 | if len(campaignDomains) == 0 { 291 | render.Status(r, http.StatusNotFound) 292 | render.JSON(w, r, render.M{"error": "No domains found"}) 293 | return 294 | } 295 | 296 | var campaignDomainList []DomainResponse 297 | for _, domain := range campaignDomains { 298 | campaignDomainList = append(campaignDomainList, DomainResponse{ 299 | Domain: domain.Site, 300 | BaseDomain: domain.BaseDomain, 301 | WwwDomain: domain.WwwDomain, 302 | Nameserver: domain.Nameserver, 303 | MXRecord: domain.MXRecord, 304 | V6Only: domain.V6Only, 305 | AsName: domain.AsName, 306 | Country: domain.Country, 307 | TsBaseDomain: domain.TsBaseDomain, 308 | TsWwwDomain: domain.TsWwwDomain, 309 | TsNameserver: domain.TsNameserver, 310 | TsMXRecord: domain.TsMXRecord, 311 | TsV6Only: domain.TsV6Only, 312 | TsCheck: domain.TsCheck, 313 | TsUpdated: domain.TsUpdated, 314 | CampaignUUID: encodeUUID(domain.CampaignID), 315 | }) 316 | } 317 | 318 | render.JSON(w, r, render.M{ 319 | "data": campaignDomainList, 320 | }) 321 | } 322 | 323 | // GetCampaignDomainLog returns the crawler log for a domain. 324 | func (rs CampaignHandler) GetCampaignDomainLog(w http.ResponseWriter, r *http.Request) { 325 | // Get campaign UUID and domain from path 326 | campaignUUID := chi.URLParam(r, "uuid") 327 | domain := chi.URLParam(r, "domain") 328 | 329 | // Decode uuid from shortuuid to google uuid 330 | decodeID, err := decodeUUID(campaignUUID) 331 | if err != nil { 332 | render.Status(r, http.StatusBadRequest) 333 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 334 | return 335 | } 336 | // Validate and parse the UUID 337 | uuid, err := uuid.Parse(decodeID.String()) 338 | if err != nil { 339 | render.Status(r, http.StatusBadRequest) 340 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 341 | return 342 | } 343 | 344 | logs, err := rs.Repo.GetCampaignDomainLog(r.Context(), uuid, domain) 345 | if err != nil { 346 | render.Status(r, http.StatusNotFound) 347 | render.JSON(w, r, render.M{"error": "domain not found"}) 348 | return 349 | } 350 | var domainlist []CampaignDomainLogResponse 351 | for _, log := range logs { 352 | var data map[string]any 353 | if err := log.Data.AssignTo(&data); err != nil { 354 | render.Status(r, http.StatusInternalServerError) 355 | render.JSON(w, r, render.M{"error": "internal server error"}) 356 | return 357 | } 358 | domainlist = append(domainlist, CampaignDomainLogResponse{ 359 | ID: log.ID, 360 | Time: log.Time, 361 | BaseDomain: data["base_domain"].(string), 362 | WwwDomain: data["www_domain"].(string), 363 | Nameserver: data["nameserver"].(string), 364 | MXRecord: data["mx_record"].(string), 365 | }) 366 | } 367 | render.JSON(w, r, domainlist) 368 | } 369 | -------------------------------------------------------------------------------- /internal/rest/changelog.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | "time" 7 | 8 | "whynoipv6/internal/core" 9 | 10 | "github.com/ggicci/httpin" 11 | "github.com/go-chi/chi/v5" 12 | "github.com/go-chi/render" 13 | "github.com/google/uuid" 14 | ) 15 | 16 | // ChangelogHandler is a handler for changelog endpoints. 17 | type ChangelogHandler struct { 18 | Repo *core.ChangelogService 19 | } 20 | 21 | // ChangelogResponse is the response for a changelog. 22 | type ChangelogResponse struct { 23 | ID int64 `json:"id"` 24 | Ts time.Time `json:"ts"` 25 | Domain string `json:"domain"` 26 | DomainURL string `json:"domain_url"` 27 | Message string `json:"message"` 28 | IPv6Status string `json:"ipv6_status"` 29 | } 30 | 31 | // Routes returns a router with all changelog endpoints mounted. 32 | func (rs ChangelogHandler) Routes() chi.Router { 33 | r := chi.NewRouter() 34 | 35 | // GET /changelog - List all changelog entries 36 | r.With(httpin.NewInput(PaginationInput{})).Get("/", rs.ChangelogList) 37 | // GET /changelog/campaign - List all campaign changelog entries 38 | r.With(httpin.NewInput(PaginationInput{})).Get("/campaign", rs.CampaignChangelogList) 39 | // GET /changelog/{domain} - List all changelog entries for a specific domain 40 | r.With(httpin.NewInput(PaginationInput{})).Get("/{domain}", rs.ChangelogByDomain) 41 | // GET /changelog/campaign/{uuid} - List all changelog entries for a specific campaign UUID 42 | r.With(httpin.NewInput(PaginationInput{})).Get("/campaign/{uuid}", rs.ChangelogByCampaign) 43 | // GET /changelog/campaign/{uuid}/{domain} - List all changelog entries for a specific domain within a campaign UUID 44 | r.With(httpin.NewInput(PaginationInput{})). 45 | Get("/campaign/{uuid}/{domain}", rs.ChangelogByCampaignDomain) 46 | 47 | return r 48 | } 49 | 50 | // ChangelogList lists all changelog entries with pagination. 51 | func (rs ChangelogHandler) ChangelogList(w http.ResponseWriter, r *http.Request) { 52 | // Retrieve pagination input from context 53 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 54 | 55 | // Limit the maximum number of entries per page to 100 56 | if paginationInput.Limit > 100 { 57 | paginationInput.Limit = 100 58 | } 59 | 60 | // Fetch changelogs from the repository 61 | changelogs, err := rs.Repo.List(r.Context(), paginationInput.Offset, paginationInput.Limit) 62 | if err != nil { 63 | // Handle any errors while fetching changelogs 64 | render.Status(r, http.StatusNotFound) 65 | render.JSON(w, r, render.M{"error": "No changelog entries found"}) 66 | return 67 | } 68 | 69 | // Convert changelogs to ChangelogResponse objects 70 | var changelogList []ChangelogResponse 71 | for _, changelog := range changelogs { 72 | changelogList = append(changelogList, ChangelogResponse{ 73 | ID: changelog.ID, 74 | Ts: changelog.Ts, 75 | Domain: changelog.Site, 76 | DomainURL: "/domain/" + changelog.Site, 77 | Message: changelog.Message, 78 | IPv6Status: changelog.IPv6Status, 79 | }) 80 | } 81 | 82 | // Send the changelog list as JSON 83 | render.JSON(w, r, changelogList) 84 | } 85 | 86 | // CampaignChangelogList lists all changelog entries with pagination. 87 | func (rs ChangelogHandler) CampaignChangelogList(w http.ResponseWriter, r *http.Request) { 88 | // Retrieve pagination input from context 89 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 90 | 91 | // Limit the maximum number of entries per page to 100 92 | if paginationInput.Limit > 100 { 93 | paginationInput.Limit = 100 94 | } 95 | 96 | // Fetch changelogs from the repository 97 | changelogs, err := rs.Repo.CampaignList( 98 | r.Context(), 99 | paginationInput.Offset, 100 | paginationInput.Limit, 101 | ) 102 | if err != nil { 103 | // Handle any errors while fetching changelogs 104 | render.Status(r, http.StatusNotFound) 105 | render.JSON(w, r, render.M{"error": "No changelog entries found for campaigns"}) 106 | return 107 | } 108 | 109 | // Convert changelogs to ChangelogResponse objects 110 | var changelogList []ChangelogResponse 111 | for _, changelog := range changelogs { 112 | changelogList = append(changelogList, ChangelogResponse{ 113 | ID: changelog.ID, 114 | Ts: changelog.Ts, 115 | Domain: changelog.Site, 116 | DomainURL: "/campaign/" + encodeUUID(changelog.CampaignID) + "/" + changelog.Site, 117 | Message: changelog.Message, 118 | IPv6Status: changelog.IPv6Status, 119 | }) 120 | } 121 | 122 | // Send the changelog list as JSON 123 | render.JSON(w, r, changelogList) 124 | } 125 | 126 | // ChangelogByDomain lists all changelog entries for a specific domain with pagination. 127 | func (rs ChangelogHandler) ChangelogByDomain(w http.ResponseWriter, r *http.Request) { 128 | // Retrieve pagination input from context 129 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 130 | 131 | // Limit the maximum number of entries per page to 100 132 | if paginationInput.Limit > 100 { 133 | paginationInput.Limit = 100 134 | } 135 | 136 | // Get domain from path 137 | site := chi.URLParam(r, "domain") 138 | 139 | // Validate domain 140 | // TODO: Move this to core package 141 | if !regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`).MatchString(site) { 142 | render.Status(r, http.StatusBadRequest) 143 | render.JSON(w, r, render.M{"error": "Invalid domain"}) 144 | return 145 | } 146 | 147 | // Fetch changelogs for the domain from the repository 148 | changelogs, err := rs.Repo.GetChangelogByDomain( 149 | r.Context(), 150 | site, 151 | paginationInput.Offset, 152 | paginationInput.Limit, 153 | ) 154 | if err != nil { 155 | render.Status(r, http.StatusNotFound) 156 | render.JSON(w, r, render.M{"error": "Unable to find changelog entries for " + site}) 157 | return 158 | } 159 | 160 | // If no changelogs are found, return 404 161 | if len(changelogs) == 0 { 162 | render.Status(r, http.StatusNotFound) 163 | render.JSON(w, r, render.M{"error": "No changelog entries found for " + site}) 164 | return 165 | } 166 | 167 | // Convert changelogs to ChangelogResponse objects 168 | var changelogList []ChangelogResponse 169 | for _, changelog := range changelogs { 170 | changelogList = append(changelogList, ChangelogResponse{ 171 | ID: changelog.ID, 172 | Ts: changelog.Ts, 173 | Domain: changelog.Site, 174 | Message: changelog.Message, 175 | IPv6Status: changelog.IPv6Status, 176 | }) 177 | } 178 | 179 | // Send the changelog list as JSON 180 | render.JSON(w, r, changelogList) 181 | } 182 | 183 | // ChangelogByCampaign lists all changelog entries for a specific campaign UUID with pagination. 184 | func (rs ChangelogHandler) ChangelogByCampaign(w http.ResponseWriter, r *http.Request) { 185 | // Retrieve pagination input from context 186 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 187 | 188 | // Limit the maximum number of entries per page to 100 189 | if paginationInput.Limit > 100 { 190 | paginationInput.Limit = 100 191 | } 192 | 193 | // Get campaign UUID from path 194 | campaignUUID := chi.URLParam(r, "uuid") 195 | // Decode uuid from shortuuid to google uuid 196 | decodeID, err := decodeUUID(campaignUUID) 197 | if err != nil { 198 | render.Status(r, http.StatusNotFound) 199 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 200 | return 201 | } 202 | 203 | // Validate and parse the UUID 204 | uuid, err := uuid.Parse(decodeID.String()) 205 | if err != nil { 206 | render.Status(r, http.StatusNotFound) 207 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 208 | return 209 | } 210 | 211 | // Fetch changelogs for the campaign from the repository 212 | changelogs, err := rs.Repo.GetChangelogByCampaign( 213 | r.Context(), 214 | uuid, 215 | paginationInput.Offset, 216 | paginationInput.Limit, 217 | ) 218 | if err != nil { 219 | render.Status(r, http.StatusNotFound) 220 | render.JSON( 221 | w, 222 | r, 223 | render.M{"error": "Unable to find changelog entries for campaign " + uuid.String()}, 224 | ) 225 | return 226 | } 227 | 228 | // If no changelogs are found, return 404 229 | if len(changelogs) == 0 { 230 | render.Status(r, http.StatusNotFound) 231 | render.JSON( 232 | w, 233 | r, 234 | render.M{"error": "No changelog entries found for campaign " + uuid.String()}, 235 | ) 236 | return 237 | } 238 | 239 | // Convert changelogs to ChangelogResponse objects 240 | var changelogList []ChangelogResponse 241 | for _, changelog := range changelogs { 242 | changelogList = append(changelogList, ChangelogResponse{ 243 | ID: changelog.ID, 244 | Ts: changelog.Ts, 245 | Domain: changelog.Site, 246 | DomainURL: "/campaign/" + encodeUUID(changelog.CampaignID) + "/" + changelog.Site, 247 | Message: changelog.Message, 248 | IPv6Status: changelog.IPv6Status, 249 | }) 250 | } 251 | 252 | // Send the changelog list as JSON 253 | render.JSON(w, r, changelogList) 254 | } 255 | 256 | // ChangelogByCampaignDomain lists all changelog entries for a specific campaign UUID and domain with pagination. 257 | func (rs ChangelogHandler) ChangelogByCampaignDomain(w http.ResponseWriter, r *http.Request) { 258 | // Retrieve pagination input from context 259 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 260 | 261 | // Limit the maximum number of entries per page to 100 262 | if paginationInput.Limit > 100 { 263 | paginationInput.Limit = 100 264 | } 265 | 266 | // Get campaign UUID and domain from path 267 | campaignUUID := chi.URLParam(r, "uuid") 268 | site := chi.URLParam(r, "domain") 269 | 270 | // Validate the domain 271 | // TODO: Move this to core package 272 | if !regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`).MatchString(site) { 273 | render.Status(r, http.StatusBadRequest) 274 | render.JSON(w, r, render.M{"error": "Invalid domain"}) 275 | return 276 | } 277 | 278 | // Decode uuid from shortuuid to google uuid 279 | decodeID, err := decodeUUID(campaignUUID) 280 | if err != nil { 281 | render.Status(r, http.StatusBadRequest) 282 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 283 | return 284 | } 285 | 286 | // Validate and parse the UUID 287 | uuid, err := uuid.Parse(decodeID.String()) 288 | if err != nil { 289 | render.Status(r, http.StatusBadRequest) 290 | render.JSON(w, r, render.M{"error": "Invalid UUID"}) 291 | return 292 | } 293 | 294 | // Fetch changelogs for the campaign and domain from the repository 295 | changelogs, err := rs.Repo.GetChangelogByCampaignDomain( 296 | r.Context(), 297 | uuid, 298 | site, 299 | paginationInput.Offset, 300 | paginationInput.Limit, 301 | ) 302 | if err != nil { 303 | render.Status(r, http.StatusNotFound) 304 | render.JSON( 305 | w, 306 | r, 307 | render.M{ 308 | "error": "Unable to find changelog entries for campaign " + campaignUUID + " and domain " + site, 309 | }, 310 | ) 311 | return 312 | } 313 | 314 | // If no changelogs are found, return 404 315 | if len(changelogs) == 0 { 316 | render.Status(r, http.StatusNotFound) 317 | render.JSON( 318 | w, 319 | r, 320 | render.M{ 321 | "error": "No changelog entries found for campaign " + campaignUUID + " and domain " + site, 322 | }, 323 | ) 324 | return 325 | } 326 | 327 | // Convert changelogs to ChangelogResponse objects 328 | var changelogList []ChangelogResponse 329 | for _, changelog := range changelogs { 330 | changelogList = append(changelogList, ChangelogResponse{ 331 | ID: changelog.ID, 332 | Ts: changelog.Ts, 333 | Domain: changelog.Site, 334 | Message: changelog.Message, 335 | IPv6Status: changelog.IPv6Status, 336 | }) 337 | } 338 | 339 | // Send the changelog list as JSON 340 | render.JSON(w, r, changelogList) 341 | } 342 | -------------------------------------------------------------------------------- /internal/rest/country.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "strconv" 7 | "strings" 8 | 9 | "whynoipv6/internal/core" 10 | 11 | "github.com/ggicci/httpin" 12 | "github.com/go-chi/chi/v5" 13 | "github.com/go-chi/render" 14 | ) 15 | 16 | // CountryHandler is a handler for managing countries in the country service. 17 | type CountryHandler struct { 18 | Repo *core.CountryService 19 | } 20 | 21 | // CountryResponse is a structured response containing country data. 22 | type CountryResponse struct { 23 | Country string `json:"country"` 24 | CountryCode string `json:"country_code"` 25 | Sites int32 `json:"sites"` 26 | V6sites int32 `json:"v6sites"` 27 | Percent float64 `json:"percent"` 28 | } 29 | 30 | // Routes returns a router with all country-related endpoints mounted. 31 | func (rs CountryHandler) Routes() chi.Router { 32 | r := chi.NewRouter() 33 | // GET /country - Retrieve a list of all countries 34 | r.Get("/", rs.CountryList) 35 | 36 | // GET /country/{code} - Retrieve information about a specific country 37 | r.Get("/{code}", rs.CountryInfo) 38 | 39 | // GET /country/{code}/sinners - Retrieve all domains without IPv6 for a specific country by country code 40 | r.With(httpin.NewInput(PaginationInput{})).Get("/{code}/sinners", rs.CountrySinners) 41 | 42 | // GET /country/{code}/heroes - Retrieve all domains with IPv6 for a specific country by country code 43 | r.With(httpin.NewInput(PaginationInput{})).Get("/{code}/heroes", rs.CountryHeroes) 44 | 45 | return r 46 | } 47 | 48 | // CountryList retrieves and returns a list of all countries. 49 | func (rs CountryHandler) CountryList(w http.ResponseWriter, r *http.Request) { 50 | countries, err := rs.Repo.List(r.Context()) 51 | if err != nil { 52 | log.Println("Error retrieving countries:", err) 53 | render.Status(r, http.StatusInternalServerError) 54 | render.JSON(w, r, render.M{"error": "internal server error"}) 55 | return 56 | } 57 | 58 | var countryList []CountryResponse 59 | for _, country := range countries { 60 | // Convert pgtype.Numeric to float64 61 | percent, err := strconv.ParseFloat(country.Percent.Int.String(), 64) 62 | if err != nil { 63 | log.Println("Error converting percentage to float64:", err) 64 | continue 65 | } 66 | percent /= 10 67 | 68 | countryList = append(countryList, CountryResponse{ 69 | Country: country.Country, 70 | CountryCode: country.CountryCode, 71 | Sites: country.Sites, 72 | V6sites: country.V6sites, 73 | Percent: percent, 74 | }) 75 | } 76 | render.JSON(w, r, countryList) 77 | } 78 | 79 | // CountryInfo retrieves and returns information about a specific country and all the domains it has. 80 | func (rs CountryHandler) CountryInfo(w http.ResponseWriter, r *http.Request) { 81 | code := chi.URLParam(r, "code") 82 | 83 | // Get the country information 84 | countryInfo, err := rs.Repo.GetCountryCode(r.Context(), strings.ToUpper(code)) 85 | if err != nil { 86 | render.Status(r, http.StatusNotFound) 87 | render.JSON(w, r, render.M{"error": "Country not found"}) 88 | return 89 | } 90 | // Convert pgtype.Numeric to float64 91 | percent, err := strconv.ParseFloat(countryInfo.Percent.Int.String(), 64) 92 | if err != nil { 93 | log.Println("Error converting percentage to float64:", err) 94 | } 95 | percent /= 10 96 | 97 | // Build the response 98 | countryRespose := CountryResponse{ 99 | Country: countryInfo.Country, 100 | CountryCode: countryInfo.CountryCode, 101 | Sites: countryInfo.Sites, 102 | V6sites: countryInfo.V6sites, 103 | Percent: percent, 104 | } 105 | 106 | render.JSON(w, r, countryRespose) 107 | } 108 | 109 | // CountrySinners retrieves and returns a list of domains without IPv6 support for a given country code (e.g. US). 110 | func (rs CountryHandler) CountrySinners(w http.ResponseWriter, r *http.Request) { 111 | // Handle query params 112 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 113 | if paginationInput.Limit > 100 { 114 | paginationInput.Limit = 100 115 | } 116 | code := chi.URLParam(r, "code") 117 | 118 | country, err := rs.Repo.GetCountryCode(r.Context(), strings.ToUpper(code)) 119 | if err != nil { 120 | render.Status(r, http.StatusNotFound) 121 | render.JSON(w, r, render.M{"error": "Country not found"}) 122 | return 123 | } 124 | 125 | // Retrieve the list of domains for the country.ID 126 | domains, err := rs.Repo.ListDomainsByCountry( 127 | r.Context(), 128 | country.ID, 129 | paginationInput.Offset, 130 | paginationInput.Limit, 131 | ) 132 | if err != nil { 133 | render.Status(r, http.StatusInternalServerError) 134 | render.JSON(w, r, render.M{"error": "internal server error"}) 135 | return 136 | } 137 | var domainList []DomainResponse 138 | for _, domain := range domains { 139 | domainList = append(domainList, DomainResponse{ 140 | Rank: domain.Rank, 141 | Domain: domain.Site, 142 | BaseDomain: domain.BaseDomain, 143 | WwwDomain: domain.WwwDomain, 144 | Nameserver: domain.Nameserver, 145 | MXRecord: domain.MXRecord, 146 | V6Only: domain.V6Only, 147 | AsName: domain.AsName, 148 | Country: domain.Country, 149 | TsBaseDomain: domain.TsBaseDomain, 150 | TsWwwDomain: domain.TsWwwDomain, 151 | TsNameserver: domain.TsNameserver, 152 | TsMXRecord: domain.TsMXRecord, 153 | TsV6Only: domain.TsV6Only, 154 | TsCheck: domain.TsCheck, 155 | TsUpdated: domain.TsUpdated, 156 | }) 157 | } 158 | render.JSON(w, r, domainList) 159 | } 160 | 161 | // CountryHeroes retrieves and returns a list of domains with IPv6 support for a given country code (e.g. US). 162 | func (rs CountryHandler) CountryHeroes(w http.ResponseWriter, r *http.Request) { 163 | // Handle query params 164 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 165 | if paginationInput.Limit > 100 { 166 | paginationInput.Limit = 100 167 | } 168 | code := chi.URLParam(r, "code") 169 | 170 | country, err := rs.Repo.GetCountryCode(r.Context(), strings.ToUpper(code)) 171 | if err != nil { 172 | render.Status(r, http.StatusNotFound) 173 | render.JSON(w, r, render.M{"error": "Country not found"}) 174 | return 175 | } 176 | 177 | // Retrieve the list of IPv6-supported domains (heroes) for the country.ID 178 | heroes, err := rs.Repo.ListDomainHeroesByCountry( 179 | r.Context(), 180 | country.ID, 181 | paginationInput.Offset, 182 | paginationInput.Limit, 183 | ) 184 | if err != nil { 185 | render.Status(r, http.StatusInternalServerError) 186 | render.JSON(w, r, render.M{"error": "internal server error"}) 187 | return 188 | } 189 | var heroList []DomainResponse 190 | for _, domain := range heroes { 191 | heroList = append(heroList, DomainResponse{ 192 | Rank: domain.Rank, 193 | Domain: domain.Site, 194 | BaseDomain: domain.BaseDomain, 195 | WwwDomain: domain.WwwDomain, 196 | Nameserver: domain.Nameserver, 197 | MXRecord: domain.MXRecord, 198 | V6Only: domain.V6Only, 199 | AsName: domain.AsName, 200 | Country: domain.Country, 201 | TsBaseDomain: domain.TsBaseDomain, 202 | TsWwwDomain: domain.TsWwwDomain, 203 | TsNameserver: domain.TsNameserver, 204 | TsMXRecord: domain.TsMXRecord, 205 | TsV6Only: domain.TsV6Only, 206 | TsCheck: domain.TsCheck, 207 | TsUpdated: domain.TsUpdated, 208 | }) 209 | } 210 | render.JSON(w, r, heroList) 211 | } 212 | -------------------------------------------------------------------------------- /internal/rest/domain.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "strings" 7 | "time" 8 | 9 | "whynoipv6/internal/core" 10 | 11 | "github.com/ggicci/httpin" 12 | "github.com/go-chi/chi/v5" 13 | "github.com/go-chi/render" 14 | ) 15 | 16 | // DomainHandler is a handler for managing domain-related operations. 17 | type DomainHandler struct { 18 | Repo *core.DomainService 19 | } 20 | 21 | // DomainResponse is the response structure for a domain. 22 | type DomainResponse struct { 23 | Rank int64 `json:"rank"` 24 | Domain string `json:"domain"` 25 | BaseDomain string `json:"base_domain"` 26 | WwwDomain string `json:"www_domain"` 27 | Nameserver string `json:"nameserver"` 28 | MXRecord string `json:"mx_record"` 29 | V6Only string `json:"v6_only"` 30 | AsName string `json:"asn"` 31 | Country string `json:"country"` 32 | TsBaseDomain time.Time `json:"ts_aaaa"` 33 | TsWwwDomain time.Time `json:"ts_www"` 34 | TsNameserver time.Time `json:"ts_ns"` 35 | TsMXRecord time.Time `json:"ts_mx"` 36 | TsV6Only time.Time `json:"ts_curl"` 37 | TsCheck time.Time `json:"ts_check"` 38 | TsUpdated time.Time `json:"ts_updated"` 39 | CampaignUUID string `json:"campaign_uuid,omitempty"` 40 | } 41 | 42 | // DomainLogResponse is the response structure for a domain log. 43 | type DomainLogResponse struct { 44 | ID int64 `json:"id"` 45 | Time time.Time `json:"time"` 46 | BaseDomain string `json:"base_domain"` 47 | WwwDomain string `json:"www_domain"` 48 | Nameserver string `json:"nameserver"` 49 | MXRecord string `json:"mx_record"` 50 | } 51 | 52 | // Routes returns a router with all domain-related endpoints mounted. 53 | func (rs DomainHandler) Routes() chi.Router { 54 | r := chi.NewRouter() 55 | 56 | // GET /domain - list all domains 57 | r.With(httpin.NewInput(PaginationInput{})).Get("/", rs.DomainList) 58 | // GET /domain/heroes - list the domains with IPv6 59 | r.With(httpin.NewInput(PaginationInput{})).Get("/heroes", rs.DomainHeroes) 60 | // GET /domain/topsinner - list the top 10-ish domains without IPv6 61 | r.Get("/topsinner", rs.TopSinner) 62 | // GET /domain/{domain} - retrieve a domain by its name 63 | r.Get("/{domain}", rs.RetrieveDomain) 64 | // GET /domain/{domain}/log - retrieve a domain by its name 65 | r.Get("/{domain}/log", rs.GetDomainLog) 66 | // GET /domain/search/{domain} - search for a domain by its name 67 | r.With(httpin.NewInput(PaginationInput{})).Get("/search/{domain}", rs.SearchDomain) 68 | 69 | return r 70 | } 71 | 72 | // DomainList returns all domains. 73 | func (rs DomainHandler) DomainList(w http.ResponseWriter, r *http.Request) { 74 | // Handle query params 75 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 76 | if paginationInput.Limit > 100 { 77 | paginationInput.Limit = 100 78 | } 79 | 80 | domains, err := rs.Repo.ListDomain(r.Context(), paginationInput.Offset, paginationInput.Limit) 81 | if err != nil { 82 | render.Status(r, http.StatusInternalServerError) 83 | render.JSON(w, r, render.M{"error": "internal server error"}) 84 | return 85 | } 86 | var domainlist []DomainResponse 87 | for _, domain := range domains { 88 | domainlist = append(domainlist, DomainResponse{ 89 | Rank: domain.Rank, 90 | Domain: domain.Site, 91 | BaseDomain: domain.BaseDomain, 92 | WwwDomain: domain.WwwDomain, 93 | Nameserver: domain.Nameserver, 94 | MXRecord: domain.MXRecord, 95 | V6Only: domain.V6Only, 96 | AsName: domain.AsName, 97 | Country: domain.Country, 98 | TsBaseDomain: domain.TsBaseDomain, 99 | TsWwwDomain: domain.TsWwwDomain, 100 | TsNameserver: domain.TsNameserver, 101 | TsMXRecord: domain.TsMXRecord, 102 | TsV6Only: domain.TsV6Only, 103 | TsCheck: domain.TsCheck, 104 | TsUpdated: domain.TsUpdated, 105 | }) 106 | } 107 | render.JSON(w, r, domainlist) 108 | } 109 | 110 | // DomainHeroes returns the domains with IPv6 support. 111 | func (rs DomainHandler) DomainHeroes(w http.ResponseWriter, r *http.Request) { 112 | // Handle query params 113 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 114 | if paginationInput.Limit > 100 { 115 | paginationInput.Limit = 100 116 | } 117 | 118 | domains, err := rs.Repo.ListDomainHeroes( 119 | r.Context(), 120 | paginationInput.Offset, 121 | paginationInput.Limit, 122 | ) 123 | if err != nil { 124 | render.Status(r, http.StatusInternalServerError) 125 | render.JSON(w, r, render.M{"error": "internal server error"}) 126 | return 127 | } 128 | var domainlist []DomainResponse 129 | for _, domain := range domains { 130 | domainlist = append(domainlist, DomainResponse{ 131 | Rank: domain.Rank, 132 | Domain: domain.Site, 133 | BaseDomain: domain.BaseDomain, 134 | WwwDomain: domain.WwwDomain, 135 | Nameserver: domain.Nameserver, 136 | MXRecord: domain.MXRecord, 137 | V6Only: domain.V6Only, 138 | AsName: domain.AsName, 139 | Country: domain.Country, 140 | TsBaseDomain: domain.TsBaseDomain, 141 | TsWwwDomain: domain.TsWwwDomain, 142 | TsNameserver: domain.TsNameserver, 143 | TsMXRecord: domain.TsMXRecord, 144 | TsV6Only: domain.TsV6Only, 145 | TsCheck: domain.TsCheck, 146 | TsUpdated: domain.TsUpdated, 147 | }) 148 | } 149 | render.JSON(w, r, domainlist) 150 | } 151 | 152 | // RetrieveDomain returns a domain based on the provided domain name. 153 | func (rs DomainHandler) RetrieveDomain(w http.ResponseWriter, r *http.Request) { 154 | d := chi.URLParam(r, "domain") 155 | domain, err := rs.Repo.ViewDomain(r.Context(), d) 156 | if err != nil { 157 | render.Status(r, http.StatusNotFound) 158 | render.JSON(w, r, render.M{"error": "domain not found"}) 159 | return 160 | } 161 | render.JSON(w, r, DomainResponse{ 162 | Rank: domain.Rank, 163 | Domain: domain.Site, 164 | BaseDomain: domain.BaseDomain, 165 | WwwDomain: domain.WwwDomain, 166 | Nameserver: domain.Nameserver, 167 | MXRecord: domain.MXRecord, 168 | V6Only: domain.V6Only, 169 | AsName: domain.AsName, 170 | Country: domain.Country, 171 | TsBaseDomain: domain.TsBaseDomain, 172 | TsWwwDomain: domain.TsWwwDomain, 173 | TsNameserver: domain.TsNameserver, 174 | TsMXRecord: domain.TsMXRecord, 175 | TsV6Only: domain.TsV6Only, 176 | TsCheck: domain.TsCheck, 177 | TsUpdated: domain.TsUpdated, 178 | }) 179 | } 180 | 181 | // SearchDomain returns a domain based on the provided domain name. 182 | func (rs DomainHandler) SearchDomain(w http.ResponseWriter, r *http.Request) { 183 | // Handle query params 184 | paginationInput := r.Context().Value(httpin.Input).(*PaginationInput) 185 | if paginationInput.Limit > 100 { 186 | paginationInput.Limit = 100 187 | } 188 | 189 | domain := chi.URLParam(r, "domain") 190 | 191 | domains, err := rs.Repo.GetDomainsByName( 192 | r.Context(), 193 | strings.ToLower(domain), 194 | paginationInput.Offset, 195 | paginationInput.Limit, 196 | ) 197 | if err != nil { 198 | render.Status(r, http.StatusNotFound) 199 | render.JSON(w, r, render.M{"error": "Internal server error"}) 200 | return 201 | } 202 | 203 | if len(domains) == 0 { 204 | render.Status(r, http.StatusNotFound) 205 | render.JSON(w, r, render.M{"error": "no domains found"}) 206 | return 207 | } 208 | 209 | var domainList []DomainResponse 210 | for _, domain := range domains { 211 | domainList = append(domainList, DomainResponse{ 212 | Rank: domain.Rank, 213 | Domain: domain.Site, 214 | BaseDomain: domain.BaseDomain, 215 | WwwDomain: domain.WwwDomain, 216 | Nameserver: domain.Nameserver, 217 | MXRecord: domain.MXRecord, 218 | V6Only: domain.V6Only, 219 | AsName: domain.AsName, 220 | Country: domain.Country, 221 | TsBaseDomain: domain.TsBaseDomain, 222 | TsWwwDomain: domain.TsWwwDomain, 223 | TsNameserver: domain.TsNameserver, 224 | TsMXRecord: domain.TsMXRecord, 225 | TsV6Only: domain.TsV6Only, 226 | TsCheck: domain.TsCheck, 227 | TsUpdated: domain.TsUpdated, 228 | }) 229 | } 230 | 231 | render.JSON(w, r, render.M{ 232 | "data": domainList, 233 | }) 234 | } 235 | 236 | // TopSinner returns the top 10-ish domains without IPv6 support. 237 | func (rs DomainHandler) TopSinner(w http.ResponseWriter, r *http.Request) { 238 | domains, err := rs.Repo.ListDomainShamers(r.Context()) 239 | if err != nil { 240 | render.Status(r, http.StatusInternalServerError) 241 | render.JSON(w, r, render.M{"error": "internal server error"}) 242 | log.Println("Error listing domain shamers:", err) 243 | return 244 | } 245 | var domainlist []DomainResponse 246 | for _, domain := range domains { 247 | domainlist = append(domainlist, DomainResponse{ 248 | Rank: domain.ID, 249 | Domain: domain.Site, 250 | BaseDomain: domain.BaseDomain, 251 | WwwDomain: domain.WwwDomain, 252 | Nameserver: domain.Nameserver, 253 | MXRecord: domain.MXRecord, 254 | V6Only: domain.V6Only, 255 | TsBaseDomain: domain.TsBaseDomain, 256 | TsWwwDomain: domain.TsWwwDomain, 257 | TsNameserver: domain.TsNameserver, 258 | TsMXRecord: domain.TsMXRecord, 259 | TsV6Only: domain.TsV6Only, 260 | TsCheck: domain.TsCheck, 261 | TsUpdated: domain.TsUpdated, 262 | }) 263 | } 264 | render.JSON(w, r, domainlist) 265 | } 266 | 267 | // GetDomainLog returns the crawler log for a domain. 268 | func (rs DomainHandler) GetDomainLog(w http.ResponseWriter, r *http.Request) { 269 | domain := chi.URLParam(r, "domain") 270 | logs, err := rs.Repo.GetDomainLog(r.Context(), domain) 271 | if err != nil { 272 | render.Status(r, http.StatusNotFound) 273 | render.JSON(w, r, render.M{"error": "domain not found"}) 274 | return 275 | } 276 | var domainlist []DomainLogResponse 277 | for _, log := range logs { 278 | var data map[string]any 279 | if err := log.Data.AssignTo(&data); err != nil { 280 | render.Status(r, http.StatusInternalServerError) 281 | render.JSON(w, r, render.M{"error": "internal server error"}) 282 | return 283 | } 284 | domainlist = append(domainlist, DomainLogResponse{ 285 | ID: log.ID, 286 | Time: log.Time, 287 | BaseDomain: data["base_domain"].(string), 288 | WwwDomain: data["www_domain"].(string), 289 | Nameserver: data["nameserver"].(string), 290 | MXRecord: data["mx_record"].(string), 291 | }) 292 | } 293 | render.JSON(w, r, domainlist) 294 | } 295 | -------------------------------------------------------------------------------- /internal/rest/error.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/go-chi/render" 7 | ) 8 | 9 | // ErrResponse is a custom renderer type for handling various errors. 10 | type ErrResponse struct { 11 | Err error `json:"-"` // low-level runtime error 12 | HTTPStatusCode int `json:"-"` // http response status code 13 | 14 | StatusText string `json:"status"` // user-level status message 15 | AppCode int64 `json:"code,omitempty"` // application-specific error code 16 | ErrorText string `json:"error,omitempty"` // application-level error message, for debugging 17 | } 18 | 19 | // Render writes data with a custom HTTP status code. 20 | func (e *ErrResponse) Render(w http.ResponseWriter, r *http.Request) error { 21 | render.Status(r, e.HTTPStatusCode) 22 | return nil 23 | } 24 | 25 | // ErrInvalidRequest returns a structured HTTP response for invalid requests. 26 | func ErrInvalidRequest(err error) render.Renderer { 27 | return &ErrResponse{ 28 | Err: err, 29 | HTTPStatusCode: http.StatusBadRequest, 30 | StatusText: "Invalid request.", 31 | ErrorText: err.Error(), 32 | } 33 | } 34 | 35 | // ErrRender returns a structured HTTP response in case of rendering errors. 36 | func ErrRender(err error) render.Renderer { 37 | return &ErrResponse{ 38 | Err: err, 39 | HTTPStatusCode: http.StatusUnprocessableEntity, 40 | StatusText: "Error rendering response.", 41 | ErrorText: err.Error(), 42 | } 43 | } 44 | 45 | // ErrNotFound returns a structured HTTP response if a resource could not be found. 46 | func ErrNotFound() render.Renderer { 47 | return &ErrResponse{ 48 | HTTPStatusCode: http.StatusNotFound, 49 | StatusText: "Resource not found.", 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /internal/rest/metric.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "whynoipv6/internal/core" 8 | 9 | "github.com/go-chi/chi/v5" 10 | "github.com/go-chi/render" 11 | "github.com/jackc/pgtype" 12 | ) 13 | 14 | // MetricHandler is a handler for managing all metric-related operations. 15 | type MetricHandler struct { 16 | Repo *core.MetricService 17 | } 18 | 19 | // MetricResponse is the response structure for a metric. 20 | type MetricResponse struct { 21 | Time time.Time `json:"time"` 22 | Data pgtype.JSONB `json:"data"` 23 | } 24 | 25 | // Routes returns a router with all metric endpoints mounted. 26 | func (rs MetricHandler) Routes() chi.Router { 27 | r := chi.NewRouter() 28 | 29 | // GET /metrics/total 30 | r.Get("/overview", rs.Overview) 31 | 32 | // GET /metrics/asn 33 | r.Get("/asn", rs.AsnMetrics) 34 | 35 | // GET /metrics/asn/search/{query} 36 | r.Get("/asn/search/{query}", rs.SearchAsn) 37 | 38 | return r 39 | } 40 | 41 | // Totals returns the aggregated metrics for all crawled domains. 42 | func (rs MetricHandler) Totals(w http.ResponseWriter, r *http.Request) { 43 | metrics, err := rs.Repo.GetMetrics(r.Context(), "domains") 44 | if err != nil { 45 | render.Status(r, http.StatusNotFound) 46 | render.JSON(w, r, render.M{"error": "metric not found"}) 47 | return 48 | } 49 | 50 | var metricList []MetricResponse 51 | for _, metric := range metrics { 52 | metricList = append(metricList, MetricResponse{ 53 | Time: metric.Time, 54 | Data: metric.Data, 55 | }) 56 | } 57 | 58 | render.JSON(w, r, metricList) 59 | } 60 | 61 | // Overview returns the aggregated metrics for all crawled domains. 62 | func (rs MetricHandler) Overview(w http.ResponseWriter, r *http.Request) { 63 | // metrics, err := rs.Repo.DomainStats(r.Context()) 64 | metrics, err := rs.Repo.DomainStats(r.Context()) 65 | if err != nil { 66 | render.Status(r, http.StatusNotFound) 67 | render.JSON(w, r, render.M{"error": "metric not found"}) 68 | return 69 | } 70 | 71 | var metricList []MetricResponse 72 | for _, metric := range metrics { 73 | metricList = append(metricList, MetricResponse{ 74 | Time: metric.Time, 75 | Data: metric.Data, 76 | }) 77 | } 78 | 79 | render.JSON(w, r, metricList) 80 | } 81 | 82 | // ASNResponse represents a BGP Autonomous System Number (ASN) and its associated information. 83 | type ASNResponse struct { 84 | ID int64 `json:"id"` 85 | Number int32 `json:"number"` 86 | Name string `json:"name"` 87 | CountV4 int32 `json:"count_v4"` 88 | CountV6 int32 `json:"count_v6"` 89 | PercentV4 float64 `json:"percent_v4,omitempty"` 90 | PercentV6 float64 `json:"percent_v6,omitempty"` 91 | } 92 | 93 | // AsnMetrics returns the aggregated metrics for all crawled domains per ASN. 94 | func (rs MetricHandler) AsnMetrics(w http.ResponseWriter, r *http.Request) { 95 | // Retrieve the order parameter from the URL 96 | order := r.URL.Query().Get("order") 97 | if order == "" { 98 | order = "ipv4" 99 | } 100 | 101 | // Validate the filter parameter 102 | if order != "ipv4" && order != "ipv6" { 103 | render.Status(r, http.StatusBadRequest) 104 | render.JSON(w, r, render.M{"error": "invalid filter parameter"}) 105 | return 106 | } 107 | 108 | asn, err := rs.Repo.AsnList(r.Context(), 0, 50, order) 109 | if err != nil { 110 | render.Status(r, http.StatusNotFound) 111 | render.JSON(w, r, render.M{"error": "metric not found"}) 112 | return 113 | } 114 | 115 | var asnList []ASNResponse 116 | for _, a := range asn { 117 | asnList = append(asnList, ASNResponse{ 118 | ID: a.ID, 119 | Number: a.Number, 120 | Name: a.Name, 121 | CountV4: a.CountV4, 122 | CountV6: a.CountV6, 123 | PercentV4: a.PercentV4, 124 | PercentV6: a.PercentV6, 125 | }) 126 | } 127 | 128 | render.JSON(w, r, asnList) 129 | } 130 | 131 | // SearchAsn returns the metrics for a given ASN. 132 | func (rs MetricHandler) SearchAsn(w http.ResponseWriter, r *http.Request) { 133 | // Retrieve the query parameter from the URL 134 | searchQuery := chi.URLParam(r, "query") 135 | 136 | asn, err := rs.Repo.SearchAsn(r.Context(), searchQuery) 137 | if err != nil { 138 | render.Status(r, http.StatusNotFound) 139 | render.JSON(w, r, render.M{"error": "asn not found"}) 140 | return 141 | } 142 | 143 | var asnList []ASNResponse 144 | for _, a := range asn { 145 | asnList = append(asnList, ASNResponse{ 146 | ID: a.ID, 147 | Number: a.Number, 148 | Name: a.Name, 149 | CountV4: a.CountV4, 150 | CountV6: a.CountV6, 151 | PercentV4: a.PercentV4, 152 | PercentV6: a.PercentV6, 153 | }) 154 | } 155 | 156 | render.JSON(w, r, asnList) 157 | } 158 | -------------------------------------------------------------------------------- /internal/rest/server.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | 7 | "github.com/go-chi/chi/v5" 8 | "github.com/go-chi/chi/v5/middleware" 9 | "github.com/go-chi/render" 10 | "github.com/google/uuid" 11 | "github.com/lithammer/shortuuid/v4" 12 | "github.com/rs/cors" 13 | ) 14 | 15 | // NewRouter instantiates a new router. 16 | func NewRouter() (*chi.Mux, error) { 17 | r := chi.NewRouter() 18 | 19 | corsMiddleware := cors.New(cors.Options{ 20 | // AllowedOrigins: []string{"https://whynoipv6.com","https://ipv6.fail"}, // Use this to allow specific origin hosts 21 | AllowedOrigins: []string{"https://*", "http://*"}, 22 | AllowedMethods: []string{"GET", "HEAD", "OPTIONS"}, 23 | AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"}, 24 | ExposedHeaders: []string{"Link"}, 25 | AllowCredentials: false, 26 | MaxAge: 300, // Maximum value not ignored by any of major browsers 27 | }) 28 | 29 | r.Use( 30 | render.SetContentType( 31 | render.ContentTypeJSON, 32 | ), // Set content-Type headers as application/json 33 | middleware.RealIP, // Logs the real ip from nginx 34 | middleware.Logger, // Log API request calls 35 | middleware.Recoverer, // Recover from panics without crashing server 36 | middleware.RequestID, // Injects a request ID into the context of each request 37 | // middleware.RedirectSlashes, // Redirect slashes to no slash URL versions 38 | middleware.NoCache, // We dont like cache! 39 | middleware.SetHeader("X-Content-Type-Options", "nosniff"), 40 | middleware.SetHeader("X-Frame-Options", "deny"), 41 | corsMiddleware.Handler, 42 | ) 43 | 44 | return r, nil 45 | } 46 | 47 | // PrintRoutes prints the routes of the application. 48 | func PrintRoutes(r *chi.Mux) { 49 | log.Println("Routes:") 50 | err := chi.Walk( 51 | r, 52 | func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { 53 | log.Printf("\t[%s]: '%s'\n", method, route) 54 | return nil 55 | }, 56 | ) 57 | if err != nil { 58 | log.Println("Error printing routes:", err) 59 | } 60 | log.Println("") 61 | } 62 | 63 | // PaginationInput is the path variables from the request. 64 | type PaginationInput struct { 65 | Offset int64 `in:"query=offset;default=0"` 66 | Limit int64 `in:"query=limit;default=50"` 67 | } 68 | 69 | // encodeUUID encodes a UUID to a short UUID. 70 | func encodeUUID(id uuid.UUID) string { 71 | id, err := uuid.Parse(id.String()) 72 | if err != nil { 73 | return "" 74 | } 75 | su := shortuuid.DefaultEncoder.Encode(id) 76 | return su 77 | } 78 | 79 | // decodeUUID decodes a short UUID to a UUID. 80 | func decodeUUID(id string) (uuid.UUID, error) { 81 | decoded, err := shortuuid.DefaultEncoder.Decode(id) 82 | if err != nil { 83 | return uuid.Nil, err 84 | } 85 | return decoded, nil 86 | } 87 | -------------------------------------------------------------------------------- /internal/toolbox/healthcheck.go: -------------------------------------------------------------------------------- 1 | package toolbox 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "whynoipv6/internal/logger" 10 | ) 11 | 12 | // HealthOK and HealthFail are the status codes for successful and failed health checks, respectively. 13 | const ( 14 | HealthOK = 0 15 | HealthFail = 1 16 | ) 17 | 18 | // HealthCheckUpdate sends a successful health check notification to BetterUptime.com. 19 | // The function takes a unique identifier (uuid) as input and sends an HTTP HEAD request to BetterUptime.com's API. 20 | // If there's an error, it will log the error message. 21 | func HealthCheckUpdate(uuid string, status int) { 22 | log := logger.GetLogger() 23 | log = log.With().Str("service", "HealthCheckUpdate").Logger() 24 | // Create an HTTP client with a 10-second timeout. 25 | httpClient := &http.Client{ 26 | Timeout: 10 * time.Second, 27 | } 28 | 29 | // Create the URL for the BetterUptime.com API. 30 | apiURL := fmt.Sprintf("https://uptime.betteruptime.com/api/v1/heartbeat/%s/%d", uuid, status) 31 | 32 | // Send the HTTP HEAD request. 33 | resp, err := httpClient.Head(apiURL) 34 | // If there's an error, log the error message. 35 | if err != nil { 36 | log.Err(err).Msg("Error while sending health check update.") 37 | return 38 | } 39 | 40 | // Close the response body when the function exits. 41 | defer func() { 42 | if err := resp.Body.Close(); err != nil { 43 | log.Err(err).Msg("Error while closing response body.") 44 | } 45 | }() 46 | 47 | // Check if the response status code indicates success (2xx). 48 | if resp.StatusCode >= 200 && resp.StatusCode < 300 { 49 | log.Debug().Msg("Successfully sent healthcheck update.") 50 | } else { 51 | // log.Printf("Failed to send health check update. Status code: %d\n", resp.StatusCode) 52 | log.Err(errors.New("Status Code:" + resp.Status)).Msg("Failed to send health check update.") 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/toolbox/notify.go: -------------------------------------------------------------------------------- 1 | package toolbox 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "encoding/json" 7 | "log" 8 | "net" 9 | "net/http" 10 | "time" 11 | 12 | "whynoipv6/internal/config" 13 | ) 14 | 15 | // apiMessage is the message from api to irc 16 | type apiMessage struct { 17 | Channel string `json:"channel"` 18 | Message string `json:"message"` 19 | } 20 | 21 | var ( 22 | httpClient *http.Client 23 | cfg *config.Config 24 | err error 25 | ) 26 | 27 | func init() { 28 | httpClient = createHTTPClient() 29 | 30 | // Read the configuration. 31 | cfg, err = config.Read() 32 | if err != nil { 33 | log.Fatal("Failed to read config: ", err) 34 | } 35 | } 36 | 37 | // createHTTPClient initializes an http.Client with better default settings. 38 | func createHTTPClient() *http.Client { 39 | netTransport := &http.Transport{ 40 | Dial: (&net.Dialer{ 41 | Timeout: 5 * time.Second, 42 | }).Dial, 43 | TLSHandshakeTimeout: 5 * time.Second, 44 | TLSClientConfig: &tls.Config{}, 45 | Proxy: http.ProxyFromEnvironment, 46 | } 47 | client := &http.Client{ 48 | Timeout: time.Duration(5) * time.Second, 49 | Transport: netTransport, 50 | } 51 | return client 52 | } 53 | 54 | // NotifyIrc sends message to irc 55 | // This is a private setup, please don't use this 56 | func NotifyIrc(m string) { 57 | // New message 58 | message := apiMessage{ 59 | Channel: "legz", 60 | Message: m, 61 | } 62 | mJSON, err := json.Marshal(message) 63 | if err != nil { 64 | log.Println(err) 65 | } 66 | req, err := http.NewRequest("POST", "https://partyvan.lasse.cloud/say", bytes.NewBuffer(mJSON)) 67 | if err != nil { 68 | log.Println(err) 69 | } 70 | 71 | // Create a Bearer token 72 | bearer := "Bearer " + cfg.IRCToken 73 | req.Header.Add("Authorization", bearer) 74 | 75 | // Send request 76 | _, err = httpClient.Do(req) 77 | if err != nil { 78 | log.Println(err) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /service_domains.yml: -------------------------------------------------------------------------------- 1 | domains: 2 | - amazonaws.com 3 | - domaincontrol.com 4 | - amazon-adsystem.com 5 | - msedge.net 6 | - tiktokv.com 7 | - trbcdn.net 8 | - adnxs.com 9 | - msftncsi.com 10 | - netflix.net 11 | - googleadservices.com 12 | - registrar-servers.com 13 | - appsflyersdk.com 14 | - amazonvideo.com 15 | - omtrdc.net 16 | - demdex.net 17 | - azurewebsites.net 18 | - mzstatic.com 19 | - userapi.com 20 | - smartadserver.com 21 | - a-msedge.net 22 | - l-msedge.net 23 | - t-msedge.net 24 | - s-msedge.net 25 | - wac-msedge.net 26 | - b-msedge.net 27 | - spo-msedge.net 28 | - msedge.net 29 | - dual-s-msedge.net 30 | - e-msedge.net 31 | - spov-msedge.net 32 | - wac-dc-msedge.net 33 | - fdv2-t-msedge.net 34 | - arc-msedge.net 35 | - arm-msedge.net 36 | - ax-msedge.net 37 | - fb-t-msedge.net 38 | - c-msedge.net 39 | - dc-msedge.net 40 | - k-msedge.net 41 | - t-s2-msedge.net 42 | - ln-msedge.net 43 | - teams-msedge.net 44 | - fbs1-t-msedge.net 45 | - l-dc-msedge.net 46 | - dual-s-dc-msedge.net 47 | - s-dc-msedge.net 48 | - arc-dc-msedge.net 49 | - a-dc-msedge.net 50 | - arm-dc-msedge.net 51 | - t-s1-msedge.net 52 | - ax-dc-msedge.net 53 | - b-dc-msedge.net 54 | - q-msedge.net 55 | - o-msedge.net 56 | - exo-msedge.net 57 | - au-msedge.net 58 | - spov-dc-msedge.net 59 | - ln-dc-msedge.net 60 | - spo-dc-msedge.net 61 | - e-dc-msedge.net 62 | - c-dc-msedge.net 63 | - k-dc-msedge.net 64 | - q-t-msedge.net 65 | - m1-msedge.net 66 | - exo-dc-msedge.net 67 | - o-dc-msedge.net 68 | - teams-dc-msedge.net 69 | - fbs2-t-msedge.net 70 | - ztna3p-msedge.net 71 | - cn-msedge.net 72 | - ztna1p-msedge.net 73 | - segment5-s-msedge.net 74 | - stormsedge.net 75 | - segment2-s-msedge.net 76 | -------------------------------------------------------------------------------- /sqlc.yaml.example: -------------------------------------------------------------------------------- 1 | version: "2" 2 | sql: 3 | - schema: "db/migrations" 4 | queries: "db/query" 5 | engine: "postgresql" 6 | gen: 7 | go: 8 | package: "db" 9 | sql_package: "pgx/v4" 10 | out: "internal/postgres/db" 11 | emit_json_tags: false 12 | emit_db_tags: false 13 | emit_prepared_queries: false 14 | emit_interface: false 15 | emit_exact_table_names: true 16 | emit_empty_slices: true 17 | emit_exported_queries: true 18 | -------------------------------------------------------------------------------- /tldbwriter.toml.example: -------------------------------------------------------------------------------- 1 | [database] 2 | host = "localhost" 3 | port = 5432 4 | user = "username" 5 | password = "password" 6 | dbname = "database" 7 | sslmode = "disable" 8 | #sslmode = "verify-full" 9 | 10 | [updater] 11 | #interval = "1h" 12 | #jittermin = 0 13 | #jittermax = 300 14 | --------------------------------------------------------------------------------