├── docs └── img │ └── certstream-server-go_logo.png ├── docker ├── docker-compose.yml └── docker-compose.metrics.yml ├── .vscode └── launch.json ├── Dockerfile ├── .gitignore ├── .github ├── workflows │ ├── changelog_reminder.yml │ ├── release_build.yml │ └── codeql-analysis.yml └── goreleaser.yml ├── go.mod ├── LICENSE ├── internal ├── web │ ├── examplecert.go │ ├── broadcastmanager.go │ ├── client.go │ └── server.go ├── metrics │ └── prometheus.go ├── models │ └── certstream.go ├── certstream │ └── certstream.go ├── config │ └── config.go └── certificatetransparency │ ├── logmetrics.go │ ├── ct-parser.go │ └── ct-watcher.go ├── cmd ├── certstream-server-go │ └── main.go └── certpicker │ └── main.go ├── Dockerfile_multistage ├── config.sample.yaml ├── .golangci.yml ├── go.sum ├── CHANGELOG.md └── README.md /docs/img/certstream-server-go_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/d-Rickyy-b/certstream-server-go/HEAD/docs/img/certstream-server-go_logo.png -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | certstream: 5 | image: 0rickyy0/certstream-server-go:latest 6 | restart: always 7 | # Configure the service to run as specific user 8 | # user: "1000:1000" 9 | ports: 10 | - 127.0.0.1:8080:80 11 | # Don't forget to open the other port in case you run the Prometheus endpoint on another port than the websocket server. 12 | # - 127.0.0.1:8081:81 13 | volumes: 14 | - ./certstream/config.yml:/app/config.yml 15 | networks: 16 | - monitoring 17 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}/cmd/certstream-server-go", 13 | "cwd": "${workspaceFolder}", 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | WORKDIR /app 4 | 5 | ENV USER=certstreamserver 6 | ENV UID=10001 7 | 8 | # Create user 9 | RUN adduser \ 10 | --disabled-password \ 11 | --gecos "" \ 12 | --home "/nonexistent" \ 13 | --shell "/sbin/nologin" \ 14 | --no-create-home \ 15 | --uid "${UID}" \ 16 | "${USER}" 17 | 18 | # Copy our static executable. 19 | COPY certstream-server-go /app/certstream-server-go 20 | COPY ./config.sample.yaml /app/config.yaml 21 | 22 | # Use an unprivileged user. 23 | USER certstreamserver:certstreamserver 24 | 25 | EXPOSE 8080 26 | 27 | ENTRYPOINT ["/app/certstream-server-go"] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | .DS_Store 8 | /certpicker 9 | /certstream-server-go 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Temp files 18 | *.tmp 19 | *.temp 20 | 21 | # Dependency directories (remove the comment below to include it) 22 | # vendor/ 23 | 24 | # Go workspace file 25 | go.work 26 | 27 | .idea 28 | .cache 29 | *.pprof 30 | dist/ 31 | 32 | # Ignore actual config files 33 | config.yaml 34 | config.yml 35 | ct_index.json 36 | -------------------------------------------------------------------------------- /.github/workflows/changelog_reminder.yml: -------------------------------------------------------------------------------- 1 | name: Changelog Reminder 2 | on: pull_request 3 | permissions: 4 | pull-requests: write 5 | contents: read 6 | 7 | jobs: 8 | remind: 9 | name: Changelog Reminder 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Changelog Reminder 15 | uses: mskelton/changelog-reminder-action@v3 16 | with: 17 | token: ${{ secrets.GITHUB_TOKEN }} 18 | message: "@${{ github.actor }} We couldn't find any modification to the CHANGELOG.md file. If your changes are not suitable for the changelog, that's fine. Otherwise please add them to the changelog!" 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/d-Rickyy-b/certstream-server-go 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.2 6 | 7 | require ( 8 | github.com/VictoriaMetrics/metrics v1.40.1 9 | github.com/go-chi/chi/v5 v5.2.3 10 | github.com/google/certificate-transparency-go v1.3.2 11 | github.com/gorilla/websocket v1.5.3 12 | gopkg.in/yaml.v3 v3.0.1 13 | ) 14 | 15 | require ( 16 | github.com/go-logr/logr v1.4.3 // indirect 17 | github.com/google/trillian v1.7.2 // indirect 18 | github.com/valyala/fastrand v1.1.0 // indirect 19 | github.com/valyala/histogram v1.2.0 // indirect 20 | golang.org/x/crypto v0.42.0 // indirect 21 | golang.org/x/net v0.44.0 // indirect 22 | golang.org/x/sys v0.36.0 // indirect 23 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 // indirect 24 | google.golang.org/grpc v1.75.1 // indirect 25 | google.golang.org/protobuf v1.36.9 // indirect 26 | k8s.io/klog/v2 v2.130.1 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 d-Rickyy-b 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /internal/web/examplecert.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/d-Rickyy-b/certstream-server-go/internal/models" 7 | ) 8 | 9 | var exampleCert models.Entry 10 | 11 | // exampleFull handles requests to the /full-stream/example.json endpoint. 12 | // It returns a JSON representation of the full example certificate. 13 | func exampleFull(w http.ResponseWriter, _ *http.Request) { 14 | w.Header().Set("Content-Type", "application/json") 15 | w.Write(exampleCert.JSON()) //nolint:errcheck 16 | } 17 | 18 | // exampleLite handles requests to the /example.json endpoint. 19 | // It returns a JSON representation of the lite example certificate. 20 | func exampleLite(w http.ResponseWriter, _ *http.Request) { 21 | w.Header().Set("Content-Type", "application/json") 22 | w.Write(exampleCert.JSONLite()) //nolint:errcheck 23 | } 24 | 25 | // exampleDomains handles requests to the /domains-only/example.json endpoint. 26 | // It returns a JSON representation of the domain data. 27 | func exampleDomains(w http.ResponseWriter, _ *http.Request) { 28 | w.Header().Set("Content-Type", "application/json") 29 | w.Write(exampleCert.JSONDomains()) //nolint:errcheck 30 | } 31 | 32 | // SetExampleCert sets one certificate as the example Cert that is returned by the example endpoints. 33 | func SetExampleCert(cert models.Entry) { 34 | exampleCert = cert 35 | } 36 | -------------------------------------------------------------------------------- /cmd/certstream-server-go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" 9 | "github.com/d-Rickyy-b/certstream-server-go/internal/config" 10 | ) 11 | 12 | // main is the entry point for the application. 13 | func main() { 14 | configFile := flag.String("config", "config.yml", "path to the config file") 15 | versionFlag := flag.Bool("version", false, "Print the version and exit") 16 | createIndexFile := flag.Bool("create-index-file", false, "Create the ct_index.json based on current STHs") 17 | flag.Parse() 18 | 19 | if *versionFlag { 20 | fmt.Printf("certstream-server-go v%s\n", config.Version) 21 | return 22 | } 23 | 24 | log.SetFlags(log.LstdFlags | log.Lshortfile) 25 | 26 | // If the user only wants to create the index file, we don't need to start the server 27 | if *createIndexFile { 28 | conf, readConfErr := config.ReadConfig(*configFile) 29 | if readConfErr != nil { 30 | log.Fatalf("Error while reading config: %v", readConfErr) 31 | } 32 | cs := certstream.NewRawCertstream(conf) 33 | 34 | createErr := cs.CreateIndexFile() 35 | if createErr != nil { 36 | log.Fatalf("Error while creating index file: %v", createErr) 37 | } 38 | 39 | return 40 | } 41 | 42 | log.Printf("Starting certstream-server-go v%s\n", config.Version) 43 | 44 | cs, err := certstream.NewCertstreamFromConfigFile(*configFile) 45 | if err != nil { 46 | log.Fatalf("Error while creating certstream server: %v", err) 47 | } 48 | 49 | cs.Start() 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/release_build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | jobs: 9 | build: 10 | name: Release build 11 | runs-on: ubuntu-latest 12 | permissions: 13 | packages: write 14 | contents: write 15 | 16 | steps: 17 | - name: Set up Go 1.22 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version: ^1.22 21 | id: go 22 | 23 | - name: Check out code into the Go module directory 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 # See: https://goreleaser.com/ci/actions/ 27 | 28 | - name: Setup QEMU # Used for cross-compiling with goreleaser / docker 29 | uses: docker/setup-qemu-action@v3 30 | 31 | - name: Setup Docker Buildx # Used for cross-compiling with goreleaser / docker 32 | uses: docker/setup-buildx-action@v3 33 | 34 | - name: Login to Docker Hub 35 | uses: docker/login-action@v3 36 | with: 37 | username: ${{ secrets.DOCKERHUB_USERNAME }} 38 | password: ${{ secrets.DOCKERHUB_TOKEN }} 39 | 40 | - name: Login to GitHub Container Registry 41 | uses: docker/login-action@v3 42 | with: 43 | registry: ghcr.io 44 | username: ${{ github.repository_owner }} 45 | password: ${{ secrets.GITHUB_TOKEN }} 46 | 47 | - name: Run GoReleaser 48 | uses: goreleaser/goreleaser-action@v5 49 | with: 50 | version: latest 51 | args: release --clean --config .github/goreleaser.yml 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | -------------------------------------------------------------------------------- /Dockerfile_multistage: -------------------------------------------------------------------------------- 1 | # Thanks to https://chemidy.medium.com/create-the-smallest-and-secured-golang-docker-image-based-on-scratch-4752223b7324 2 | ############################ 3 | # STEP 1 build executable binary 4 | ############################ 5 | FROM golang:alpine AS builder 6 | 7 | ENV USER=certstreamserver 8 | ENV UID=10001 9 | 10 | # Create user 11 | RUN adduser \ 12 | --disabled-password \ 13 | --gecos "" \ 14 | --home "/nonexistent" \ 15 | --shell "/sbin/nologin" \ 16 | --no-create-home \ 17 | --uid "${UID}" \ 18 | "${USER}" 19 | 20 | # Install git. Git is required for fetching the dependencies. 21 | RUN apk update && apk add --no-cache git 22 | WORKDIR $GOPATH/src/certstream-server-go/ 23 | COPY . . 24 | 25 | # Fetch dependencies. 26 | RUN go mod download 27 | 28 | # Build the binary. 29 | RUN go build -ldflags="-w -s" -o /go/bin/certstream-server-go $GOPATH/src/certstream-server-go/cmd/certstream-server-go/ 30 | RUN chown -R "${USER}:${USER}" /go/bin/certstream-server-go 31 | 32 | ############################ 33 | # STEP 2 build a small image 34 | ############################ 35 | FROM alpine 36 | 37 | WORKDIR /app 38 | 39 | # Import the user and group files from the builder. 40 | COPY --from=builder /etc/passwd /etc/passwd 41 | COPY --from=builder /etc/group /etc/group 42 | 43 | # Copy our static executable. 44 | COPY --from=builder /go/bin/certstream-server-go /app/certstream-server-go 45 | COPY --chown=certstreamserver:certstreamserver ./config.sample.yaml /app/config.yaml 46 | 47 | # Use an unprivileged user. 48 | USER certstreamserver:certstreamserver 49 | 50 | EXPOSE 8080 51 | 52 | ENTRYPOINT ["/app/certstream-server-go"] 53 | -------------------------------------------------------------------------------- /docker/docker-compose.metrics.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | # Make sure to create the sub directories "prometheus", "prometheus_data", "grafana", "grafana_data" and "certstream" 4 | # and create the config files for all three services. For further details please refer to https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics 5 | 6 | networks: 7 | monitoring: 8 | driver: bridge 9 | ipam: 10 | config: 11 | - subnet: 172.90.0.0/24 12 | gateway: 172.90.0.1 13 | 14 | services: 15 | prometheus: 16 | image: prom/prometheus:v2.40.5 17 | restart: always 18 | # Configure the service to run as specific user. 19 | # user: "1000:1000" 20 | volumes: 21 | - ./prometheus/:/etc/prometheus/ 22 | - ./prometheus_data:/prometheus/ 23 | command: 24 | - '--config.file=/etc/prometheus/prometheus.yml' 25 | - '--storage.tsdb.path=/prometheus' 26 | - '--storage.tsdb.retention.time=1y' 27 | - '--web.console.libraries=/etc/prometheus/console_libraries' 28 | - '--web.console.templates=/etc/prometheus/consoles' 29 | - '--web.enable-lifecycle' 30 | ports: 31 | # Exposing Prometheus is NOT required, if you don't want to access it from outside the Docker network. 32 | # Using localhost enables you to use a reverse proxy (e.g. with basic auth) to access Prometheus in a more secure way. 33 | - 127.0.0.1:9090:9090 34 | networks: 35 | - monitoring 36 | extra_hosts: 37 | - "host.docker.internal:host-gateway" 38 | 39 | grafana: 40 | image: grafana/grafana:9.3.1 41 | restart: always 42 | # Configure the service to run as specific user. 43 | # user: "1000:1000" 44 | depends_on: 45 | - prometheus 46 | ports: 47 | - 127.0.0.1:8082:3000 48 | volumes: 49 | - ./grafana_data:/var/lib/grafana 50 | - ./grafana/provisioning/:/etc/grafana/provisioning/ 51 | env_file: 52 | # changes to the grafana env file require a rebuild of the container. 53 | - ./grafana/config.monitoring 54 | networks: 55 | - monitoring 56 | 57 | certstream: 58 | image: 0rickyy0/certstream-server-go:latest 59 | restart: always 60 | # Configure the service to run as specific user. 61 | # user: "1000:1000" 62 | ports: 63 | - 127.0.0.1:8080:80 64 | # Don't forget to open the other port in case you run the Prometheus endpoint on another port than the websocket server. 65 | # - 127.0.0.1:8081:81 66 | volumes: 67 | - ./certstream/config.yml:/app/config.yml 68 | networks: 69 | - monitoring 70 | -------------------------------------------------------------------------------- /config.sample.yaml: -------------------------------------------------------------------------------- 1 | webserver: 2 | # For IPv6, set the listen_addr to "::" 3 | listen_addr: "0.0.0.0" 4 | listen_port: 8080 5 | # If you want to use a reverse proxy in front of the server, set this to true 6 | # It will use the X-Forwarded-For header to get the real IP of the client 7 | real_ip: false 8 | full_url: "/full-stream" 9 | lite_url: "/" 10 | domains_only_url: "/domains-only" 11 | cert_path: "" 12 | cert_key_path: "" 13 | compression_enabled: false 14 | 15 | prometheus: 16 | enabled: true 17 | listen_addr: "0.0.0.0" 18 | listen_port: 8080 19 | metrics_url: "/metrics" 20 | expose_system_metrics: false 21 | real_ip: false 22 | whitelist: 23 | - "127.0.0.1/8" 24 | 25 | general: 26 | # DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. 27 | disable_default_logs: false 28 | # When you want to add logs that are not contained in the log list provided by 29 | # Google (https://www.gstatic.com/ct/log_list/v3/log_list.json), you can add them here. 30 | additional_logs: 31 | - url: https://ct.googleapis.com/logs/us1/mirrors/digicert_nessie2022 32 | operator: "DigiCert" 33 | description: "DigiCert Nessie2022 log" 34 | 35 | # To optimize the performance of the server, you can overwrite the size of different buffers 36 | # For low CPU, low memory machines, you should reduce the buffer sizes to save memory in case the CPU is maxed. 37 | buffer_sizes: 38 | # Buffer for each websocket connection 39 | websocket: 300 40 | # Buffer for each CT log connection 41 | ctlog: 1000 42 | # Combined buffer for the broadcast manager 43 | broadcastmanager: 10000 44 | 45 | # Google regularly updates the log list. If this option is set to true, the server will remove all logs no longer listed in the Google log list. 46 | # This option defaults to true. See https://github.com/d-Rickyy-b/certstream-server-go/issues/51 47 | drop_old_logs: true 48 | 49 | # Options for resuming certificate downloads after restart 50 | recovery: 51 | # If enabled, the server will resume downloading certificates from the last processed and stored index for each log. 52 | # If there is no ct_index_file or for a specific log there is no index entry, the server will start from index 0. 53 | # Be aware that this leads to a massive number of certificates being downloaded. 54 | # Depending on your server's performance and network connection, this could be up to 10.000 certificates per second. 55 | # Make sure your infrastructure can handle this! 56 | enabled: true 57 | # Path to the file where indices are stored. Be aware that a temp file in the same path with the same name and ".tmp" as suffix will be created. 58 | # If there are no write permissions to the path, the server will not be able to store the indices. 59 | ct_index_file: "./ct_index.json" 60 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '43 9 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v3 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v3 73 | -------------------------------------------------------------------------------- /cmd/certpicker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "strings" 11 | "time" 12 | 13 | "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" 14 | "github.com/d-Rickyy-b/certstream-server-go/internal/config" 15 | 16 | ct "github.com/google/certificate-transparency-go" 17 | "github.com/google/certificate-transparency-go/client" 18 | "github.com/google/certificate-transparency-go/jsonclient" 19 | "github.com/google/certificate-transparency-go/scanner" 20 | ) 21 | 22 | var userAgent = fmt.Sprintf("Certstream v%s (github.com/d-Rickyy-b/certstream-server-go)", config.Version) 23 | 24 | func main() { 25 | ctLogFlag := flag.String("log", "", "URL of the CT log - e.g. ct.googleapis.com/logs/eu1/xenon2025h2") 26 | certIDFlag := flag.Int64("cert", 0, "ID of the certificate to fetch from the CT log") 27 | chainFlag := flag.Bool("chain", false, "Include full chain for the certificate") 28 | asDERFlag := flag.Bool("asder", false, "Include DER encoding of the certificate") 29 | flag.Parse() 30 | 31 | ctLog := *ctLogFlag 32 | certID := *certIDFlag 33 | 34 | if ctLog == "" { 35 | log.Fatalln("CT log URL is required") 36 | } 37 | if !strings.HasPrefix(ctLog, "https://") { 38 | ctLog = "https://" + ctLog 39 | } 40 | 41 | // Initialize the http client and json client provided by the ct library 42 | hc := http.Client{Timeout: 30 * time.Second} 43 | jsonClient, e := client.New(ctLog, &hc, jsonclient.Options{UserAgent: userAgent}) 44 | if e != nil { 45 | log.Fatalln("Error creating JSON client:", e) 46 | } 47 | 48 | // Get entries from CT log 49 | c, _ := context.WithTimeout(context.Background(), 10*time.Second) 50 | entries, getEntriesErr := jsonClient.GetRawEntries(c, certID, certID) 51 | if getEntriesErr != nil { 52 | log.Fatalln("Error getting entries from CT log: ", getEntriesErr) 53 | } 54 | 55 | // Loop over entries and pars each one. 56 | for _, leafEntry := range entries.Entries { 57 | rawLogEntry, err := ct.RawLogEntryFromLeaf(certID, &leafEntry) 58 | if err != nil { 59 | log.Fatalln("Error creating raw log entry: ", err) 60 | } 61 | 62 | entry, parseErr := certificatetransparency.ParseCertstreamEntry(rawLogEntry, "N/A", "N/A", ctLog) 63 | if parseErr != nil { 64 | log.Fatalln("Error parsing certstream entry: ", parseErr) 65 | } 66 | 67 | // Check if the entry is a certificate or precertificate 68 | if logEntry, toLogEntryErr := rawLogEntry.ToLogEntry(); toLogEntryErr != nil { 69 | log.Println("Error converting rawLogEntry to logEntry: ", toLogEntryErr) 70 | } else { 71 | matcher := scanner.MatchAll{} 72 | if logEntry.X509Cert != nil && matcher.CertificateMatches(logEntry.X509Cert) { 73 | entry.Data.UpdateType = "X509LogEntry" 74 | } 75 | if logEntry.Precert != nil && matcher.PrecertificateMatches(logEntry.Precert) { 76 | entry.Data.UpdateType = "PrecertLogEntry" 77 | } 78 | } 79 | 80 | // Remove DER encoding and chain if not requested 81 | if !*asDERFlag { 82 | entry.Data.LeafCert.AsDER = "" 83 | for i := range entry.Data.Chain { 84 | entry.Data.Chain[i].AsDER = "" 85 | } 86 | } 87 | 88 | // Remove chain if not requested 89 | if !*chainFlag { 90 | entry.Data.Chain = nil 91 | } 92 | 93 | // Marshal the certificate entry to JSON and pretty print it 94 | result, marshalErr := json.MarshalIndent(entry, "", " ") 95 | if marshalErr != nil { 96 | return 97 | } 98 | 99 | fmt.Println(string(result)) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Options for analysis running. 2 | run: 3 | # The default concurrency value is the number of available CPU. 4 | concurrency: 4 5 | 6 | # Timeout for analysis, e.g. 30s, 5m. 7 | # Default: 1m 8 | timeout: 1m 9 | 10 | # Include test files or not. 11 | # Default: true 12 | tests: false 13 | 14 | # Which files to skip: they will be analyzed, but issues from them won't be reported. 15 | # Default value is empty list, but there is no need to include all autogenerated files, 16 | # we confidently recognize autogenerated files. 17 | 18 | #skip-files: 19 | # - ".*\\.my\\.go$" 20 | # - lib/bad.go 21 | 22 | # Allowed values: readonly|vendor|mod 23 | # By default, it isn't set. 24 | modules-download-mode: readonly 25 | 26 | # Allow multiple parallel golangci-lint instances running. 27 | # If false (default) - golangci-lint acquires file lock on start. 28 | allow-parallel-runners: false 29 | 30 | linters: 31 | # Run only fast linters from enabled linters set (first run won't be fast) 32 | # Default: false 33 | # fast: true 34 | 35 | # Disable all linters. 36 | disable-all: true 37 | 38 | # Enable specific linter 39 | # https://golangci-lint.run/usage/linters/#enabled-by-default 40 | enable: 41 | - bodyclose 42 | - containedctx 43 | # - deadcode 44 | - depguard 45 | - dogsled 46 | - dupl 47 | - dupword 48 | - durationcheck 49 | - errcheck 50 | - errchkjson 51 | - errname 52 | - errorlint 53 | - execinquery 54 | - exhaustive 55 | - exportloopref 56 | - gochecknoinits 57 | - gocritic 58 | - godot 59 | # - godox 60 | - gofmt 61 | # - gofumpt 62 | - goimports 63 | # - golint 64 | # - gomnd 65 | - gomoddirectives 66 | - gomodguard 67 | - goprintffuncname 68 | - gosec 69 | - gosimple 70 | - govet 71 | - grouper 72 | - importas 73 | - ineffassign 74 | - ireturn 75 | - lll 76 | - loggercheck 77 | - maintidx 78 | - makezero 79 | - misspell 80 | - nakedret 81 | - nestif 82 | - nilerr 83 | - nilnil 84 | - nlreturn 85 | - nolintlint 86 | - nosprintfhostport 87 | - paralleltest 88 | - prealloc 89 | - predeclared 90 | - promlinter 91 | - reassign 92 | - revive 93 | - staticcheck 94 | - stylecheck 95 | - tenv 96 | - typecheck 97 | - unconvert 98 | - unparam 99 | - unused 100 | - usestdlibvars 101 | - varnamelen 102 | - wastedassign 103 | - whitespace 104 | - wrapcheck 105 | - wsl 106 | 107 | linters-settings: 108 | nlreturn: 109 | # Size of the block (including return statement that is still "OK") so no return split required. 110 | # Default: 1 111 | block-size: 2 112 | wsl: 113 | enforce-err-cuddling: true 114 | allow-cuddle-declarations: true 115 | allow-assign-and-call: true 116 | allow-cuddle-with-calls: 117 | - log.Println 118 | - log.Printf 119 | - RLock 120 | - RUnlock 121 | - Lock 122 | - Unlock 123 | allow-assign-and-anything: true 124 | gofumpt: 125 | extra-rules: true 126 | lll: 127 | line-length: 160 128 | varnamelen: 129 | ignore-decls: 130 | - w io.Writer 131 | - w io.WriteCloser 132 | - w http.ResponseWriter 133 | - r *http.Request 134 | - r chi.Router 135 | - r *chi.Mux 136 | - i int 137 | 138 | issues: 139 | # List of regexps of issue texts to exclude. 140 | exclude-rules: 141 | # Exclude some linters from running on tests files. 142 | - path: _test\.go 143 | linters: 144 | - nlreturn 145 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/VictoriaMetrics/metrics v1.40.1 h1:FrF5uJRpIVj9fayWcn8xgiI+FYsKGMslzPuOXjdeyR4= 2 | github.com/VictoriaMetrics/metrics v1.40.1/go.mod h1:XE4uudAAIRaJE614Tl5HMrtoEU6+GDZO4QTnNSsZRuA= 3 | github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= 4 | github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= 5 | github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= 6 | github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 7 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 8 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 9 | github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= 10 | github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= 11 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 12 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 13 | github.com/google/trillian v1.7.2 h1:EPBxc4YWY4Ak8tcuhyFleY+zYlbCDCa4Sn24e1Ka8Js= 14 | github.com/google/trillian v1.7.2/go.mod h1:mfQJW4qRH6/ilABtPYNBerVJAJ/upxHLX81zxNQw05s= 15 | github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= 16 | github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 17 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 18 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 19 | github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= 20 | github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= 21 | github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= 22 | github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= 23 | github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= 24 | github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= 25 | golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= 26 | golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= 27 | golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= 28 | golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= 29 | golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= 30 | golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 31 | golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= 32 | golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= 33 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 h1:V1jCN2HBa8sySkR5vLcCSqJSTMv093Rw9EJefhQGP7M= 34 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ= 35 | google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= 36 | google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= 37 | google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= 38 | google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 40 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 41 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 42 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 43 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 44 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 45 | -------------------------------------------------------------------------------- /.github/goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: certstream-server-go 2 | 3 | before: 4 | hooks: 5 | - go mod download 6 | 7 | builds: 8 | - main: ./cmd/certstream-server-go 9 | ldflags: -s -w -X github.com/d-Rickyy-b/certstream-server-go/internal/config.Version={{.Version}} 10 | env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - darwin 15 | - windows 16 | goarch: 17 | - 386 18 | - amd64 19 | - arm 20 | - arm64 21 | ignore: 22 | - goos: darwin 23 | goarch: 386 24 | - goos: darwin 25 | goarch: arm 26 | - goos: windows 27 | goarch: arm 28 | - goos: windows 29 | goarch: arm64 30 | - goos: windows 31 | goarch: 386 32 | checksum: 33 | name_template: '{{.ProjectName}}_{{.Version}}_checksums.txt' 34 | changelog: 35 | disable: true 36 | 37 | dockers: 38 | - image_templates: 39 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' 40 | - '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:{{.Tag}}{{ end }}' 41 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' 42 | - '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest-amd64{{ end }}' 43 | goarch: amd64 44 | use: buildx 45 | extra_files: 46 | - config.sample.yaml 47 | build_flag_templates: 48 | - "--pull" 49 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 50 | - "--label=org.opencontainers.image.description=Certstream server written in Go" 51 | - "--label=org.opencontainers.image.created={{.Date}}" 52 | - "--label=org.opencontainers.image.source=https://github.com/d-Rickyy-b/certstream-server-go" 53 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 54 | - "--label=org.opencontainers.image.version={{.Version}}" 55 | - "--platform=linux/amd64" 56 | 57 | - image_templates: 58 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' 59 | - '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:latest-arm64{{ end }}' 60 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' 61 | - '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest-arm64{{ end }}' 62 | goarch: arm64 63 | use: buildx 64 | extra_files: 65 | - config.sample.yaml 66 | build_flag_templates: 67 | - "--pull" 68 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 69 | - "--label=org.opencontainers.image.description=Certstream server written in Go" 70 | - "--label=org.opencontainers.image.created={{.Date}}" 71 | - "--label=org.opencontainers.image.source=https://github.com/d-Rickyy-b/certstream-server-go" 72 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 73 | - "--label=org.opencontainers.image.version={{.Version}}" 74 | - "--platform=linux/arm64" 75 | 76 | docker_manifests: 77 | - name_template: '0rickyy0/{{.ProjectName}}:{{.Tag}}' 78 | image_templates: 79 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' 80 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' 81 | 82 | - name_template: 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}' 83 | image_templates: 84 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' 85 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' 86 | 87 | - name_template: '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:latest{{ end }}' 88 | image_templates: 89 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' 90 | - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' 91 | 92 | - name_template: '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest{{ end }}' 93 | image_templates: 94 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' 95 | - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' 96 | 97 | archives: 98 | - format: binary 99 | name_template: >- 100 | {{- .ProjectName }}_ 101 | {{- .Version}}_ 102 | {{- if eq .Os "darwin" }}macOS{{- else }}{{ .Os }}{{ end }}_ 103 | {{- if eq .Arch "386" }}i386{{- else }}{{ .Arch }}{{ end }} 104 | -------------------------------------------------------------------------------- /internal/web/broadcastmanager.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "log" 5 | "sync" 6 | 7 | "github.com/d-Rickyy-b/certstream-server-go/internal/models" 8 | ) 9 | 10 | type BroadcastManager struct { 11 | Broadcast chan models.Entry 12 | clients []*client 13 | clientLock sync.RWMutex 14 | } 15 | 16 | // registerClient adds a client to the list of clients of the BroadcastManager. 17 | // The client will receive certificate broadcasts right after registration. 18 | func (bm *BroadcastManager) registerClient(c *client) { 19 | bm.clientLock.Lock() 20 | bm.clients = append(bm.clients, c) 21 | log.Printf("Clients: %d, Capacity: %d\n", len(bm.clients), cap(bm.clients)) 22 | bm.clientLock.Unlock() 23 | } 24 | 25 | // unregisterClient removes a client from the list of clients of the BroadcastManager. 26 | // The client will no longer receive certificate broadcasts right after unregistering. 27 | func (bm *BroadcastManager) unregisterClient(c *client) { 28 | bm.clientLock.Lock() 29 | 30 | for i, client := range bm.clients { 31 | if c == client { 32 | // Copy the last element of the slice to the position of the removed element 33 | // Then remove the last element by re-slicing 34 | bm.clients[i] = bm.clients[len(bm.clients)-1] 35 | bm.clients[len(bm.clients)-1] = nil 36 | bm.clients = bm.clients[:len(bm.clients)-1] 37 | 38 | // Close the broadcast channel of the client, otherwise this leads to a memory leak 39 | close(c.broadcastChan) 40 | 41 | break 42 | } 43 | } 44 | 45 | bm.clientLock.Unlock() 46 | } 47 | 48 | // ClientFullCount returns the current number of clients connected to the service on the `full` endpoint. 49 | func (bm *BroadcastManager) ClientFullCount() (count int64) { 50 | return bm.clientCountByType(SubTypeFull) 51 | } 52 | 53 | // ClientLiteCount returns the current number of clients connected to the service on the `lite` endpoint. 54 | func (bm *BroadcastManager) ClientLiteCount() (count int64) { 55 | return bm.clientCountByType(SubTypeLite) 56 | } 57 | 58 | // ClientDomainsCount returns the current number of clients connected to the service on the `domains-only` endpoint. 59 | func (bm *BroadcastManager) ClientDomainsCount() (count int64) { 60 | return bm.clientCountByType(SubTypeDomain) 61 | } 62 | 63 | // clientCountByType returns the current number of clients connected to the service on the endpoint matching 64 | // the specified SubscriptionType. 65 | func (bm *BroadcastManager) clientCountByType(subType SubscriptionType) (count int64) { 66 | bm.clientLock.RLock() 67 | defer bm.clientLock.RUnlock() 68 | 69 | for _, c := range bm.clients { 70 | if c.subType == subType { 71 | count++ 72 | } 73 | } 74 | 75 | return count 76 | } 77 | 78 | func (bm *BroadcastManager) GetSkippedCerts() map[string]uint64 { 79 | bm.clientLock.RLock() 80 | defer bm.clientLock.RUnlock() 81 | 82 | skippedCerts := make(map[string]uint64, len(bm.clients)) 83 | for _, c := range bm.clients { 84 | skippedCerts[c.name] = c.skippedCerts 85 | } 86 | 87 | return skippedCerts 88 | } 89 | 90 | // broadcaster is run in a goroutine and handles the dispatching of entries to clients. 91 | func (bm *BroadcastManager) broadcaster() { 92 | for { 93 | var data []byte 94 | 95 | entry := <-bm.Broadcast 96 | dataLite := entry.JSONLite() 97 | dataFull := entry.JSON() 98 | dataDomain := entry.JSONDomains() 99 | 100 | bm.clientLock.RLock() 101 | 102 | for _, c := range bm.clients { 103 | switch c.subType { 104 | case SubTypeLite: 105 | data = dataLite 106 | case SubTypeFull: 107 | data = dataFull 108 | case SubTypeDomain: 109 | data = dataDomain 110 | default: 111 | log.Printf("Unknown subscription type '%d' for client '%s'. Skipping this client!\n", c.subType, c.name) 112 | continue 113 | } 114 | 115 | select { 116 | case c.broadcastChan <- data: 117 | default: 118 | // Default case is executed if the client's broadcast channel is full. 119 | c.skippedCerts++ 120 | if c.skippedCerts%1000 == 1 { 121 | log.Printf("Not providing client '%s' with cert because client's buffer is full. The client can't keep up. Skipped certs: %d\n", c.name, c.skippedCerts) 122 | } 123 | } 124 | } 125 | 126 | bm.clientLock.RUnlock() 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /internal/web/client.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "log" 5 | "strings" 6 | "time" 7 | 8 | "github.com/gorilla/websocket" 9 | ) 10 | 11 | const ( 12 | SubTypeFull SubscriptionType = iota 13 | SubTypeLite 14 | SubTypeDomain 15 | ) 16 | 17 | type SubscriptionType int 18 | 19 | // client represents a single client's connection to the server. 20 | type client struct { 21 | conn *websocket.Conn 22 | broadcastChan chan []byte 23 | name string 24 | subType SubscriptionType 25 | skippedCerts uint64 26 | } 27 | 28 | func newClient(conn *websocket.Conn, subType SubscriptionType, name string, certBufferSize int) *client { 29 | return &client{ 30 | conn: conn, 31 | broadcastChan: make(chan []byte, certBufferSize), 32 | name: name, 33 | subType: subType, 34 | } 35 | } 36 | 37 | // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. 38 | func (c *client) broadcastHandler() { 39 | writeWait := 60 * time.Second 40 | pingTicker := time.NewTicker(30 * time.Second) 41 | 42 | defer func() { 43 | log.Println("Closing broadcast handler for client:", c.conn.RemoteAddr()) 44 | 45 | pingTicker.Stop() 46 | 47 | _ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) 48 | _ = c.conn.WriteMessage(websocket.CloseMessage, []byte{}) 49 | _ = c.conn.Close() 50 | }() 51 | 52 | for { 53 | select { 54 | case <-pingTicker.C: 55 | _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) 56 | 57 | if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { 58 | return 59 | } 60 | case message := <-c.broadcastChan: 61 | _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) 62 | 63 | w, err := c.conn.NextWriter(websocket.TextMessage) 64 | if err != nil { 65 | log.Printf("Error while getting next writer: %v\n", err) 66 | return 67 | } 68 | 69 | _, writeErr := w.Write(message) 70 | if writeErr != nil { 71 | log.Printf("Error while writing: %v\n", writeErr) 72 | } 73 | 74 | if closeErr := w.Close(); closeErr != nil { 75 | log.Printf("Error while closing: %v\n", closeErr) 76 | return 77 | } 78 | } 79 | } 80 | } 81 | 82 | // listenWebsocket is running in the background on a goroutine and listens for messages from the client. 83 | // It responds to ping messages with a pong message. It closes the connection if the client sends 84 | // a close message or no ping is received within 65 seconds. 85 | func (c *client) listenWebsocket() { 86 | defer func() { 87 | _ = c.conn.Close() 88 | ClientHandler.unregisterClient(c) 89 | }() 90 | 91 | readWait := 65 * time.Second 92 | 93 | c.conn.SetReadLimit(512) 94 | _ = c.conn.SetReadDeadline(time.Now().Add(readWait)) 95 | 96 | defaultPingHandler := c.conn.PingHandler() 97 | 98 | c.conn.SetPingHandler(func(appData string) error { 99 | // Ping received - reset the deadline 100 | err := c.conn.SetReadDeadline(time.Now().Add(readWait)) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | return defaultPingHandler(appData) 106 | }) 107 | 108 | c.conn.SetPongHandler(func(string) error { 109 | // Pong received - reset the deadline 110 | err := c.conn.SetReadDeadline(time.Now().Add(readWait)) 111 | return err 112 | }) 113 | 114 | // Handle messages from the client 115 | for { 116 | // ignore any message sent from clients - we only handle errors (aka. disconnects) 117 | _, _, readErr := c.conn.ReadMessage() 118 | if readErr != nil { 119 | if websocket.IsUnexpectedCloseError(readErr, websocket.CloseGoingAway, websocket.CloseNormalClosure) { 120 | log.Printf("Unexpected websocket close error: %v\n", readErr) 121 | } 122 | 123 | if strings.Contains(strings.ToLower(readErr.Error()), "i/o timeout") { 124 | log.Printf("No ping received from client: %v\n", c.conn.RemoteAddr()) 125 | closeMessage := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "No ping received!") 126 | c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(5*time.Second)) //nolint:errcheck 127 | } else if strings.Contains(strings.ToLower(readErr.Error()), "an existing connection was forcibly closed by the remote host") { 128 | log.Printf("Connection to client lost: %v\n", c.conn.RemoteAddr()) 129 | } 130 | 131 | log.Printf("Disconnecting client %v!\n", c.conn.RemoteAddr()) 132 | 133 | break 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /internal/metrics/prometheus.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" 11 | "github.com/d-Rickyy-b/certstream-server-go/internal/web" 12 | 13 | "github.com/VictoriaMetrics/metrics" 14 | ) 15 | 16 | var ( 17 | ctLogMetricsInitialized = false 18 | ctLogMetricsInitMutex = &sync.Mutex{} 19 | 20 | tempCertMetricsLastRefreshed = time.Time{} 21 | tempCertMetrics = certificatetransparency.CTMetrics{} 22 | tempCertMetricsMutex = &sync.RWMutex{} 23 | 24 | // Number of currently connected clients. 25 | fullClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"full\"}", func() float64 { 26 | return float64(web.ClientHandler.ClientFullCount()) 27 | }) 28 | liteClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"lite\"}", func() float64 { 29 | return float64(web.ClientHandler.ClientLiteCount()) 30 | }) 31 | domainClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"domain\"}", func() float64 { 32 | return float64(web.ClientHandler.ClientDomainsCount()) 33 | }) 34 | 35 | // Number of certificates processed by the CT watcher. 36 | processedCertificates = metrics.NewGauge("certstreamservergo_certificates_total{type=\"regular\"}", func() float64 { 37 | return float64(certificatetransparency.GetProcessedCerts()) 38 | }) 39 | processedPreCertificates = metrics.NewGauge("certstreamservergo_certificates_total{type=\"precert\"}", func() float64 { 40 | return float64(certificatetransparency.GetProcessedPrecerts()) 41 | }) 42 | ) 43 | 44 | // WritePrometheus provides an easy way to write metrics to a writer. 45 | func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { 46 | ctLogMetricsInitMutex.Lock() 47 | if !ctLogMetricsInitialized { 48 | initCtLogMetrics() 49 | } 50 | ctLogMetricsInitMutex.Unlock() 51 | 52 | getSkippedCertMetrics() 53 | 54 | metrics.WritePrometheus(w, exposeProcessMetrics) 55 | } 56 | 57 | // For having metrics regarding each individual CT log, we need to register them manually. 58 | // initCtLogMetrics fetches all the CT Logs and registers one metric per log. 59 | func initCtLogMetrics() { 60 | logs := certificatetransparency.GetLogOperators() 61 | 62 | for operator, urls := range logs { 63 | operator := operator // Copy variable to new scope 64 | 65 | for i := range urls { 66 | url := urls[i] 67 | name := fmt.Sprintf("certstreamservergo_certs_by_log_total{url=\"%s\",operator=\"%s\"}", url, operator) 68 | metrics.NewGauge(name, func() float64 { 69 | return float64(getCertCountForLog(operator, url)) 70 | }) 71 | } 72 | } 73 | 74 | if len(logs) > 0 { 75 | ctLogMetricsInitialized = true 76 | } 77 | } 78 | 79 | // getCertCountForLog returns the number of certificates processed from a specific CT log. 80 | // It caches the result for 5 seconds. Subsequent calls to this method will return the cached result. 81 | func getCertCountForLog(operatorName, logname string) int64 { 82 | tempCertMetricsMutex.Lock() 83 | defer tempCertMetricsMutex.Unlock() 84 | 85 | // Add some caching to avoid having to lock the mutex every time 86 | if time.Since(tempCertMetricsLastRefreshed) > time.Second*5 { 87 | tempCertMetricsLastRefreshed = time.Now() 88 | tempCertMetrics = certificatetransparency.GetCertMetrics() 89 | } 90 | 91 | return tempCertMetrics[operatorName][logname] 92 | } 93 | 94 | // getSkippedCertMetrics gets the number of skipped certificates for each client and creates metrics for it. 95 | // It also removes metrics for clients that are not connected anymore. 96 | func getSkippedCertMetrics() { 97 | skippedCerts := web.ClientHandler.GetSkippedCerts() 98 | for clientName := range skippedCerts { 99 | // Get or register a new counter for each client 100 | metricName := fmt.Sprintf("certstreamservergo_skipped_certs{client=\"%s\"}", clientName) 101 | c := metrics.GetOrCreateCounter(metricName) 102 | c.Set(skippedCerts[clientName]) 103 | } 104 | 105 | // Remove all metrics that are not in the list of current client skipped cert metrics 106 | // Get a list of current client skipped cert metrics 107 | for _, metricName := range metrics.ListMetricNames() { 108 | if !strings.HasPrefix(metricName, "certstreamservergo_skipped_certs") { 109 | continue 110 | } 111 | 112 | clientName := strings.TrimPrefix(metricName, "certstreamservergo_skipped_certs{client=\"") 113 | clientName = strings.TrimSuffix(clientName, "\"}") 114 | 115 | // Check if the registered metric is in the list of current client skipped cert metrics 116 | // If not, unregister the metric 117 | _, exists := skippedCerts[clientName] 118 | if !exists { 119 | metrics.UnregisterMetric(metricName) 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /internal/models/certstream.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "log" 7 | ) 8 | 9 | type Entry struct { 10 | Data Data `json:"data"` 11 | MessageType string `json:"message_type"` 12 | cachedJSON []byte 13 | cachedJSONLite []byte 14 | } 15 | 16 | // Clone returns a new copy of the Entry. 17 | func (e *Entry) Clone() Entry { 18 | return Entry{ 19 | Data: e.Data, 20 | MessageType: e.MessageType, 21 | cachedJSON: e.cachedJSON, 22 | cachedJSONLite: e.cachedJSONLite, 23 | } 24 | } 25 | 26 | // JSON returns the json encoded Entry as byte slice and caches it for later access. 27 | func (e *Entry) JSON() []byte { 28 | if len(e.cachedJSON) > 0 { 29 | return e.cachedJSON 30 | } 31 | e.cachedJSON = e.entryToJSONBytes() 32 | 33 | return e.cachedJSON 34 | } 35 | 36 | // JSONNoCache returns the json encoded Entry as byte slice without caching it. 37 | func (e *Entry) JSONNoCache() []byte { 38 | return e.entryToJSONBytes() 39 | } 40 | 41 | // JSONLite does the same as JSON() but removes the chain and cert's DER representation. 42 | func (e *Entry) JSONLite() []byte { 43 | if len(e.cachedJSONLite) > 0 { 44 | return e.cachedJSONLite 45 | } 46 | e.cachedJSONLite = e.JSONLiteNoCache() 47 | 48 | return e.cachedJSONLite 49 | } 50 | 51 | // JSONLiteNoCache does the same as JSONNoCache() but removes the chain and cert's DER representation. 52 | func (e *Entry) JSONLiteNoCache() []byte { 53 | newEntry := e.Clone() 54 | newEntry.Data.Chain = nil 55 | newEntry.Data.LeafCert.AsDER = "" 56 | 57 | return newEntry.entryToJSONBytes() 58 | } 59 | 60 | // JSONDomains returns the json encoded domains (DomainsEntry) as byte slice. 61 | func (e *Entry) JSONDomains() []byte { 62 | domainsEntry := DomainsEntry{ 63 | Data: e.Data.LeafCert.AllDomains, 64 | MessageType: "dns_entries", 65 | } 66 | 67 | domainsEntryBytes, err := json.Marshal(domainsEntry) 68 | if err != nil { 69 | log.Println(err) 70 | } 71 | 72 | return domainsEntryBytes 73 | } 74 | 75 | // entryToJSONBytes encodes an Entry to a JSON byte slice. 76 | func (e *Entry) entryToJSONBytes() []byte { 77 | buf := bytes.Buffer{} 78 | enc := json.NewEncoder(&buf) 79 | enc.SetEscapeHTML(false) 80 | 81 | err := enc.Encode(e) 82 | if err != nil { 83 | log.Println(err) 84 | } 85 | 86 | return buf.Bytes() 87 | } 88 | 89 | type Data struct { 90 | CertIndex uint64 `json:"cert_index"` 91 | CertLink string `json:"cert_link"` 92 | Chain []LeafCert `json:"chain,omitempty"` 93 | LeafCert LeafCert `json:"leaf_cert"` 94 | Seen float64 `json:"seen"` 95 | Source Source `json:"source"` 96 | UpdateType string `json:"update_type"` 97 | } 98 | 99 | type Source struct { 100 | Name string `json:"name"` 101 | URL string `json:"url"` 102 | Operator string `json:"-"` 103 | NormalizedURL string `json:"-"` 104 | } 105 | 106 | type LeafCert struct { 107 | AllDomains []string `json:"all_domains"` 108 | AsDER string `json:"as_der,omitempty"` 109 | Extensions Extensions `json:"extensions"` 110 | Fingerprint string `json:"fingerprint"` 111 | SHA1 string `json:"sha1"` 112 | SHA256 string `json:"sha256"` 113 | NotAfter int64 `json:"not_after"` 114 | NotBefore int64 `json:"not_before"` 115 | SerialNumber string `json:"serial_number"` 116 | SignatureAlgorithm string `json:"signature_algorithm"` 117 | Subject Subject `json:"subject"` 118 | Issuer Subject `json:"issuer"` 119 | IsCA bool `json:"is_ca"` 120 | } 121 | 122 | type Subject struct { 123 | C *string `json:"C"` 124 | CN *string `json:"CN"` 125 | L *string `json:"L"` 126 | O *string `json:"O"` 127 | OU *string `json:"OU"` 128 | ST *string `json:"ST"` 129 | Aggregated *string `json:"aggregated"` 130 | EmailAddress *string `json:"email_address"` 131 | } 132 | 133 | type Extensions struct { 134 | AuthorityInfoAccess *string `json:"authorityInfoAccess,omitempty"` 135 | AuthorityKeyIdentifier *string `json:"authorityKeyIdentifier,omitempty"` 136 | BasicConstraints *string `json:"basicConstraints,omitempty"` 137 | CertificatePolicies *string `json:"certificatePolicies,omitempty"` 138 | CtlSignedCertificateTimestamp *string `json:"ctlSignedCertificateTimestamp,omitempty"` 139 | ExtendedKeyUsage *string `json:"extendedKeyUsage,omitempty"` 140 | KeyUsage *string `json:"keyUsage,omitempty"` 141 | SubjectAltName *string `json:"subjectAltName,omitempty"` 142 | SubjectKeyIdentifier *string `json:"subjectKeyIdentifier,omitempty"` 143 | CTLPoisonByte bool `json:"ctlPoisonByte,omitempty"` 144 | } 145 | 146 | type DomainsEntry struct { 147 | Data []string `json:"data"` 148 | MessageType string `json:"message_type"` 149 | } 150 | -------------------------------------------------------------------------------- /internal/certstream/certstream.go: -------------------------------------------------------------------------------- 1 | package certstream 2 | 3 | // The certstream package provides the main entry point for the certstream-server-go application. 4 | // It initializes the webserver and the watcher for the certificate transparency logs. 5 | // It also handles signals for graceful shutdown of the server. 6 | 7 | import ( 8 | "log" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | 13 | "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" 14 | "github.com/d-Rickyy-b/certstream-server-go/internal/config" 15 | "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" 16 | "github.com/d-Rickyy-b/certstream-server-go/internal/web" 17 | ) 18 | 19 | type Certstream struct { 20 | webserver *web.WebServer 21 | metricsServer *web.WebServer 22 | watcher *certificatetransparency.Watcher 23 | config config.Config 24 | } 25 | 26 | func NewRawCertstream(config config.Config) *Certstream { 27 | cs := Certstream{} 28 | cs.config = config 29 | 30 | return &cs 31 | } 32 | 33 | // NewCertstreamServer creates a new Certstream server from a config struct. 34 | func NewCertstreamServer(config config.Config) (*Certstream, error) { 35 | cs := Certstream{} 36 | cs.config = config 37 | 38 | // Initialize the webserver used for the websocket server 39 | webserver := web.NewWebsocketServer(config.Webserver.ListenAddr, config.Webserver.ListenPort, config.Webserver.CertPath, config.Webserver.CertKeyPath) 40 | cs.webserver = webserver 41 | 42 | // Setup metrics server 43 | cs.setupMetrics(webserver) 44 | 45 | return &cs, nil 46 | } 47 | 48 | // NewCertstreamFromConfigFile creates a new Certstream server from a config file. 49 | func NewCertstreamFromConfigFile(configPath string) (*Certstream, error) { 50 | conf, err := config.ReadConfig(configPath) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return NewCertstreamServer(conf) 56 | } 57 | 58 | // setupMetrics configures the webserver to handle prometheus metrics according to the config. 59 | func (cs *Certstream) setupMetrics(webserver *web.WebServer) { 60 | if cs.config.Prometheus.Enabled { 61 | // If prometheus is enabled, and interface is either unconfigured or same as webserver config, use existing webserver 62 | if (cs.config.Prometheus.ListenAddr == "" || cs.config.Prometheus.ListenAddr == cs.config.Webserver.ListenAddr) && 63 | (cs.config.Prometheus.ListenPort == 0 || cs.config.Prometheus.ListenPort == cs.config.Webserver.ListenPort) { 64 | log.Println("Starting prometheus server on same interface as webserver") 65 | webserver.RegisterPrometheus(cs.config.Prometheus.MetricsURL, metrics.WritePrometheus) 66 | } else { 67 | log.Println("Starting prometheus server on new interface") 68 | cs.metricsServer = web.NewMetricsServer(cs.config.Prometheus.ListenAddr, cs.config.Prometheus.ListenPort, cs.config.Prometheus.CertPath, cs.config.Prometheus.CertKeyPath) 69 | cs.metricsServer.RegisterPrometheus(cs.config.Prometheus.MetricsURL, metrics.WritePrometheus) 70 | } 71 | } 72 | } 73 | 74 | // Start starts the webserver and the watcher. 75 | // This is a blocking function that will run until the server is stopped. 76 | func (cs *Certstream) Start() { 77 | log.Printf("Starting certstream-server-go v%s\n", config.Version) 78 | 79 | // handle signals in a separate goroutine 80 | signals := make(chan os.Signal, 1) 81 | signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 82 | go signalHandler(signals, cs.Stop) 83 | 84 | // If there is no watcher initialized, create a new one 85 | if cs.watcher == nil { 86 | cs.watcher = &certificatetransparency.Watcher{} 87 | } 88 | 89 | // Start webserver and metrics server 90 | if cs.webserver == nil { 91 | log.Fatalln("Webserver not initialized! Exiting...") 92 | } 93 | 94 | go cs.webserver.Start() 95 | 96 | if cs.metricsServer != nil { 97 | go cs.metricsServer.Start() 98 | } 99 | 100 | // Start the watcher - this is a blocking function 101 | cs.watcher.Start() 102 | } 103 | 104 | // Stop stops the watcher and the webserver. 105 | func (cs *Certstream) Stop() { 106 | if cs.watcher != nil { 107 | cs.watcher.Stop() 108 | } 109 | 110 | if cs.webserver != nil { 111 | cs.webserver.Stop() 112 | } 113 | 114 | if cs.metricsServer != nil { 115 | cs.metricsServer.Stop() 116 | } 117 | } 118 | 119 | // CreateIndexFile creates the index file for the certificate transparency logs. 120 | // It gets only called when the CLI flag --create-index-file is set. 121 | func (cs *Certstream) CreateIndexFile() error { 122 | // If there is no watcher initialized, create a new one 123 | if cs.watcher == nil { 124 | cs.watcher = &certificatetransparency.Watcher{} 125 | } 126 | 127 | return cs.watcher.CreateIndexFile(cs.config.General.Recovery.CTIndexFile) 128 | } 129 | 130 | // signalHandler listens for signals in order to gracefully shut down the server. 131 | // Executes the callback function when a signal is received. 132 | func signalHandler(signals chan os.Signal, callback func()) { 133 | log.Println("Listening for signals...") 134 | sig := <-signals 135 | log.Printf("Received signal %v. Shutting down...\n", sig) 136 | callback() 137 | os.Exit(0) 138 | } 139 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | ### Added 10 | - Ability to store and resume processing of certs from where it left off after a restart - see sample config "recovery" (#49) 11 | - New CLI switch for creating an index file from a CT log (#49) 12 | - Check for retired CT logs and prevent them from being watched / stop watching them (#77) 13 | - Accept websocket connections from all origins 14 | - Option to disable the default logs provided by Google - see sample config "disable_default_logs" 15 | ### Changed 16 | ### Removed 17 | - Non-functional Dodo log from sample config (#78) 18 | ### Fixed 19 | - Properly remove stopped ct log workers (#74) 20 | - Added missing fields certificatePolicies and ctlPoisonByte (#85) 21 | - Prevent race condition caused by simultaneous rw access to logmetrics 22 | ### Docs 23 | 24 | ## [v1.8.2] - 2025-11-22 25 | ### Fixed 26 | - Added missing fields certificatePolicies and ctlPoisonByte (#85) 27 | 28 | ## [v1.8.1] - 2025-05-04 29 | ### Fixed 30 | - No longer reject URLs with trailing slashes defined in the `additional_logs` config (#62) 31 | - When using `drop_old_logs` in the config, the server won't remove logs defined in `additional_logs` anymore (#64) 32 | 33 | ## [v1.8.0] - 2025-05-03 34 | ### Security 35 | - Close several CVEs in x/crypto and x/net dependencies (#59) 36 | 37 | ### Added 38 | - New CLI tool for fetching certificates from a CT log (#47) 39 | - Ability to add custom CT logs to the config (#56) 40 | - Remove old CT logs as soon as they are removed from the Google CT Loglist (#60) 41 | - New configuration for buffer sizes (#58) 42 | 43 | ### Fixed 44 | - Properly handle IPv6 addresses in config (#61) 45 | 46 | ## [1.7.1] - 2025-05-03 47 | ### Fixed 48 | - Properly handle IPv6 addresses in config (#61) 49 | 50 | ## [1.7.0] - 2024-08-20 51 | ### Added 52 | - Support for websocket compression - disabled by default (#40) 53 | - Support for non-browsers by implementing server initiated heartbeats (#39) 54 | - Start new ct-watchers as new ct logs become available (#42) 55 | - More logging to document currently watched logs (03d878e) 56 | 57 | ### Changed 58 | - Changed log output to be better grepable (5c055cc) 59 | - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d) 60 | 61 | ### Fixed 62 | - Fixed a possible race condition when accessing metrics 63 | 64 | ## [1.6.0] - 2024-03-05 65 | ### Added 66 | - New metric for skipped certs per client (#34) 67 | 68 | ## [1.5.2] - 2024-02-17 69 | ### Fixed 70 | - Fixed an issue with ip whitelists for the websocket server (#33) 71 | 72 | ## [1.5.1] - 2024-01-18 73 | ### Fixed 74 | - Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null 75 | 76 | ## [1.5.0] - 2023-12-21 77 | ### Added 78 | - New `-version` switch to print version and exit afterwards 79 | - Print version on every run of the tool 80 | - Count and log number of skipped certificates per client 81 | 82 | ### Changed 83 | - Update to chi/v5 84 | - Update ct-watcher timeout from 5 to 30 seconds 85 | 86 | ### Fixed 87 | - Prevent invalid subscription types to be used 88 | - Kill connection after broadcasthandler was stopped 89 | 90 | ## [1.4.0] - 2023-11-29 91 | ### Added 92 | - Config option to use X-Forwarded-For or X-Real-IP header as client IP 93 | - Config option to whitelist client IPs for both websocket and metrics endpoints 94 | - Config option to enable system metrics (cpu, memory, etc.) 95 | 96 | ## [1.3.2] - 2023-11-28 97 | ### Fixed 98 | - Memory leak related to clients disconnecting from the websocket not being handled properly 99 | 100 | ## [1.3.1] - 2023-09-18 101 | ### Changed 102 | - Updated config.sample.yaml to run both certstream and prometheus metrics on same socket 103 | 104 | ### Docs 105 | - Fixed wrong docker command in readme 106 | 107 | ## [1.3.0] - 2023-04-11 108 | ### Added 109 | - Calculate and display Sha256 sum of certificate 110 | 111 | ### Changed 112 | - Update dependencies 113 | - Better logging for CT log errors 114 | 115 | ### Fixed 116 | - End execution after all workers stopped 117 | - Implement timeout for the http client 118 | - Keep ct watcher from crashing upon a connection reset from server 119 | 120 | ## [1.2.2] - 2023-01-10 121 | ### Added 122 | - Two docker-compose files 123 | - Check for presence of .yml or .yaml files in the current directory 124 | 125 | ### Fixed 126 | - Handle sudden disconnects of CT logs 127 | 128 | ### Docs 129 | - Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics) 130 | 131 | ## [1.2.1] - 2022-12-16 132 | ### Changed 133 | - Updated ci pipeline to use new setup-go and checkout actions 134 | - Use correct package name `github.com/d-Rickyy-b/certstream-server-go` 135 | 136 | ## [1.2.0] - 2022-12-15 137 | ### Added 138 | - Log x-Forwarded-For header for requests 139 | - More logging for certain error situations 140 | - Add operator to ct log cert count metrics 141 | 142 | ### Changed 143 | - Updated certificate-transparency-go dependency to v1.1.4 144 | - Code improvements, adhering to styleguide 145 | - Rename module to certstream-server-go 146 | - Use log_list.json instead of all_logs_list.json 147 | 148 | ## [1.1.0] - 2022-10-19 149 | Fix for missing loglist urls. 150 | 151 | ### Fixed 152 | Fixed the connection issue due to the offline Google loglist urls. 153 | 154 | ## [1.0.0] - 2022-08-08 155 | Initial release! First stable version of certstream-server-go is published as v1.0.0 156 | 157 | [unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.2...HEAD 158 | [1.8.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.1...v1.8.2 159 | [1.8.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.8.0...v1.8.1 160 | [1.8.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.7.1...v1.8.0 161 | [1.7.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.7.0...v1.7.1 162 | [1.7.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.6.0...v1.7.0 163 | [1.6.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.2...v1.6.0 164 | [1.5.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.1...v1.5.2 165 | [1.5.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.0...v1.5.1 166 | [1.5.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.4.0...v1.5.0 167 | [1.4.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.2...v1.4.0 168 | [1.3.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.1...v1.3.2 169 | [1.3.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.0...v1.3.1 170 | [1.3.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.2...v1.3.0 171 | [1.2.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.1...v1.2.2 172 | [1.2.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.0...v1.2.1 173 | [1.2.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.1.0...v1.2.0 174 | [1.1.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.0.0...v1.1.0 175 | [1.0.0]: https://github.com/d-Rickyy-b/certstream-server-go/tree/v1.0.0 176 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "os" 7 | "path/filepath" 8 | "regexp" 9 | "strings" 10 | 11 | "gopkg.in/yaml.v3" 12 | ) 13 | 14 | var ( 15 | AppConfig Config 16 | Version = "1.8.1" 17 | ) 18 | 19 | type ServerConfig struct { 20 | ListenAddr string `yaml:"listen_addr"` 21 | ListenPort int `yaml:"listen_port"` 22 | CertPath string `yaml:"cert_path"` 23 | CertKeyPath string `yaml:"cert_key_path"` 24 | RealIP bool `yaml:"real_ip"` 25 | Whitelist []string `yaml:"whitelist"` 26 | } 27 | 28 | type LogConfig struct { 29 | Operator string `yaml:"operator"` 30 | URL string `yaml:"url"` 31 | Description string `yaml:"description"` 32 | } 33 | 34 | type BufferSizes struct { 35 | Websocket int `yaml:"websocket"` 36 | CTLog int `yaml:"ctlog"` 37 | BroadcastManager int `yaml:"broadcastmanager"` 38 | } 39 | 40 | type Config struct { 41 | Webserver struct { 42 | ServerConfig `yaml:",inline"` 43 | FullURL string `yaml:"full_url"` 44 | LiteURL string `yaml:"lite_url"` 45 | DomainsOnlyURL string `yaml:"domains_only_url"` 46 | CompressionEnabled bool `yaml:"compression_enabled"` 47 | } 48 | Prometheus struct { 49 | ServerConfig `yaml:",inline"` 50 | Enabled bool `yaml:"enabled"` 51 | MetricsURL string `yaml:"metrics_url"` 52 | ExposeSystemMetrics bool `yaml:"expose_system_metrics"` 53 | } 54 | General struct { 55 | // DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled. 56 | DisableDefaultLogs bool `yaml:"disable_default_logs"` 57 | // AdditionalLogs contains additional logs provided by the user that can be used in addition to the default logs. 58 | AdditionalLogs []LogConfig `yaml:"additional_logs"` 59 | BufferSizes BufferSizes `yaml:"buffer_sizes"` 60 | DropOldLogs *bool `yaml:"drop_old_logs"` 61 | Recovery struct { 62 | Enabled bool `yaml:"enabled"` 63 | CTIndexFile string `yaml:"ct_index_file"` 64 | } `yaml:"recovery"` 65 | } 66 | } 67 | 68 | // ReadConfig reads the config file and returns a filled Config struct. 69 | func ReadConfig(configPath string) (Config, error) { 70 | log.Printf("Reading config file '%s'...\n", configPath) 71 | 72 | conf, parseErr := parseConfigFromFile(configPath) 73 | if parseErr != nil { 74 | log.Fatalln("Error while parsing yaml file:", parseErr) 75 | } 76 | 77 | if !validateConfig(conf) { 78 | log.Fatalln("Invalid config") 79 | } 80 | AppConfig = *conf 81 | 82 | return *conf, nil 83 | } 84 | 85 | // parseConfigFromFile reads the config file as bytes and passes it to parseConfigFromBytes. 86 | // It returns a filled Config struct. 87 | func parseConfigFromFile(configFile string) (*Config, error) { 88 | if configFile == "" { 89 | configFile = "config.yml" 90 | } 91 | 92 | // Check if the file exists 93 | absPath, err := filepath.Abs(configFile) 94 | if err != nil { 95 | log.Printf("Couldn't convert to absolute path: '%s'\n", configFile) 96 | return &Config{}, err 97 | } 98 | 99 | if _, statErr := os.Stat(absPath); os.IsNotExist(statErr) { 100 | log.Printf("Config file '%s' does not exist\n", absPath) 101 | ext := filepath.Ext(absPath) 102 | absPath = strings.TrimSuffix(absPath, ext) 103 | 104 | switch ext { 105 | case ".yaml": 106 | absPath += ".yml" 107 | case ".yml": 108 | absPath += ".yaml" 109 | default: 110 | log.Printf("Config file '%s' does not have a valid extension\n", configFile) 111 | return &Config{}, statErr 112 | } 113 | 114 | if _, secondStatErr := os.Stat(absPath); os.IsNotExist(secondStatErr) { 115 | log.Printf("Config file '%s' does not exist\n", absPath) 116 | return &Config{}, secondStatErr 117 | } 118 | } 119 | log.Printf("File '%s' exists\n", absPath) 120 | 121 | yamlFileContent, readErr := os.ReadFile(absPath) 122 | if readErr != nil { 123 | return &Config{}, readErr 124 | } 125 | 126 | conf, parseErr := parseConfigFromBytes(yamlFileContent) 127 | if parseErr != nil { 128 | return &Config{}, parseErr 129 | } 130 | 131 | return conf, nil 132 | } 133 | 134 | // parseConfigFromBytes parses the config bytes and returns a filled Config struct. 135 | func parseConfigFromBytes(data []byte) (*Config, error) { 136 | var config Config 137 | 138 | err := yaml.Unmarshal(data, &config) 139 | if err != nil { 140 | return &config, err 141 | } 142 | 143 | return &config, nil 144 | } 145 | 146 | // validateConfig validates the config values and sets defaults for missing values. 147 | func validateConfig(config *Config) bool { 148 | // Still matches invalid IP addresses but good enough for detecting completely wrong formats 149 | URLPathRegex := regexp.MustCompile(`^(/[a-zA-Z0-9\-._]+)+$`) 150 | URLRegex := regexp.MustCompile(`^https?://[a-zA-Z0-9\-._]+(:[0-9]+)?(/[a-zA-Z0-9\-._]+)*/?$`) 151 | 152 | // Check webserver config 153 | if config.Webserver.ListenAddr == "" || net.ParseIP(config.Webserver.ListenAddr) == nil { 154 | log.Fatalln("Webhook listen IP is not a valid IP: ", config.Webserver.ListenAddr) 155 | return false 156 | } 157 | 158 | if config.Webserver.ListenPort == 0 { 159 | log.Fatalln("Webhook listen port is not set") 160 | return false 161 | } 162 | 163 | if config.Webserver.FullURL == "" || !URLPathRegex.MatchString(config.Webserver.FullURL) { 164 | log.Println("Webhook full URL is not set or does not match pattern '/...'") 165 | config.Webserver.FullURL = "/full-stream" 166 | } 167 | 168 | if config.Webserver.LiteURL == "" || !URLPathRegex.MatchString(config.Webserver.FullURL) { 169 | log.Println("Webhook lite URL is not set or does not match pattern '/...'") 170 | config.Webserver.LiteURL = "/" 171 | } 172 | 173 | if config.Webserver.DomainsOnlyURL == "" || !URLPathRegex.MatchString(config.Webserver.DomainsOnlyURL) { 174 | log.Println("Webhook domains only URL is not set or does not match pattern '/...'") 175 | config.Webserver.FullURL = "/domains-only" 176 | } 177 | 178 | if config.Webserver.FullURL == config.Webserver.LiteURL { 179 | log.Fatalln("Webhook full URL is the same as lite URL - please fix the config!") 180 | } 181 | 182 | if config.Webserver.DomainsOnlyURL == "" { 183 | config.Webserver.FullURL = "/domains-only" 184 | } 185 | 186 | if config.Prometheus.Enabled { 187 | if config.Prometheus.ListenAddr == "" || net.ParseIP(config.Prometheus.ListenAddr) == nil { 188 | log.Fatalln("Metrics export IP is not a valid IP") 189 | return false 190 | } 191 | 192 | if config.Prometheus.ListenPort == 0 { 193 | log.Fatalln("Metrics export port is not set") 194 | return false 195 | } 196 | 197 | if config.Prometheus.Whitelist == nil { 198 | config.Prometheus.Whitelist = []string{} 199 | } 200 | 201 | // Check if IPs in whitelist match pattern 202 | for _, ip := range config.Prometheus.Whitelist { 203 | if net.ParseIP(ip) == nil { 204 | // Provided entry is not an IP, check if it's a CIDR range 205 | _, _, err := net.ParseCIDR(ip) 206 | if err != nil { 207 | log.Fatalln("Invalid IP in metrics whitelist: ", ip) 208 | return false 209 | } 210 | } 211 | } 212 | } 213 | 214 | var validLogs []LogConfig 215 | if len(config.General.AdditionalLogs) > 0 { 216 | for _, ctLog := range config.General.AdditionalLogs { 217 | if !URLRegex.MatchString(ctLog.URL) { 218 | log.Println("Ignoring invalid additional log URL: ", ctLog.URL) 219 | continue 220 | } 221 | 222 | validLogs = append(validLogs, ctLog) 223 | } 224 | } else if len(config.General.AdditionalLogs) == 0 && config.General.DisableDefaultLogs { 225 | log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.") 226 | } 227 | 228 | config.General.AdditionalLogs = validLogs 229 | 230 | if config.General.BufferSizes.Websocket <= 0 { 231 | config.General.BufferSizes.Websocket = 300 232 | } 233 | 234 | if config.General.BufferSizes.CTLog <= 0 { 235 | config.General.BufferSizes.CTLog = 1000 236 | } 237 | 238 | if config.General.BufferSizes.BroadcastManager <= 0 { 239 | config.General.BufferSizes.BroadcastManager = 10000 240 | } 241 | 242 | // If the cleanup flag is not set, default to true 243 | if config.General.DropOldLogs == nil { 244 | log.Println("drop_old_logs is not set, defaulting to true") 245 | defaultCleanup := true 246 | config.General.DropOldLogs = &defaultCleanup 247 | } 248 | 249 | if config.General.Recovery.Enabled && config.General.Recovery.CTIndexFile == "" { 250 | log.Println("Recovery enabled but no index file specified. Defaulting to ./ct_index.json") 251 | config.General.Recovery.CTIndexFile = "./ct_index.json" 252 | } 253 | 254 | return true 255 | } 256 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![certstream-server-go logo](https://github.com/d-Rickyy-b/certstream-server-go/blob/master/docs/img/certstream-server-go_logo.png?raw=true) 2 | 3 | # Certstream Server Go 4 | 5 | [![build](https://github.com/d-Rickyy-b/certstream-server-go/actions/workflows/release_build.yml/badge.svg)](https://github.com/d-Rickyy-b/certstream-server-go/actions/workflows/release_build.yml) 6 | [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/0rickyy0/certstream-server-go?label=docker&sort=semver)](https://hub.docker.com/repository/docker/0rickyy0/certstream-server-go) 7 | [![Go Reference](https://pkg.go.dev/badge/github.com/d-Rickyy-b/certstream-server-go.svg)](https://pkg.go.dev/github.com/d-Rickyy-b/certstream-server-go) 8 | 9 | This project aims to be a drop-in replacement for the [certstream server](https://github.com/CaliDog/certstream-server/) by Calidog. This tool aggregates, parses, and streams certificate data from multiple [certificate transparency logs](https://www.certificate-transparency.org/what-is-ct) via websocket connections to the clients. 10 | 11 | Everyone can use this project to analyze newly created TLS certificates as they are issued. 12 | 13 | ## Motivation 14 | 15 | From the moment I first found out about the certificate transparency logs, I was absolutely amazed by the great software of [Calidog](https://github.com/CaliDog/), which made the transparency log easier accessible for everyone. 16 | Their software "Certstream" parses the log and provides it in an easy-to-use format: JSON. 17 | 18 | After creating my first application that utilized the certstream server, I found that the hosted (demo) version of the server wasn't as reliable as I thought it would be. 19 | I got disconnects and sometimes other errors. Eventually, the provided server was still only thought to be **a demo**. 20 | 21 | I quickly thought about running my own instance of certstream. But I didn't want to install Elixir/Erlang on my server. Sure, I could have used Docker, but on second thought, I was really into the idea of creating an alternative server written in Go. 22 | 23 | "Why Go?", you might ask. Because it is a great language that compiles to native binaries on all major architectures and OSes. All the cool kids are using it right now. 24 | 25 | ## Getting started 26 | 27 | Setting up an instance of the certstream server is straightforward. You can either download and compile the code yourself, or use one of the [precompiled binaries](https://github.com/d-Rickyy-b/certstream-server-go/releases). 28 | 29 | By default, certstream-server-go will monitor all logs listed in the [Google Log list](https://www.gstatic.com/ct/log_list/v3/log_list.json), which are also included in the Chrome browser. There are more CT logs available than the ones listed there. Google provides [another list with all known CT logs](https://www.gstatic.com/ct/log_list/v3/all_logs_list.json). But not all of them might be relevant to you. Or maybe you are running your own CT log and want to monitor that as well? 30 | 31 | You can define additional logs in the config file. Check out the [sample config file](https://github.com/d-Rickyy-b/certstream-server-go/blob/master/config.sample.yaml) 32 | 33 | ### Docker 34 | 35 | There's also a prebuilt [Docker image](https://hub.docker.com/repository/docker/0rickyy0/certstream-server-go) available. 36 | You can use it by running this command: 37 | 38 | `docker run -d -v /path/to/config.yaml:/app/config.yaml -p 8080:8080 0rickyy0/certstream-server-go` 39 | 40 | > [!WARNING] 41 | > If you don't mount your own config file, the default config (config.sample.yaml) will be used. For more details, check out the [wiki](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Configuration). 42 | 43 | ## Connecting 44 | 45 | certstream-server-go offers multiple endpoints to connect to. 46 | 47 | | Config | Default | Function | 48 | |--------------------|-----------------|-------------------------------------------------------------------------------------------| 49 | | `full_url` | `/full-stream` | Constant stream of new certificates with all details available | 50 | | `lite_url` | `/` | Constant stream of new certificates with reduced details (no `as_der` and `chain` fields) | 51 | | `domains_only_url` | `/domains-only` | Constant stream of domains found in new certificates | 52 | 53 | You can connect to the certstream-server by opening a **websocket connection** to any of the aforementioned endpoints. 54 | After you're connected, certificate information will be streamed to your websocket. 55 | 56 | The server requires you to send a **ping message** at least every 60 seconds (it's recommended to use an interval of 30s for pings). 57 | If the server does not receive a ping message for more than this time, it will disconnect you. 58 | The server will **not** send out ping messages to your client. 59 | 60 | Read more about ping/pong WebSocket messages in the [Mozilla Developer Docs](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers#pings_and_pongs_the_heartbeat_of_websockets). 61 | 62 | ### Performance 63 | 64 | At idle (no clients connected), the server uses about **40 MB** of RAM, **14.5 Mbit/s** and **4–10% CPU** (Oracle Free Tier) on average while processing around **250–300 certificates per second**. 65 | 66 | ### Network considerations 67 | 68 | This tool requires outgoing access to the public internet to connect to the [Google Log list](https://www.gstatic.com/ct/log_list/v3/log_list.json) and the CT logs themselves. 69 | So if you happen to this tool in a corporate environment (e.g., behind a proxy/firewall), make sure to allow outgoing connections to gstatic.com and the CT logs you want to connect to. 70 | 71 | If you plan to connect clients to the server from outside your local network, make sure to allow incoming connections to the port you configured in the config file (webserver.listen_port). 72 | 73 | ### Monitoring 74 | 75 | **certstream-server-go** also offers a Prometheus metrics endpoint at `/metrics`. You can use this to monitor the server with Prometheus and Grafana. 76 | For an in-depth guide on how to do this, please refer to the [wiki](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics). 77 | 78 | ![grafana dashboard](https://user-images.githubusercontent.com/5798157/211434271-4350766d-2942-4fcb-8fda-f131f3f61cea.png) 79 | 80 | ### Example 81 | 82 | To receive a live example for any of the endpoints, send an HTTP GET request to the endpoints with `/example.json` appended to the endpoint. 83 | For example: `/full-stream/example.json`. This shows the lite format of a certificate update. 84 | 85 | ```json 86 | { 87 | "data": { 88 | "cert_index": 712420366, 89 | "cert_link": "https://yeti2022-2.ct.digicert.com/log/ct/v1/get-entries?start=712420366&end=712420366", 90 | "leaf_cert": { 91 | "all_domains": [ 92 | "cmslieferhit.e06.k-k.de" 93 | ], 94 | "extensions": { 95 | "authorityInfoAccess": "URI:http://r3.i.lencr.org/, URI:http://r3.o.lencr.org", 96 | "authorityKeyIdentifier": "keyid:14:2e:b3:17:b7:58:56:cb:ae:50:09:40:e6:1f:af:9d:8b:14:c2:c6", 97 | "basicConstraints": "CA:FALSE", 98 | "keyUsage": "Digital Signature, Key Encipherment", 99 | "subjectAltName": "DNS:cmslieferhit.e06.k-k.de", 100 | "subjectKeyIdentifier": "keyid:4e:cb:ae:47:84:a8:92:f7:e7:de:78:d1:00:9e:d9:cc:80:ac:0b:ce" 101 | }, 102 | "fingerprint": "27:58:3D:01:3D:71:B8:D3:A6:6E:2C:7A:86:3A:E9:1F:DB:F0:1B:5D", 103 | "sha1": "27:58:3D:01:3D:71:B8:D3:A6:6E:2C:7A:86:3A:E9:1F:DB:F0:1B:5D", 104 | "sha256": "57:61:38:C0:3C:03:A3:34:6A:0B:32:89:11:1B:74:AB:8A:DF:A5:02:9F:06:43:E6:F3:0E:69:F3:0E:4E:4E:FC", 105 | "not_after": 1667028404, 106 | "not_before": 1659252405, 107 | "serial_number": "0498BDF812FAF923FEBD5EF7B374899FC61A", 108 | "signature_algorithm": "sha256, rsa", 109 | "subject": { 110 | "C": null, 111 | "CN": "cmslieferhit.e06.k-k.de", 112 | "L": null, 113 | "O": null, 114 | "OU": null, 115 | "ST": null, 116 | "aggregated": "/CN=cmslieferhit.e06.k-k.de", 117 | "email_address": null 118 | }, 119 | "issuer": { 120 | "C": "US", 121 | "CN": "R3", 122 | "L": null, 123 | "O": "Let's Encrypt", 124 | "OU": null, 125 | "ST": null, 126 | "aggregated": "/C=US/CN=R3/O=Let's Encrypt", 127 | "email_address": null 128 | }, 129 | "is_ca": false 130 | }, 131 | "seen": 1659301203.904, 132 | "source": { 133 | "name": "DigiCert Yeti2022-2 Log", 134 | "url": "https://yeti2022-2.ct.digicert.com/log" 135 | }, 136 | "update_type": "PrecertLogEntry" 137 | }, 138 | "message_type": "certificate_update" 139 | } 140 | ``` 141 | -------------------------------------------------------------------------------- /internal/certificatetransparency/logmetrics.go: -------------------------------------------------------------------------------- 1 | package certificatetransparency 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "maps" 8 | "os" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type ( 14 | // OperatorLogs is a map of operator names to a list of CT log urls, operated by said operator. 15 | OperatorLogs map[string][]string 16 | // OperatorMetric is a map of CT log urls to the number of certs processed by said log. 17 | OperatorMetric map[string]int64 18 | // CTMetrics is a map of operator names to a map of CT log urls to the number of certs processed by said log. 19 | CTMetrics map[string]OperatorMetric 20 | // CTCertIndex is a map of CT log urls to the last processed certficate index on the said log. 21 | CTCertIndex map[string]uint64 22 | ) 23 | 24 | var ( 25 | processedCerts int64 26 | processedPrecerts int64 27 | metrics = LogMetrics{metrics: make(CTMetrics), index: make(CTCertIndex)} 28 | ) 29 | 30 | // LogMetrics is a struct that holds a map of metrics for each CT log grouped by operator. 31 | // Metrics can be accessed and written concurrently through the Get, Set and Inc methods. 32 | type LogMetrics struct { 33 | mutex sync.RWMutex 34 | metrics CTMetrics 35 | index CTCertIndex 36 | } 37 | 38 | // GetCTMetrics returns a copy of the internal metrics map. 39 | func (m *LogMetrics) GetCTMetrics() CTMetrics { 40 | m.mutex.RLock() 41 | defer m.mutex.RUnlock() 42 | 43 | copiedMap := make(CTMetrics) 44 | for operator, urls := range m.metrics { 45 | copiedMap[operator] = make(OperatorMetric) 46 | for url, count := range urls { 47 | copiedMap[operator][url] = count 48 | } 49 | } 50 | 51 | return copiedMap 52 | } 53 | 54 | // OperatorLogMapping returns a map of operator names to a list of CT logs. 55 | func (m *LogMetrics) OperatorLogMapping() OperatorLogs { 56 | m.mutex.RLock() 57 | defer m.mutex.RUnlock() 58 | 59 | logOperators := make(map[string][]string, len(m.metrics)) 60 | 61 | for operator, urls := range m.metrics { 62 | urlList := make([]string, len(urls)) 63 | counter := 0 64 | 65 | for url := range urls { 66 | urlList[counter] = url 67 | counter++ 68 | } 69 | logOperators[operator] = urlList 70 | } 71 | 72 | return logOperators 73 | } 74 | 75 | // Init initializes the internal metrics map with the given operator names and CT log urls if it doesn't exist yet. 76 | func (m *LogMetrics) Init(operator, url string) { 77 | m.mutex.Lock() 78 | defer m.mutex.Unlock() 79 | 80 | // if the operator does not exist, create a new entry 81 | if _, ok := m.metrics[operator]; !ok { 82 | m.metrics[operator] = make(OperatorMetric) 83 | } 84 | 85 | // if the operator exists but the url does not, create a new entry 86 | if _, ok := m.metrics[operator][url]; !ok { 87 | m.metrics[operator][url] = 0 88 | } 89 | 90 | // if url index does not exist, create a new entry 91 | if _, ok := m.index[url]; !ok { 92 | m.index[url] = 0 93 | } 94 | } 95 | 96 | // Get the metric for a given operator and ct url. 97 | func (m *LogMetrics) Get(operator, url string) int64 { 98 | // Despite this being a getter, we still need to fully lock the mutex because we might modify the map if the requested operator does not exist. 99 | m.mutex.Lock() 100 | defer m.mutex.Unlock() 101 | 102 | if _, ok := m.metrics[operator]; !ok { 103 | m.metrics[operator] = make(OperatorMetric) 104 | } 105 | 106 | return m.metrics[operator][url] 107 | } 108 | 109 | // Set the metric for a given operator and ct url. 110 | func (m *LogMetrics) Set(operator, url string, value int64) { 111 | m.mutex.Lock() 112 | defer m.mutex.Unlock() 113 | 114 | if _, ok := m.metrics[operator]; !ok { 115 | m.metrics[operator] = make(OperatorMetric) 116 | } 117 | 118 | m.metrics[operator][url] = value 119 | } 120 | 121 | // Inc the metric for a given operator and ct url. 122 | func (m *LogMetrics) Inc(operator, url string, index uint64) { 123 | m.mutex.Lock() 124 | defer m.mutex.Unlock() 125 | 126 | if _, ok := m.metrics[operator]; !ok { 127 | m.metrics[operator] = make(OperatorMetric) 128 | } 129 | 130 | m.metrics[operator][url]++ 131 | 132 | m.index[url] = index 133 | } 134 | 135 | // GetAllCTIndexes returns a copy of the internal CT index map. 136 | func (m *LogMetrics) GetAllCTIndexes() CTCertIndex { 137 | m.mutex.RLock() 138 | defer m.mutex.RUnlock() 139 | 140 | // make a copy of the index and return it, since map is a reference type 141 | copyOfIndex := make(CTCertIndex) 142 | maps.Copy(copyOfIndex, m.index) 143 | 144 | return copyOfIndex 145 | } 146 | 147 | // GetCTIndex returns the last cert index processed for a given CT url. 148 | func (m *LogMetrics) GetCTIndex(url string) uint64 { 149 | m.mutex.RLock() 150 | defer m.mutex.RUnlock() 151 | 152 | index, ok := m.index[url] 153 | if !ok { 154 | return 0 155 | } 156 | 157 | return index 158 | } 159 | 160 | // SetCTIndex sets the index for a given CT url. 161 | func (m *LogMetrics) SetCTIndex(url string, index uint64) { 162 | m.mutex.Lock() 163 | defer m.mutex.Unlock() 164 | 165 | log.Println("Setting CT index for ", url, " to ", index) 166 | m.index[url] = index 167 | } 168 | 169 | // LoadCTIndex loads the last cert index processed for each CT url if it exists. 170 | func (m *LogMetrics) LoadCTIndex(ctIndexFilePath string) { 171 | m.mutex.Lock() 172 | defer m.mutex.Unlock() 173 | 174 | bytes, readErr := os.ReadFile(ctIndexFilePath) 175 | if readErr != nil { 176 | // Create the file if it doesn't exist 177 | if os.IsNotExist(readErr) { 178 | err := createCTIndexFile(ctIndexFilePath, m) 179 | if err != nil { 180 | log.Printf("Error creating CT index file: '%s'\n", ctIndexFilePath) 181 | log.Panicln(err) 182 | } 183 | } else { 184 | // If the file exists but we can't read it, log the error and panic 185 | log.Panicln(readErr) 186 | } 187 | } 188 | 189 | jerr := json.Unmarshal(bytes, &m.index) 190 | if jerr != nil { 191 | log.Printf("Error unmarshalling CT index file: '%s'\n", ctIndexFilePath) 192 | log.Panicln(jerr) 193 | } 194 | 195 | log.Println("Successfully loaded saved CT indexes") 196 | } 197 | 198 | func createCTIndexFile(ctIndexFilePath string, m *LogMetrics) error { 199 | m.mutex.RLock() 200 | defer m.mutex.RUnlock() 201 | 202 | log.Printf("Specified CT index file does not exist: '%s'\n", ctIndexFilePath) 203 | log.Println("Creating CT index file now!") 204 | 205 | file, createErr := os.Create(ctIndexFilePath) 206 | if createErr != nil { 207 | log.Printf("Error creating CT index file: '%s'\n", ctIndexFilePath) 208 | log.Panicln(createErr) 209 | } 210 | 211 | bytes, marshalErr := json.Marshal(m.index) 212 | if marshalErr != nil { 213 | return marshalErr 214 | } 215 | _, writeErr := file.Write(bytes) 216 | if writeErr != nil { 217 | log.Printf("Error writing to CT index file: '%s'\n", ctIndexFilePath) 218 | log.Panicln(writeErr) 219 | } 220 | file.Close() 221 | 222 | return nil 223 | } 224 | 225 | // SaveCertIndexesAtInterval saves the index of CTLogs at given intervals. 226 | // We first create a temp file and write the index data to it. Only then do we move the temp file to the actual 227 | // permanent index file. This prevents the last good index file from being clobbered if the program was shutdown/killed 228 | // in-between the write operation. 229 | func (m *LogMetrics) SaveCertIndexesAtInterval(interval time.Duration, ctIndexFilePath string) { 230 | ticker := time.NewTicker(interval) 231 | defer ticker.Stop() 232 | 233 | for range ticker.C { 234 | m.SaveCertIndexes(ctIndexFilePath) 235 | } 236 | } 237 | 238 | // SaveCertIndexes saves the index of CTLogs to a file. 239 | func (m *LogMetrics) SaveCertIndexes(ctIndexFilePath string) { 240 | tempFilePath := fmt.Sprintf("%s.tmp", ctIndexFilePath) 241 | 242 | // Get the index data 243 | ctIndex := m.GetAllCTIndexes() 244 | bytes, cerr := json.MarshalIndent(ctIndex, "", " ") 245 | if cerr != nil { 246 | log.Panic(cerr) 247 | } 248 | 249 | // Save data to a temporary file first 250 | file, openErr := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) 251 | if openErr != nil { 252 | log.Println("Could not save CT index to temporary file: ", openErr) 253 | return 254 | } 255 | 256 | truncateErr := file.Truncate(0) 257 | if truncateErr != nil { 258 | log.Println("Error truncating CT index temp file: ", truncateErr) 259 | return 260 | } 261 | // TODO: check for short writes 262 | _, writeErr := file.Write(bytes) 263 | if writeErr != nil { 264 | log.Println("Error writing to CT index temp file: ", writeErr) 265 | return 266 | } 267 | syncErr := file.Sync() 268 | if syncErr != nil { 269 | log.Println("Error syncing CT index temp file: ", syncErr) 270 | return 271 | } 272 | 273 | file.Close() 274 | 275 | // Atomically move the temp file to the permanent file 276 | renameErr := os.Rename(tempFilePath, ctIndexFilePath) 277 | if renameErr != nil { 278 | log.Println("Error renaming CT index temp file: ", renameErr) 279 | return 280 | } 281 | } 282 | 283 | // GetProcessedCerts returns the total number of processed certificates. 284 | func GetProcessedCerts() int64 { 285 | return processedCerts 286 | } 287 | 288 | // GetProcessedPrecerts returns the total number of processed precertificates. 289 | func GetProcessedPrecerts() int64 { 290 | return processedPrecerts 291 | } 292 | 293 | func GetCertMetrics() CTMetrics { 294 | return metrics.GetCTMetrics() 295 | } 296 | 297 | func GetLogOperators() map[string][]string { 298 | return metrics.OperatorLogMapping() 299 | } 300 | -------------------------------------------------------------------------------- /internal/web/server.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net" 10 | "net/http" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/go-chi/chi/v5" 15 | "github.com/go-chi/chi/v5/middleware" 16 | 17 | "github.com/d-Rickyy-b/certstream-server-go/internal/config" 18 | "github.com/d-Rickyy-b/certstream-server-go/internal/models" 19 | 20 | "github.com/gorilla/websocket" 21 | ) 22 | 23 | var ( 24 | ClientHandler = BroadcastManager{} 25 | upgrader websocket.Upgrader 26 | ) 27 | 28 | // WebServer is a struct that holds the necessary information to run a webserver. 29 | // It is used for the websocket server as well as the metrics server. 30 | type WebServer struct { 31 | networkIf string 32 | port int 33 | routes *chi.Mux 34 | server *http.Server 35 | certPath string 36 | keyPath string 37 | } 38 | 39 | // RegisterPrometheus registers a new handler that listens on the given url and calls the given function 40 | // in order to provide metrics for a prometheus server. This function signature was used, because VictoriaMetrics 41 | // offers exactly this function signature. 42 | func (ws *WebServer) RegisterPrometheus(url string, callback func(w io.Writer, exposeProcessMetrics bool)) { 43 | ws.routes.HandleFunc(url, func(w http.ResponseWriter, _ *http.Request) { 44 | callback(w, config.AppConfig.Prometheus.ExposeSystemMetrics) 45 | }) 46 | } 47 | 48 | // IPWhitelist returns a middleware that checks if the IP of the client is in the whitelist. 49 | func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { 50 | // build a list of whitelisted IPs and CIDRs 51 | log.Println("Building IP whitelist...") 52 | 53 | var ipList []net.IP 54 | var cidrList []net.IPNet 55 | 56 | for _, element := range whitelist { 57 | _, ipNet, err := net.ParseCIDR(element) 58 | if err != nil { 59 | var ip net.IP 60 | if ip = net.ParseIP(element); ip == nil { 61 | log.Println("Invalid IP in metrics whitelist: ", element) 62 | 63 | continue 64 | } 65 | 66 | ipList = append(ipList, ip) 67 | 68 | continue 69 | } 70 | 71 | cidrList = append(cidrList, *ipNet) 72 | } 73 | 74 | log.Println("IP whitelist: ", ipList) 75 | log.Println("CIDR whitelist: ", cidrList) 76 | 77 | return func(next http.Handler) http.Handler { 78 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 79 | // if the whitelist is empty, just continue 80 | if len(ipList) == 0 && len(cidrList) == 0 { 81 | next.ServeHTTP(w, r) 82 | return 83 | } 84 | 85 | ipString, _, err := net.SplitHostPort(r.RemoteAddr) 86 | if err != nil { 87 | http.Error(w, "InternalServerError", http.StatusInternalServerError) 88 | return 89 | } 90 | 91 | ip := net.ParseIP(ipString) 92 | 93 | for _, cidr := range cidrList { 94 | if cidr.Contains(ip) { 95 | next.ServeHTTP(w, r) 96 | return 97 | } 98 | } 99 | 100 | for _, whitelistedIP := range ipList { 101 | if whitelistedIP.Equal(ip) { 102 | next.ServeHTTP(w, r) 103 | return 104 | } 105 | } 106 | 107 | log.Printf("IP %s not in whitelist, rejecting request\n", r.RemoteAddr) 108 | http.Error(w, "Forbidden", http.StatusForbidden) 109 | }) 110 | } 111 | } 112 | 113 | // initFullWebsocket is called when a client connects to the /full-stream endpoint. 114 | // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. 115 | func initFullWebsocket(w http.ResponseWriter, r *http.Request) { 116 | connection, err := upgradeConnection(w, r) 117 | if err != nil { 118 | log.Println("Error while trying to upgrade connection:", err) 119 | return 120 | } 121 | 122 | setupClient(connection, SubTypeFull, r.RemoteAddr) 123 | } 124 | 125 | // initLiteWebsocket is called when a client connects to the / endpoint. 126 | // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. 127 | func initLiteWebsocket(w http.ResponseWriter, r *http.Request) { 128 | connection, err := upgradeConnection(w, r) 129 | if err != nil { 130 | log.Println("Error while trying to upgrade connection:", err) 131 | return 132 | } 133 | 134 | setupClient(connection, SubTypeLite, r.RemoteAddr) 135 | } 136 | 137 | // initDomainWebsocket is called when a client connects to the /domains-only endpoint. 138 | // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. 139 | func initDomainWebsocket(w http.ResponseWriter, r *http.Request) { 140 | connection, err := upgradeConnection(w, r) 141 | if err != nil { 142 | log.Println("Error while trying to upgrade connection:", err) 143 | return 144 | } 145 | 146 | setupClient(connection, SubTypeDomain, r.RemoteAddr) 147 | } 148 | 149 | // upgradeConnection upgrades the connection to a websocket and returns the connection. 150 | func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error) { 151 | var remoteAddr string 152 | 153 | xForwardedFor := r.Header.Get("X-Forwarded-For") 154 | if xForwardedFor != "" { 155 | remoteAddr = fmt.Sprintf("'%s' (X-Forwarded-For: '%s')", r.RemoteAddr, xForwardedFor) 156 | } else { 157 | remoteAddr = fmt.Sprintf("'%s'", r.RemoteAddr) 158 | } 159 | 160 | log.Printf("Starting new websocket for %s - %s\n", remoteAddr, r.URL) 161 | 162 | connection, err := upgrader.Upgrade(w, r, nil) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | defaultCloseHandler := connection.CloseHandler() 168 | connection.SetCloseHandler(func(code int, text string) error { 169 | log.Printf("Stopping websocket for %s - %s\n", remoteAddr, r.URL) 170 | return defaultCloseHandler(code, text) 171 | }) 172 | 173 | return connection, nil 174 | } 175 | 176 | // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. 177 | func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) { 178 | c := newClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket) 179 | go c.broadcastHandler() 180 | go c.listenWebsocket() 181 | 182 | ClientHandler.registerClient(c) 183 | } 184 | 185 | // setupWebsocketRoutes configures all the routes necessary for the websocket webserver. 186 | func setupWebsocketRoutes(r *chi.Mux) { 187 | r.Use(middleware.Recoverer) 188 | r.Route("/", func(r chi.Router) { 189 | r.Route(config.AppConfig.Webserver.FullURL, func(r chi.Router) { 190 | r.HandleFunc("/", initFullWebsocket) 191 | r.HandleFunc("/example.json", exampleFull) 192 | }) 193 | 194 | r.Route(config.AppConfig.Webserver.LiteURL, func(r chi.Router) { 195 | r.HandleFunc("/", initLiteWebsocket) 196 | r.HandleFunc("/example.json", exampleLite) 197 | }) 198 | 199 | r.Route(config.AppConfig.Webserver.DomainsOnlyURL, func(r chi.Router) { 200 | r.HandleFunc("/", initDomainWebsocket) 201 | r.HandleFunc("/example.json", exampleDomains) 202 | }) 203 | }) 204 | } 205 | 206 | func (ws *WebServer) initServer() { 207 | addr := net.JoinHostPort(ws.networkIf, strconv.Itoa(ws.port)) 208 | 209 | tlsConfig := &tls.Config{ 210 | MinVersion: tls.VersionTLS12, 211 | CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519}, 212 | CipherSuites: []uint16{ 213 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 214 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 215 | tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, 216 | tls.TLS_RSA_WITH_AES_256_GCM_SHA384, 217 | tls.TLS_RSA_WITH_AES_256_CBC_SHA, 218 | }, 219 | } 220 | 221 | ws.server = &http.Server{ 222 | Addr: addr, 223 | Handler: ws.routes, 224 | TLSConfig: tlsConfig, 225 | IdleTimeout: time.Minute, 226 | ReadTimeout: 10 * time.Second, 227 | ReadHeaderTimeout: 2 * time.Second, 228 | WriteTimeout: 10 * time.Second, 229 | } 230 | } 231 | 232 | // NewMetricsServer creates a new webserver that listens on the given port and provides metrics for a metrics server. 233 | func NewMetricsServer(networkIf string, port int, certPath, keyPath string) *WebServer { 234 | server := &WebServer{ 235 | networkIf: networkIf, 236 | port: port, 237 | routes: chi.NewRouter(), 238 | certPath: certPath, 239 | keyPath: keyPath, 240 | } 241 | server.routes.Use(middleware.Recoverer) 242 | 243 | if config.AppConfig.Prometheus.RealIP { 244 | server.routes.Use(middleware.RealIP) 245 | } 246 | 247 | // Enable IP whitelist if configured 248 | if len(config.AppConfig.Prometheus.Whitelist) > 0 { 249 | server.routes.Use(IPWhitelist(config.AppConfig.Prometheus.Whitelist)) 250 | } 251 | 252 | server.initServer() 253 | 254 | return server 255 | } 256 | 257 | // NewWebsocketServer starts a new webserver and initialized it with the necessary routes. 258 | // It also starts the broadcaster in ClientHandler as a background job and takes care of 259 | // setting up websocket.Upgrader. 260 | func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *WebServer { 261 | server := &WebServer{ 262 | networkIf: networkIf, 263 | port: port, 264 | routes: chi.NewRouter(), 265 | certPath: certPath, 266 | keyPath: keyPath, 267 | } 268 | 269 | upgrader = websocket.Upgrader{ 270 | EnableCompression: config.AppConfig.Webserver.CompressionEnabled, 271 | CheckOrigin: func(_ *http.Request) bool { 272 | // Allow all connections by default 273 | return true 274 | }, 275 | } 276 | 277 | if config.AppConfig.Webserver.RealIP { 278 | server.routes.Use(middleware.RealIP) 279 | } 280 | 281 | // Enable IP whitelist if configured 282 | if len(config.AppConfig.Webserver.Whitelist) > 0 { 283 | server.routes.Use(IPWhitelist(config.AppConfig.Webserver.Whitelist)) 284 | } 285 | 286 | setupWebsocketRoutes(server.routes) 287 | server.initServer() 288 | 289 | ClientHandler.Broadcast = make(chan models.Entry, config.AppConfig.General.BufferSizes.BroadcastManager) 290 | go ClientHandler.broadcaster() 291 | 292 | return server 293 | } 294 | 295 | // Start initializes the webserver and starts listening for connections. 296 | func (ws *WebServer) Start() { 297 | log.Printf("Starting webserver on %s\n", ws.server.Addr) 298 | 299 | var err error 300 | if ws.keyPath != "" && ws.certPath != "" { 301 | err = ws.server.ListenAndServeTLS(ws.certPath, ws.keyPath) 302 | } else { 303 | err = ws.server.ListenAndServe() 304 | } 305 | 306 | if err != nil { 307 | log.Fatal("Error while serving webserver: ", err) 308 | } 309 | } 310 | 311 | // Stop tries to stop the webserver gracefully. If it doesn't stop within 15 seconds, it is forcefully closed. 312 | func (ws *WebServer) Stop() { 313 | log.Println("Stopping webserver...") 314 | 315 | // If the server did not stop within 15 seconds, forcefully close it 316 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) 317 | defer cancel() 318 | 319 | if err := ws.server.Shutdown(ctx); err != nil { 320 | log.Fatal("Error while stopping webserver: ", err) 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /internal/certificatetransparency/ct-parser.go: -------------------------------------------------------------------------------- 1 | package certificatetransparency 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" //nolint:gosec 6 | "crypto/sha256" 7 | "encoding/base64" 8 | "encoding/hex" 9 | "errors" 10 | "fmt" 11 | "hash" 12 | "log" 13 | "math/big" 14 | "strings" 15 | "time" 16 | 17 | "github.com/d-Rickyy-b/certstream-server-go/internal/models" 18 | 19 | ct "github.com/google/certificate-transparency-go" 20 | "github.com/google/certificate-transparency-go/x509" 21 | "github.com/google/certificate-transparency-go/x509/pkix" 22 | ) 23 | 24 | // parseData converts a *ct.RawLogEntry struct into a certstream.Data struct by copying some values and calculating others. 25 | func parseData(entry *ct.RawLogEntry, operatorName, logName, ctURL string) (models.Data, error) { 26 | certLink := fmt.Sprintf("%s/ct/v1/get-entries?start=%d&end=%d", ctURL, entry.Index, entry.Index) 27 | 28 | // Create main data structure 29 | data := models.Data{ 30 | CertIndex: uint64(entry.Index), 31 | CertLink: certLink, 32 | Seen: float64(time.Now().UnixMilli()) / 1_000, 33 | Source: models.Source{ 34 | Name: logName, 35 | URL: ctURL, 36 | Operator: operatorName, 37 | NormalizedURL: normalizeCtlogURL(ctURL), 38 | }, 39 | UpdateType: "X509LogEntry", 40 | } 41 | 42 | // Convert RawLogEntry to ct.LogEntry 43 | logEntry, conversionErr := entry.ToLogEntry() 44 | if conversionErr != nil { 45 | log.Println("Could not convert entry to LogEntry: ", conversionErr) 46 | return models.Data{}, conversionErr 47 | } 48 | 49 | var cert *x509.Certificate 50 | var rawData []byte 51 | var isPrecert bool 52 | 53 | switch { 54 | case logEntry.X509Cert != nil: 55 | cert = logEntry.X509Cert 56 | rawData = logEntry.X509Cert.Raw 57 | isPrecert = false 58 | case logEntry.Precert != nil: 59 | cert = logEntry.Precert.TBSCertificate 60 | rawData = logEntry.Precert.Submitted.Data 61 | isPrecert = true 62 | default: 63 | return models.Data{}, errors.New("could not parse entry: no certificate found") 64 | } 65 | 66 | // Calculate certificate hash from the raw DER bytes of the certificate 67 | data.LeafCert = leafCertFromX509cert(*cert) 68 | 69 | // recalculate hashes if the certificate is a precertificate 70 | if isPrecert { 71 | calculatedHash := calculateSHA1(rawData) 72 | data.LeafCert.Fingerprint = calculatedHash 73 | data.LeafCert.SHA1 = calculatedHash 74 | data.LeafCert.SHA256 = calculateSHA256(rawData) 75 | 76 | // Since we use the TBSCertificate to parse the LeafCert, the PoisonByte indicator cannot be set by our parser. 77 | // According to RFC 6962 Section 3.2 the `"tbs_certificate" is the DER-encoded TBSCertificate (see [RFC5280]) 78 | // component of the Precertificate -- that is, without the signature and the poison extension.` 79 | // Since the PoisonByte Extension `is to ensure that the Precertificate cannot be validated by a standard 80 | // X.509v3 client`, we can safely set it for each precertificate. 81 | data.LeafCert.Extensions.CTLPoisonByte = true 82 | } 83 | 84 | certAsDER := base64.StdEncoding.EncodeToString(entry.Cert.Data) 85 | data.LeafCert.AsDER = certAsDER 86 | 87 | var parseErr error 88 | data.Chain, parseErr = parseCertificateChain(logEntry) 89 | if parseErr != nil { 90 | log.Println("Could not parse certificate chain: ", parseErr) 91 | return models.Data{}, parseErr 92 | } 93 | 94 | return data, nil 95 | } 96 | 97 | // parseCertificateChain returns the certificate chain in form of a []LeafCert from the given *ct.LogEntry. 98 | func parseCertificateChain(logEntry *ct.LogEntry) ([]models.LeafCert, error) { 99 | chain := make([]models.LeafCert, len(logEntry.Chain)) 100 | 101 | for i, chainEntry := range logEntry.Chain { 102 | myCert, parseErr := x509.ParseCertificate(chainEntry.Data) 103 | if parseErr != nil { 104 | log.Println("Error parsing certificate: ", parseErr) 105 | return nil, parseErr 106 | } 107 | 108 | leafCert := leafCertFromX509cert(*myCert) 109 | chain[i] = leafCert 110 | } 111 | 112 | return chain, nil 113 | } 114 | 115 | // leafCertFromX509cert converts a x509.Certificate to the custom LeafCert data structure. 116 | func leafCertFromX509cert(cert x509.Certificate) models.LeafCert { 117 | leafCert := models.LeafCert{ 118 | AllDomains: cert.DNSNames, 119 | Extensions: models.Extensions{}, 120 | NotAfter: cert.NotAfter.Unix(), 121 | NotBefore: cert.NotBefore.Unix(), 122 | SerialNumber: formatSerialNumber(cert.SerialNumber), 123 | SignatureAlgorithm: parseSignatureAlgorithm(cert.SignatureAlgorithm), 124 | IsCA: cert.IsCA, 125 | } 126 | 127 | // The zero value of DomainsEntry.Data is nil, but we want an empty array - especially for json marshalling later. 128 | if leafCert.AllDomains == nil { 129 | leafCert.AllDomains = []string{} 130 | } 131 | 132 | leafCert.Subject = buildSubject(cert.Subject) 133 | if *leafCert.Subject.CN != "" && !leafCert.IsCA { 134 | domainAlreadyAdded := false 135 | // TODO check if CN matches domain regex 136 | for _, domain := range leafCert.AllDomains { 137 | if domain == *leafCert.Subject.CN { 138 | domainAlreadyAdded = true 139 | break 140 | } 141 | } 142 | 143 | if !domainAlreadyAdded { 144 | leafCert.AllDomains = append(leafCert.AllDomains, *leafCert.Subject.CN) 145 | } 146 | } 147 | 148 | leafCert.Issuer = buildSubject(cert.Issuer) 149 | 150 | leafCert.AsDER = base64.StdEncoding.EncodeToString(cert.Raw) 151 | leafCert.Fingerprint = calculateSHA1(cert.Raw) 152 | leafCert.SHA1 = leafCert.Fingerprint 153 | leafCert.SHA256 = calculateSHA256(cert.Raw) 154 | 155 | // TODO fix Extensions - check x509util.go 156 | for _, extension := range cert.Extensions { 157 | switch { 158 | case extension.Id.Equal(x509.OIDExtensionAuthorityKeyId): 159 | leafCert.Extensions.AuthorityKeyIdentifier = formatKeyID(cert.AuthorityKeyId) 160 | case extension.Id.Equal(x509.OIDExtensionKeyUsage): 161 | keyUsage := keyUsageToString(cert.KeyUsage) 162 | leafCert.Extensions.KeyUsage = &keyUsage 163 | case extension.Id.Equal(x509.OIDExtensionSubjectKeyId): 164 | leafCert.Extensions.SubjectKeyIdentifier = formatKeyID(cert.SubjectKeyId) 165 | case extension.Id.Equal(x509.OIDExtensionBasicConstraints): 166 | isCA := strings.ToUpper(fmt.Sprintf("CA:%t", cert.IsCA)) 167 | leafCert.Extensions.BasicConstraints = &isCA 168 | case extension.Id.Equal(x509.OIDExtensionSubjectAltName): 169 | var buf bytes.Buffer 170 | for _, name := range cert.DNSNames { 171 | commaAppend(&buf, "DNS:"+name) 172 | } 173 | 174 | for _, email := range cert.EmailAddresses { 175 | commaAppend(&buf, "email:"+email) 176 | } 177 | 178 | for _, ip := range cert.IPAddresses { 179 | commaAppend(&buf, "IP Address:"+ip.String()) 180 | } 181 | 182 | subjectAltName := buf.String() 183 | leafCert.Extensions.SubjectAltName = &subjectAltName 184 | case extension.Id.Equal(x509.OIDExtensionAuthorityInfoAccess): 185 | var buf bytes.Buffer 186 | for _, issuer := range cert.IssuingCertificateURL { 187 | commaAppend(&buf, "URI:"+issuer) 188 | } 189 | 190 | for _, ocsp := range cert.OCSPServer { 191 | commaAppend(&buf, "URI:"+ocsp) 192 | } 193 | 194 | result := buf.String() 195 | leafCert.Extensions.AuthorityInfoAccess = &result 196 | case extension.Id.Equal(x509.OIDExtensionCTPoison): 197 | leafCert.Extensions.CTLPoisonByte = true 198 | case extension.Id.Equal(x509.OIDExtensionCertificatePolicies): 199 | var result string 200 | for _, policy := range cert.PolicyIdentifiers { 201 | // The current way of joining the string leaves us with a trailing newline, 202 | // but that's how the original certstream server does it too. 203 | result += fmt.Sprintf("Policy: %s\n", policy) 204 | } 205 | leafCert.Extensions.CertificatePolicies = &result 206 | } 207 | } 208 | 209 | return leafCert 210 | } 211 | 212 | // buildSubject generates a Subject struct from the given pkix.Name. 213 | func buildSubject(certSubject pkix.Name) models.Subject { 214 | subject := models.Subject{ 215 | C: parseName(certSubject.Country), 216 | CN: &certSubject.CommonName, 217 | L: parseName(certSubject.Locality), 218 | O: parseName(certSubject.Organization), 219 | OU: parseName(certSubject.OrganizationalUnit), 220 | ST: parseName(certSubject.StreetAddress), 221 | } 222 | 223 | var aggregated string 224 | 225 | if subject.C != nil { 226 | aggregated += fmt.Sprintf("/C=%s", *subject.C) 227 | } 228 | 229 | if subject.CN != nil { 230 | aggregated += fmt.Sprintf("/CN=%s", *subject.CN) 231 | } 232 | 233 | if subject.L != nil { 234 | aggregated += fmt.Sprintf("/L=%s", *subject.L) 235 | } 236 | 237 | if subject.O != nil { 238 | aggregated += fmt.Sprintf("/O=%s", *subject.O) 239 | } 240 | 241 | if subject.OU != nil { 242 | aggregated += fmt.Sprintf("/OU=%s", *subject.OU) 243 | } 244 | 245 | if subject.ST != nil { 246 | aggregated += fmt.Sprintf("/ST=%s", *subject.ST) 247 | } 248 | 249 | subject.Aggregated = &aggregated 250 | 251 | return subject 252 | } 253 | 254 | // formatKeyID transforms the AuthorityKeyIdentifier to be more readable. 255 | func formatKeyID(keyID []byte) *string { 256 | tmp := hex.EncodeToString(keyID) 257 | var digest string 258 | 259 | for i := 0; i < len(tmp); i += 2 { 260 | digest = digest + ":" + tmp[i:i+2] 261 | } 262 | 263 | digest = strings.TrimLeft(digest, ":") 264 | digest = fmt.Sprintf("keyid:%s", digest) 265 | 266 | return &digest 267 | } 268 | 269 | func formatSerialNumber(serialNumber *big.Int) string { 270 | sn := fmt.Sprintf("%X", serialNumber) 271 | if len(sn)%2 == 1 { 272 | sn = "0" + sn 273 | } 274 | 275 | return sn 276 | } 277 | 278 | func parseName(input []string) *string { 279 | if input == nil { 280 | return nil 281 | } 282 | 283 | var result string 284 | for _, s := range input { 285 | if len(result) > 0 { 286 | result += "," 287 | } 288 | 289 | result += s 290 | } 291 | 292 | return &result 293 | } 294 | 295 | // calculateHash takes a hash.Hash implementation and calculates the hash of the given data. 296 | // It returns the hash in the format "XX:XX:XX:...". 297 | func calculateHash(data []byte, certHasher hash.Hash) string { 298 | _, e := certHasher.Write(data) 299 | if e != nil { 300 | log.Printf("Error while hashing cert: %s\n", e) 301 | return "" 302 | } 303 | 304 | certHash := fmt.Sprintf("%02x", certHasher.Sum(nil)) 305 | certHash = strings.ToUpper(certHash) 306 | 307 | var result bytes.Buffer 308 | for i := 0; i < len(certHash); i++ { 309 | if i%2 == 0 && i > 0 { 310 | result.WriteByte(':') 311 | } 312 | c := certHash[i] 313 | result.WriteByte(c) 314 | } 315 | 316 | return result.String() 317 | } 318 | 319 | // calculateSHA1 calculates the SHA1 fingerprint of the given data. 320 | func calculateSHA1(data []byte) string { 321 | return calculateHash(data, sha1.New()) //nolint:gosec 322 | } 323 | 324 | // calculateSHA256 calculates the SHA256 fingerprint of the given data. 325 | func calculateSHA256(data []byte) string { 326 | return calculateHash(data, sha256.New()) 327 | } 328 | 329 | func parseSignatureAlgorithm(signatureAlgoritm x509.SignatureAlgorithm) string { 330 | switch signatureAlgoritm { 331 | case x509.MD2WithRSA: 332 | return "md2, rsa" 333 | case x509.MD5WithRSA: 334 | return "md5, rsa" 335 | case x509.SHA1WithRSA: 336 | return "sha1, rsa" 337 | case x509.SHA256WithRSA: 338 | return "sha256, rsa" 339 | case x509.SHA384WithRSA: 340 | return "sha384, rsa" 341 | case x509.SHA512WithRSA: 342 | return "sha512, rsa" 343 | case x509.SHA256WithRSAPSS: 344 | return "sha256, rsa-pss" 345 | case x509.SHA384WithRSAPSS: 346 | return "sha384, rsa-pss" 347 | case x509.SHA512WithRSAPSS: 348 | return "sha512, rsa-pss" 349 | case x509.DSAWithSHA1: 350 | return "dsa, sha1" 351 | case x509.DSAWithSHA256: 352 | return "dsa, sha256" 353 | case x509.ECDSAWithSHA1: 354 | return "ecdsa, sha1" 355 | case x509.ECDSAWithSHA256: 356 | return "ecdsa, sha256" 357 | case x509.ECDSAWithSHA384: 358 | return "ecdsa, sha384" 359 | case x509.ECDSAWithSHA512: 360 | return "ecdsa, sha512" 361 | case x509.PureEd25519: 362 | return "ed25519" 363 | case x509.UnknownSignatureAlgorithm: 364 | fallthrough 365 | default: 366 | return "unknown" 367 | } 368 | } 369 | 370 | // commaAppend lets you append a string with a comma prepended to a buffer. 371 | func commaAppend(buf *bytes.Buffer, s string) { 372 | if buf.Len() > 0 { 373 | buf.WriteString(", ") 374 | } 375 | 376 | buf.WriteString(s) 377 | } 378 | 379 | func keyUsageToString(k x509.KeyUsage) string { 380 | var buf bytes.Buffer 381 | if k&x509.KeyUsageDigitalSignature != 0 { 382 | commaAppend(&buf, "Digital Signature") 383 | } 384 | 385 | if k&x509.KeyUsageContentCommitment != 0 { 386 | commaAppend(&buf, "Content Commitment") 387 | } 388 | 389 | if k&x509.KeyUsageKeyEncipherment != 0 { 390 | commaAppend(&buf, "Key Encipherment") 391 | } 392 | 393 | if k&x509.KeyUsageDataEncipherment != 0 { 394 | commaAppend(&buf, "Data Encipherment") 395 | } 396 | 397 | if k&x509.KeyUsageKeyAgreement != 0 { 398 | commaAppend(&buf, "Key Agreement") 399 | } 400 | 401 | if k&x509.KeyUsageCertSign != 0 { 402 | commaAppend(&buf, "Certificate Signing") 403 | } 404 | 405 | if k&x509.KeyUsageCRLSign != 0 { 406 | commaAppend(&buf, "CRL Signing") 407 | } 408 | 409 | if k&x509.KeyUsageEncipherOnly != 0 { 410 | commaAppend(&buf, "Encipher Only") 411 | } 412 | 413 | if k&x509.KeyUsageDecipherOnly != 0 { 414 | commaAppend(&buf, "Decipher Only") 415 | } 416 | 417 | return buf.String() 418 | } 419 | 420 | // ParseCertstreamEntry creates an Entry from a ct.RawLogEntry. 421 | func ParseCertstreamEntry(rawEntry *ct.RawLogEntry, operatorName, logname, ctURL string) (models.Entry, error) { 422 | if rawEntry == nil { 423 | return models.Entry{}, errors.New("certstream entry is nil") 424 | } 425 | 426 | data, err := parseData(rawEntry, operatorName, logname, ctURL) 427 | if err != nil { 428 | return models.Entry{}, err 429 | } 430 | 431 | entry := models.Entry{ 432 | Data: data, 433 | MessageType: "certificate_update", 434 | } 435 | 436 | return entry, nil 437 | } 438 | -------------------------------------------------------------------------------- /internal/certificatetransparency/ct-watcher.go: -------------------------------------------------------------------------------- 1 | package certificatetransparency 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "path/filepath" 11 | "strings" 12 | "sync" 13 | "sync/atomic" 14 | "time" 15 | 16 | "github.com/d-Rickyy-b/certstream-server-go/internal/config" 17 | "github.com/d-Rickyy-b/certstream-server-go/internal/models" 18 | "github.com/d-Rickyy-b/certstream-server-go/internal/web" 19 | 20 | ct "github.com/google/certificate-transparency-go" 21 | "github.com/google/certificate-transparency-go/client" 22 | "github.com/google/certificate-transparency-go/jsonclient" 23 | "github.com/google/certificate-transparency-go/loglist3" 24 | "github.com/google/certificate-transparency-go/scanner" 25 | ) 26 | 27 | var ( 28 | errCreatingClient = errors.New("failed to create JSON client") 29 | errFetchingSTHFailed = errors.New("failed to fetch STH") 30 | userAgent = fmt.Sprintf("Certstream Server v%s (github.com/d-Rickyy-b/certstream-server-go)", config.Version) 31 | ) 32 | 33 | // Watcher describes a component that watches for new certificates in a CT log. 34 | type Watcher struct { 35 | workers []*worker 36 | workersMu sync.RWMutex 37 | wg sync.WaitGroup 38 | context context.Context 39 | certChan chan models.Entry 40 | cancelFunc context.CancelFunc 41 | } 42 | 43 | // NewWatcher creates a new Watcher. 44 | func NewWatcher(certChan chan models.Entry) *Watcher { 45 | return &Watcher{ 46 | certChan: certChan, 47 | } 48 | } 49 | 50 | // Start starts the watcher. This method is blocking. 51 | func (w *Watcher) Start() { 52 | w.context, w.cancelFunc = context.WithCancel(context.Background()) 53 | 54 | // Create new certChan if it doesn't exist yet 55 | if w.certChan == nil { 56 | w.certChan = make(chan models.Entry, 5000) 57 | } 58 | 59 | if config.AppConfig.General.Recovery.Enabled { 60 | ctIndexFilePath, err := filepath.Abs(config.AppConfig.General.Recovery.CTIndexFile) 61 | if err != nil { 62 | log.Printf("Error getting absolute path for CT index file: '%s', %s\n", config.AppConfig.General.Recovery.CTIndexFile, err) 63 | return 64 | } 65 | // Load Saved CT Indexes 66 | metrics.LoadCTIndex(ctIndexFilePath) 67 | // Save CTIndexes at regular intervals 68 | go metrics.SaveCertIndexesAtInterval(time.Second*30, ctIndexFilePath) // save indexes every X seconds 69 | } 70 | 71 | // initialize the watcher with currently available logs 72 | w.updateLogs() 73 | 74 | log.Println("Started CT watcher") 75 | go certHandler(w.certChan) 76 | go w.watchNewLogs() 77 | 78 | // Wait for all workers to finish 79 | w.wg.Wait() 80 | close(w.certChan) 81 | } 82 | 83 | // watchNewLogs monitors the ct log list for new logs and starts a worker for each new log found. 84 | // This method is blocking. It can be stopped by cancelling the context. 85 | func (w *Watcher) watchNewLogs() { 86 | // Check for new logs once every hour 87 | ticker := time.NewTicker(1 * time.Hour) 88 | for { 89 | select { 90 | case <-ticker.C: 91 | w.updateLogs() 92 | case <-w.context.Done(): 93 | ticker.Stop() 94 | return 95 | } 96 | } 97 | } 98 | 99 | // updateLogs checks the transparency log list for new logs and adds new 0workers for those to the watcher. 100 | func (w *Watcher) updateLogs() { 101 | // Get a list of urls of all CT logs 102 | logList, err := getAllLogs() 103 | if err != nil { 104 | log.Println(err) 105 | return 106 | } 107 | 108 | w.addNewlyAvailableLogs(logList) 109 | 110 | if *config.AppConfig.General.DropOldLogs { 111 | w.dropRemovedLogs(logList) 112 | } 113 | } 114 | 115 | // addNewlyAvailableLogs checks the transparency log list for new Log servers and adds workers for those to the watcher. 116 | func (w *Watcher) addNewlyAvailableLogs(logList loglist3.LogList) { 117 | log.Println("Checking for new ct logs...") 118 | 119 | w.workersMu.Lock() 120 | defer w.workersMu.Unlock() 121 | newCTs := 0 122 | 123 | // Check the ct log list for new, unwatched logs 124 | // For each CT log, create a worker and start downloading certs 125 | for _, operator := range logList.Operators { 126 | // Iterate over each log of the operator 127 | for _, transparencyLog := range operator.Logs { 128 | newURL := normalizeCtlogURL(transparencyLog.URL) 129 | 130 | if transparencyLog.State.LogStatus() == loglist3.RetiredLogStatus { 131 | log.Printf("Skipping retired CT log: %s\n", newURL) 132 | continue 133 | } 134 | 135 | // Check if the log is already being watched 136 | alreadyWatched := false 137 | 138 | for _, ctWorker := range w.workers { 139 | workerURL := normalizeCtlogURL(ctWorker.ctURL) 140 | if workerURL == newURL { 141 | alreadyWatched = true 142 | break 143 | } 144 | } 145 | 146 | // If the log is already being watched, continue 147 | if alreadyWatched { 148 | continue 149 | } 150 | 151 | w.wg.Add(1) 152 | newCTs++ 153 | 154 | // Metrics are initialized with 0. 155 | // Only if recovery is enabled, it is initialized with the last saved index. 156 | lastCTIndex := metrics.GetCTIndex(normalizeCtlogURL(transparencyLog.URL)) 157 | ctWorker := worker{ 158 | name: transparencyLog.Description, 159 | operatorName: operator.Name, 160 | ctURL: transparencyLog.URL, 161 | entryChan: w.certChan, 162 | ctIndex: lastCTIndex, 163 | } 164 | w.workers = append(w.workers, &ctWorker) 165 | metrics.Init(operator.Name, normalizeCtlogURL(transparencyLog.URL)) 166 | 167 | // Start a goroutine for each worker 168 | go func() { 169 | defer w.wg.Done() 170 | ctWorker.startDownloadingCerts(w.context) 171 | w.discardWorker(&ctWorker) 172 | }() 173 | } 174 | } 175 | 176 | log.Printf("New ct logs found: %d\n", newCTs) 177 | log.Printf("Currently monitored ct logs: %d\n", len(w.workers)) 178 | } 179 | 180 | // discardWorker removes a worker from the watcher's list of workers. 181 | // This needs to be done when a worker stops. 182 | func (w *Watcher) discardWorker(worker *worker) { 183 | log.Println("Removing worker for CT log:", worker.ctURL) 184 | 185 | w.workersMu.Lock() 186 | defer w.workersMu.Unlock() 187 | 188 | for i, wo := range w.workers { 189 | if wo == worker { 190 | w.workers = append(w.workers[:i], w.workers[i+1:]...) 191 | return 192 | } 193 | } 194 | } 195 | 196 | // dropRemovedLogs checks if any of the currently monitored logs are no longer in the log list or are retired. 197 | // If they are not, the CT Logs are probably no longer relevant and the corresponding workers will be stopped. 198 | func (w *Watcher) dropRemovedLogs(logList loglist3.LogList) { 199 | removedCTs := 0 200 | 201 | // Iterate over all workers and check if they are still in the logList 202 | // If they are not, the CT Logs are probably no longer relevant. 203 | // We should stop the worker if that didn't already happen. 204 | for _, ctWorker := range w.workers { 205 | workerURL := normalizeCtlogURL(ctWorker.ctURL) 206 | 207 | onLogList := false 208 | for _, operator := range logList.Operators { 209 | if ctWorker.operatorName != operator.Name { 210 | // This operator is not the one we're looking for 211 | continue 212 | } 213 | 214 | // Iterate over each log of the operator 215 | for _, transparencyLog := range operator.Logs { 216 | // Remove retired logs from the list 217 | if transparencyLog.State.LogStatus() == loglist3.RetiredLogStatus { 218 | // Skip retired logs 219 | continue 220 | } 221 | 222 | // Check if the log is already being watched 223 | logListURL := normalizeCtlogURL(transparencyLog.URL) 224 | if workerURL == logListURL { 225 | onLogList = true 226 | break 227 | } 228 | } 229 | 230 | // Prevent further loop iterations 231 | if onLogList { 232 | break 233 | } 234 | } 235 | 236 | // Make sure to not drop logs that are defined locally in the additional logs list 237 | for _, additionalLogConfig := range config.AppConfig.General.AdditionalLogs { 238 | additionalLogListURL := normalizeCtlogURL(additionalLogConfig.URL) 239 | if workerURL == additionalLogListURL { 240 | onLogList = true 241 | break 242 | } 243 | } 244 | 245 | // If the log is not in the loglist, stop the worker 246 | if !onLogList { 247 | log.Printf("Stopping worker. CT URL not found in LogList or retired: '%s'\n", ctWorker.ctURL) 248 | removedCTs++ 249 | ctWorker.stop() 250 | } 251 | } 252 | 253 | log.Printf("Removed ct logs: %d\n", removedCTs) 254 | log.Printf("Currently monitored ct logs: %d\n", len(w.workers)) 255 | } 256 | 257 | // Stop stops the watcher. 258 | func (w *Watcher) Stop() { 259 | log.Printf("Stopping watcher\n") 260 | 261 | if config.AppConfig.General.Recovery.Enabled { 262 | // Store current CT Indexes before shutting down 263 | filePath := config.AppConfig.General.Recovery.CTIndexFile 264 | metrics.SaveCertIndexes(filePath) 265 | } 266 | 267 | w.cancelFunc() 268 | } 269 | 270 | // CreateIndexFile creates a ct_index.json file based on the current STHs of all availble logs. 271 | func (w *Watcher) CreateIndexFile(filePath string) error { 272 | logs, err := getAllLogs() 273 | if err != nil { 274 | return err 275 | } 276 | 277 | w.context, w.cancelFunc = context.WithCancel(context.Background()) 278 | log.Println("Fetching current STH for all logs...") 279 | for _, operator := range logs.Operators { 280 | // Iterate over each log of the operator 281 | for _, transparencyLog := range operator.Logs { 282 | // Check if the log is already being watched 283 | metrics.Init(operator.Name, normalizeCtlogURL(transparencyLog.URL)) 284 | log.Println("Fetching STH for", normalizeCtlogURL(transparencyLog.URL)) 285 | 286 | hc := http.Client{Timeout: 5 * time.Second} 287 | jsonClient, e := client.New(transparencyLog.URL, &hc, jsonclient.Options{UserAgent: userAgent}) 288 | if e != nil { 289 | log.Printf("Error creating JSON client: %s\n", e) 290 | continue 291 | } 292 | 293 | sth, getSTHerr := jsonClient.GetSTH(w.context) 294 | if getSTHerr != nil { 295 | // TODO this can happen due to a 429 error. We should retry the request 296 | log.Printf("Could not get STH for '%s': %s\n", transparencyLog.URL, getSTHerr) 297 | continue 298 | } 299 | 300 | metrics.SetCTIndex(normalizeCtlogURL(transparencyLog.URL), sth.TreeSize) 301 | } 302 | } 303 | w.cancelFunc() 304 | 305 | metrics.SaveCertIndexes(filePath) 306 | log.Println("Index file saved to", filePath) 307 | 308 | return nil 309 | } 310 | 311 | // A worker processes a single CT log. 312 | type worker struct { 313 | name string 314 | operatorName string 315 | ctURL string 316 | entryChan chan models.Entry 317 | ctIndex uint64 318 | mu sync.Mutex 319 | running bool 320 | cancel context.CancelFunc 321 | } 322 | 323 | // startDownloadingCerts starts downloading certificates from the CT log. This method is blocking. 324 | func (w *worker) startDownloadingCerts(ctx context.Context) { 325 | ctx, w.cancel = context.WithCancel(ctx) 326 | 327 | // Normalize CT URL. We remove trailing slashes and prepend "https://" if it's not already there. 328 | w.ctURL = strings.TrimRight(w.ctURL, "/") 329 | if !strings.HasPrefix(w.ctURL, "https://") && !strings.HasPrefix(w.ctURL, "http://") { 330 | w.ctURL = "https://" + w.ctURL 331 | } 332 | 333 | log.Printf("Initializing worker for CT log: %s\n", w.ctURL) 334 | defer log.Printf("Stopping worker for CT log: %s\n", w.ctURL) 335 | 336 | w.mu.Lock() 337 | if w.running { 338 | log.Printf("Worker for '%s' already running\n", w.ctURL) 339 | w.mu.Unlock() 340 | 341 | return 342 | } 343 | 344 | w.running = true 345 | defer func() { w.running = false }() 346 | w.mu.Unlock() 347 | 348 | for { 349 | log.Printf("Starting worker for CT log: %s\n", w.ctURL) 350 | workerErr := w.runWorker(ctx) 351 | if workerErr != nil { 352 | if errors.Is(workerErr, errFetchingSTHFailed) { 353 | // TODO this could happen due to a 429 error. We should retry the request 354 | log.Printf("Worker for '%s' failed - could not fetch STH\n", w.ctURL) 355 | return 356 | } else if errors.Is(workerErr, errCreatingClient) { 357 | log.Printf("Worker for '%s' failed - could not create client\n", w.ctURL) 358 | return 359 | } else if strings.Contains(workerErr.Error(), "no such host") { 360 | log.Printf("Worker for '%s' failed to resolve host: %s\n", w.ctURL, workerErr) 361 | return 362 | } 363 | 364 | log.Printf("Worker for '%s' failed with unexpected error: %s\n", w.ctURL, workerErr) 365 | } 366 | 367 | // Check if the context was cancelled 368 | select { 369 | case <-ctx.Done(): 370 | log.Printf("Context was cancelled; Stopping worker for '%s'\n", w.ctURL) 371 | 372 | return 373 | default: 374 | log.Printf("Worker for '%s' sleeping for 5 seconds due to error\n", w.ctURL) 375 | time.Sleep(5 * time.Second) 376 | log.Printf("Restarting worker for '%s'\n", w.ctURL) 377 | 378 | continue 379 | } 380 | } 381 | } 382 | 383 | func (w *worker) stop() { 384 | w.mu.Lock() 385 | defer w.mu.Unlock() 386 | 387 | w.cancel() 388 | } 389 | 390 | // runWorker runs a single worker for a single CT log. This method is blocking. 391 | func (w *worker) runWorker(ctx context.Context) error { 392 | hc := http.Client{Timeout: 30 * time.Second} 393 | jsonClient, e := client.New(w.ctURL, &hc, jsonclient.Options{UserAgent: userAgent}) 394 | if e != nil { 395 | log.Printf("Error creating JSON client: %s\n", e) 396 | return errCreatingClient 397 | } 398 | 399 | // If recovery is enabled and the CT index is set, we start at the saved index. Otherwise we start at the latest STH. 400 | validSavedCTIndexExists := config.AppConfig.General.Recovery.Enabled 401 | if !validSavedCTIndexExists { 402 | sth, getSTHerr := jsonClient.GetSTH(ctx) 403 | if getSTHerr != nil { 404 | // TODO this can happen due to a 429 error. We should retry the request 405 | log.Printf("Could not get STH for '%s': %s\n", w.ctURL, getSTHerr) 406 | return errFetchingSTHFailed 407 | } 408 | // Start at the latest STH to skip all the past certificates 409 | w.ctIndex = sth.TreeSize 410 | } 411 | 412 | certScanner := scanner.NewScanner(jsonClient, scanner.ScannerOptions{ 413 | FetcherOptions: scanner.FetcherOptions{ 414 | BatchSize: 100, 415 | ParallelFetch: 1, 416 | StartIndex: int64(w.ctIndex), 417 | Continuous: true, 418 | }, 419 | Matcher: scanner.MatchAll{}, 420 | PrecertOnly: false, 421 | NumWorkers: 1, 422 | BufferSize: config.AppConfig.General.BufferSizes.CTLog, 423 | }) 424 | 425 | scanErr := certScanner.Scan(ctx, w.foundCertCallback, w.foundPrecertCallback) 426 | if scanErr != nil { 427 | log.Println("Scan error: ", scanErr) 428 | return scanErr 429 | } 430 | 431 | log.Printf("Exiting worker %s without error!\n", w.ctURL) 432 | 433 | return nil 434 | } 435 | 436 | // foundCertCallback is the callback that handles cases where new regular certs are found. 437 | func (w *worker) foundCertCallback(rawEntry *ct.RawLogEntry) { 438 | entry, parseErr := ParseCertstreamEntry(rawEntry, w.operatorName, w.name, w.ctURL) 439 | if parseErr != nil { 440 | log.Println("Error parsing certstream entry: ", parseErr) 441 | return 442 | } 443 | 444 | entry.Data.UpdateType = "X509LogEntry" 445 | w.entryChan <- entry 446 | 447 | atomic.AddInt64(&processedCerts, 1) 448 | } 449 | 450 | // foundPrecertCallback is the callback that handles cases where new precerts are found. 451 | func (w *worker) foundPrecertCallback(rawEntry *ct.RawLogEntry) { 452 | entry, parseErr := ParseCertstreamEntry(rawEntry, w.operatorName, w.name, w.ctURL) 453 | if parseErr != nil { 454 | log.Println("Error parsing certstream entry: ", parseErr) 455 | return 456 | } 457 | 458 | entry.Data.UpdateType = "PrecertLogEntry" 459 | w.entryChan <- entry 460 | 461 | atomic.AddInt64(&processedPrecerts, 1) 462 | } 463 | 464 | // certHandler takes the entries out of the entryChan channel and broadcasts them to all clients. 465 | // Only a single instance of the certHandler runs per certstream server. 466 | func certHandler(entryChan chan models.Entry) { 467 | var processed int64 468 | 469 | for { 470 | entry := <-entryChan 471 | processed++ 472 | 473 | if processed%1000 == 0 { 474 | log.Printf("Processed %d entries | Queue length: %d\n", processed, len(entryChan)) 475 | // Every thousandth entry, we store one certificate as example 476 | web.SetExampleCert(entry) 477 | } 478 | 479 | // Run json encoding in the background and send the result to the clients. 480 | web.ClientHandler.Broadcast <- entry 481 | 482 | // Update metrics 483 | url := entry.Data.Source.NormalizedURL 484 | operator := entry.Data.Source.Operator 485 | index := entry.Data.CertIndex 486 | 487 | metrics.Inc(operator, url, index) 488 | } 489 | } 490 | 491 | // getGoogleLogList fetches the list of all CT logs from Google Chromes CT LogList. 492 | func getGoogleLogList() (loglist3.LogList, error) { 493 | // Download the list of all logs from ctLogInfo and decode json 494 | resp, err := http.Get(loglist3.LogListURL) 495 | if err != nil { 496 | return loglist3.LogList{}, err 497 | } 498 | defer resp.Body.Close() 499 | 500 | if resp.StatusCode != http.StatusOK { 501 | return loglist3.LogList{}, errors.New("failed to download loglist") 502 | } 503 | 504 | bodyBytes, readErr := io.ReadAll(resp.Body) 505 | if readErr != nil { 506 | log.Panic(readErr) 507 | } 508 | 509 | allLogs, parseErr := loglist3.NewFromJSON(bodyBytes) 510 | if parseErr != nil { 511 | return loglist3.LogList{}, parseErr 512 | } 513 | 514 | return *allLogs, nil 515 | } 516 | 517 | // getAllLogs returns a list of all CT logs. 518 | func getAllLogs() (loglist3.LogList, error) { 519 | var allLogs loglist3.LogList 520 | var err error 521 | 522 | // Ability to disable default logs, if the user only wants to monitor custom logs. 523 | if !config.AppConfig.General.DisableDefaultLogs { 524 | allLogs, err = getGoogleLogList() 525 | if err != nil { 526 | log.Printf("Error fetching log list from Google: %s\n", err) 527 | return loglist3.LogList{}, fmt.Errorf("failed to fetch log list from Google: %w", err) 528 | } 529 | } 530 | 531 | // Add manually added logs from config to the allLogs list 532 | if config.AppConfig.General.AdditionalLogs == nil { 533 | return allLogs, nil 534 | } 535 | 536 | for _, additionalLog := range config.AppConfig.General.AdditionalLogs { 537 | customLog := loglist3.Log{ 538 | URL: additionalLog.URL, 539 | Description: additionalLog.Description, 540 | } 541 | 542 | operatorFound := false 543 | for _, operator := range allLogs.Operators { 544 | if operator.Name == additionalLog.Operator { 545 | // TODO Check if the log is already in the list 546 | operator.Logs = append(operator.Logs, &customLog) 547 | operatorFound = true 548 | 549 | break 550 | } 551 | } 552 | 553 | if !operatorFound { 554 | newOperator := loglist3.Operator{ 555 | Name: additionalLog.Operator, 556 | Logs: []*loglist3.Log{&customLog}, 557 | } 558 | allLogs.Operators = append(allLogs.Operators, &newOperator) 559 | } 560 | } 561 | 562 | return allLogs, nil 563 | } 564 | 565 | func normalizeCtlogURL(input string) string { 566 | input = strings.TrimPrefix(input, "https://") 567 | input = strings.TrimPrefix(input, "http://") 568 | input = strings.TrimSuffix(input, "/") 569 | 570 | return input 571 | } 572 | --------------------------------------------------------------------------------