├── .dockerignore ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── PATENTS ├── README.md ├── all_test.go ├── checks.bash ├── cmd ├── ejobs │ └── main.go ├── ejson2csv │ └── main.go ├── govulncheck_compare │ ├── govulncheck_compare.go │ └── govulncheck_compare_test.go ├── govulncheck_sandbox │ ├── govulncheck_sandbox.go │ └── govulncheck_sandbox_test.go ├── vulndbreqs │ └── main.go └── worker │ ├── Dockerfile │ └── main.go ├── config.json.commented ├── deploy └── worker.yaml ├── devtools ├── bqerrors.sh ├── deploy.sh └── lib.sh ├── go.mod ├── go.sum ├── internal ├── analysis │ ├── analysis.go │ └── analysis_test.go ├── bigquery │ ├── bigquery.go │ └── bigquery_test.go ├── buildbinary │ ├── bin.go │ └── bin_test.go ├── buildtest │ ├── buildtest.go │ └── buildtest_test.go ├── config │ └── config.go ├── derrors │ └── derrors.go ├── ejson2csv │ ├── ejson2csv.go │ ├── ejson2csv_test.go │ └── testdata │ │ └── sample.json ├── fstore │ └── fstore.go ├── goenv.go ├── goenv_test.go ├── govulncheck │ ├── govulncheck.go │ ├── govulncheck_test.go │ ├── govulncheck_unix.go │ └── handler.go ├── govulncheckapi │ ├── handler.go │ └── result.go ├── jobs │ ├── firestore.go │ ├── firestore_test.go │ └── job.go ├── log │ ├── cloud_handler.go │ ├── line_handler.go │ └── log.go ├── modules │ ├── modules.go │ └── modules_test.go ├── observe │ └── observe.go ├── osv │ ├── osv.go │ └── review_status.go ├── pkgsitedb │ ├── db.go │ ├── db_plan9.go │ └── db_test.go ├── proxy │ ├── cache.go │ ├── client.go │ ├── client_test.go │ ├── proxytest │ │ ├── module.go │ │ ├── proxytest.go │ │ └── server.go │ └── testdata │ │ ├── basic@v1.0.0.txtar │ │ ├── basic@v1.1.0.txtar │ │ ├── build-constraints@v1.0.0.txtar │ │ ├── deprecated@v1.0.0.txtar │ │ ├── deprecated@v1.1.0.txtar │ │ ├── generics@v1.0.0.txtar │ │ ├── multi@v1.0.0.txtar │ │ ├── nonredist@v1.0.0.txtar │ │ ├── quote@v1.0.0.txtar │ │ ├── quote@v1.1.0.txtar │ │ ├── quote@v1.2.0.txtar │ │ ├── quote@v1.3.0.txtar │ │ ├── quote@v1.4.0.txtar │ │ ├── quote@v1.5.0.txtar │ │ ├── quote@v3.0.0.txtar │ │ ├── quote@v3.1.0.txtar │ │ ├── retractions@v1.0.0.txtar │ │ ├── retractions@v1.1.0.txtar │ │ ├── retractions@v1.2.0.txtar │ │ ├── single@v1.0.0.txtar │ │ ├── symbols@v1.0.0.txtar │ │ ├── symbols@v1.1.0.txtar │ │ └── symbols@v1.2.0.txtar ├── queue │ ├── queue.go │ └── queue_test.go ├── sandbox │ ├── Makefile │ ├── runner.go │ ├── sandbox.go │ ├── sandbox_test.go │ └── testdata │ │ ├── bundle │ │ ├── config.json │ │ └── rootfs │ │ │ ├── .dockerenv │ │ │ └── etc │ │ │ ├── hostname │ │ │ ├── hosts │ │ │ ├── mtab │ │ │ └── resolv.conf │ │ └── printargs.go ├── scan │ ├── parse.go │ ├── parse_test.go │ └── testdata │ │ └── modules.txt ├── secrets.go ├── testdata │ ├── module │ │ ├── go.mod │ │ ├── go.sum │ │ └── vuln.go │ ├── multipleBinModule │ │ ├── go.mod │ │ ├── main.go │ │ ├── multipleBinModule │ │ │ └── main.go │ │ ├── p1 │ │ │ └── main.go │ │ └── p2 │ │ │ └── file.go │ └── vulndb │ │ ├── ID │ │ ├── GO-2020-0015.json │ │ └── GO-2021-0113.json │ │ └── index │ │ ├── db.json │ │ ├── modules.json │ │ └── vulns.json ├── testing │ ├── testenv.go │ └── testhelper │ │ └── testhelper.go ├── version │ ├── version.go │ └── version_test.go ├── vulndb │ ├── vulndb.go │ └── vulndb_test.go ├── vulndbreqs │ ├── bq.go │ ├── bq_test.go │ ├── compute.go │ ├── compute_test.go │ ├── iterator.go │ └── testdata │ │ └── logfile.json └── worker │ ├── analysis.go │ ├── analysis_test.go │ ├── enqueue.go │ ├── govulncheck.go │ ├── govulncheck_enqueue.go │ ├── govulncheck_enqueue_test.go │ ├── govulncheck_scan.go │ ├── govulncheck_scan_test.go │ ├── jobs.go │ ├── jobs_test.go │ ├── scan.go │ ├── scan_test.go │ ├── server.go │ ├── testdata │ ├── analyzer │ │ └── analyzer.go │ ├── module │ │ ├── a.go │ │ └── go.mod │ └── modules.txt │ ├── vulndb.go │ └── vulndb_test.go ├── terraform ├── README.md ├── environment │ └── worker.tf └── main.tf └── tools.go /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | devtools 3 | deploy 4 | terraform 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.terraform/* 2 | .terraform.lock.hcl 3 | go-image.tar.gz 4 | go-vulndb 5 | config.json 6 | docker-build 7 | vars.env 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Go 2 | 3 | This repository is part of the Go open source project. 4 | 5 | It is the work of hundreds of contributors. We appreciate your help! 6 | 7 | ## Contributing code 8 | 9 | Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) before sending patches. 10 | 11 | Unless otherwise noted, the Go source files are distributed under 12 | the BSD-style license found in the LICENSE file. 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 The Go Authors. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google LLC nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | # Makefile for common tasks. 6 | 7 | default: 8 | @echo "usage: make TARGET" 9 | 10 | # Copy a gzipped tar of a Go docker image from a bucket. 11 | # This is the image that is used by the sandbox. 12 | # The file must live in the repo directory so that cmd/worker/Dockerfile 13 | # can access it. 14 | # We assume that the file exists in the bucket. 15 | # See the rule below for how to create the image. 16 | go-image.tar.gz: 17 | gsutil cp gs://go-ecosystem/$@ $@ 18 | 19 | # Use this rule to build a go image for a specific Go version. 20 | # E.g. 21 | # make go-image-1.19.4.tar.gz 22 | # To change the sandbox image permanently, copy it to GCP: 23 | # 24 | # gsutil cp go-image.1.19.4.tar.gz gs://go-ecosystem/go-image.tar.gz 25 | # Then delete the local copy. 26 | go-image-%.tar.gz: 27 | docker export $(shell docker create golang:$*) | gzip > go-image-$*.tar.gz 28 | 29 | # Download the Go vulnerability DB to a local directory, so vulndb can access it 30 | # from the sandbox, which has no network connectivity. 31 | # 32 | # Get the last-modified time of the index.json file, which is reported as the 33 | # last-modified time of the DB, and save it to a local file. (The last-modified 34 | # time of the local index.json is not what we want: it is the time that gsutil cp 35 | # wrote the file.) 36 | # 37 | # This directory must live in the repo directory so that cmd/worker/Dockerfile 38 | # can access it. 39 | go-vulndb: 40 | gsutil -m -q cp -r gs://go-vulndb . 41 | gsutil stat gs://go-vulndb/index.json | \ 42 | awk '$$1 == "Update" { for (i = 4; i <= NF; i++) printf("%s ", $$i); printf("\n"); }' \ 43 | > go-vulndb/LAST_MODIFIED 44 | 45 | # Remove comments from a json file. 46 | %.json: %.json.commented 47 | sed '/^[ \t]*#/d' $< > $@ 48 | 49 | IMAGE := ecosystem-worker-test 50 | 51 | # Enable the docker container to authenticate to Google Cloud. 52 | # This assumes the user has run "gcloud auth application-default login". 53 | DOCKER_AUTH_ARGS := -v "$(HOME)/.config/gcloud:/creds" \ 54 | --env GOOGLE_APPLICATION_CREDENTIALS=/creds/application_default_credentials.json 55 | 56 | DOCKER_RUN_ARGS := --rm --privileged -p 8080:8080 \ 57 | --env GO_ECOSYSTEM_BINARY_BUCKET=go-ecosystem \ 58 | $(DOCKER_AUTH_ARGS) 59 | 60 | DOCKER_ID_FILE := /tmp/ecosystem-docker-container-id 61 | 62 | 63 | # Build a docker image for testing. 64 | # This target is a local file that marks the time of the last 65 | # docker build. We use a file because make uses only local file timestamps to determine 66 | # whether a target needs to be regenerated. 67 | docker-build: go-image.tar.gz go-vulndb cmd/worker/*.go internal/**/*.go cmd/govulncheck_sandbox/* config.json cmd/worker/Dockerfile 68 | docker build -f cmd/worker/Dockerfile -t $(IMAGE) . \ 69 | --build-arg DOCKER_IMAGE=$(IMAGE) \ 70 | --build-arg BQ_DATASET=disable 71 | touch $@ 72 | 73 | 74 | # Run the docker image locally, for testing. 75 | # The worker will start and listen at port 8080. 76 | docker-run: docker-build 77 | docker run $(DOCKER_RUN_ARGS) $(IMAGE) 78 | 79 | # Run the docker image and enter an interactive shell. 80 | # The worker does not start. 81 | docker-run-shell: docker-build 82 | docker run -it $(DOCKER_RUN_ARGS) $(IMAGE) /bin/bash 83 | 84 | # Run the docker image in the background, waiting until the server is ready. 85 | docker-run-bg: docker-build 86 | docker run --detach $(DOCKER_RUN_ARGS) $(IMAGE) > $(DOCKER_ID_FILE) 87 | while ! curl -s --head http://localhost:8080 > /dev/null; do sleep 1; done 88 | 89 | test: docker-run-bg govulncheck-test analysis-test 90 | docker container stop `cat $(DOCKER_ID_FILE)` 91 | 92 | GOVULNCHECK_TEST_FILE := /tmp/vtest.out 93 | 94 | # Test by scanning a small module. 95 | govulncheck-test: 96 | curl -s 'http://localhost:8080/govulncheck/scan/github.com/fossas/fossa-cli@v1.1.10?importedby=1&serve=true' > $(GOVULNCHECK_TEST_FILE) 97 | @if [[ `grep -c GO-2020-0016 $(GOVULNCHECK_TEST_FILE)` -ge 2 ]]; then \ 98 | echo PASS; \ 99 | rm $(GOVULNCHECK_TEST_FILE); \ 100 | else \ 101 | echo FAIL; \ 102 | echo "output in $(GOVULNCHECK_TEST_FILE)"; \ 103 | docker container stop `cat $(DOCKER_ID_FILE)`; \ 104 | exit 1; \ 105 | fi 106 | 107 | ANALYSIS_TEST_FILE := /tmp/atest.out 108 | 109 | analysis-test: 110 | curl -sa 'http://localhost:8080/analysis/scan/github.com/jba/cli@v0.6.0?binary=findcall&args=-name+stringsCut&serve=true' > $(ANALYSIS_TEST_FILE) 111 | @if grep -q Diagnostics $(ANALYSIS_TEST_FILE); then \ 112 | echo PASS; \ 113 | rm $(ANALYSIS_TEST_FILE); \ 114 | else \ 115 | echo FAIL; \ 116 | echo "output in $(ANALYSIS_TEST_FILE)"; \ 117 | docker container stop `cat $(DOCKER_ID_FILE)`; \ 118 | exit 1; \ 119 | fi 120 | 121 | clean: 122 | rm -f go-image.tar.gz 123 | rm -rf go-vulndb 124 | rm -f config.json 125 | rm -f govulncheck_sandbox 126 | 127 | .PHONY: docker-run docker-run-bg test govulncheck-test analysis-test \ 128 | clean build-go-image 129 | -------------------------------------------------------------------------------- /PATENTS: -------------------------------------------------------------------------------- 1 | Additional IP Rights Grant (Patents) 2 | 3 | "This implementation" means the copyrightable works distributed by 4 | Google as part of the Go project. 5 | 6 | Google hereby grants to You a perpetual, worldwide, non-exclusive, 7 | no-charge, royalty-free, irrevocable (except as stated in this section) 8 | patent license to make, have made, use, offer to sell, sell, import, 9 | transfer and otherwise run, modify and propagate the contents of this 10 | implementation of Go, where such license applies only to those patent 11 | claims, both currently owned or controlled by Google and acquired in 12 | the future, licensable by Google that are necessarily infringed by this 13 | implementation of Go. This grant does not include claims that would be 14 | infringed only as a consequence of further modification of this 15 | implementation. If you or your agent or exclusive licensee institute or 16 | order or agree to the institution of patent litigation against any 17 | entity (including a cross-claim or counterclaim in a lawsuit) alleging 18 | that this implementation of Go or any code incorporated within this 19 | implementation of Go constitutes direct or contributory patent 20 | infringement, or inducement of patent infringement, then any patent 21 | rights granted to you under this License for this implementation of Go 22 | shall terminate as of the date such litigation is filed. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pkgsite-metrics 2 | 3 | This repository contains code that enables collecting and evaluating 4 | metrics for the Go ecosystem. 5 | 6 | ## Report Issues / Send Patches 7 | 8 | This repository uses Gerrit for code changes. To learn how to submit changes to 9 | this repository, see https://golang.org/doc/contribute.html. 10 | 11 | The main issue tracker for the time repository is located at 12 | https://github.com/golang/go/issues. Prefix your issue with 13 | "x/pkgsite-metrics:" in the subject line, so it is easy to find. 14 | -------------------------------------------------------------------------------- /all_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.17 && !windows 6 | 7 | package main 8 | 9 | import ( 10 | "bufio" 11 | "io/fs" 12 | "os" 13 | "regexp" 14 | "strings" 15 | "testing" 16 | ) 17 | 18 | var goHeader = regexp.MustCompile(`^// Copyright 20\d\d The Go Authors\. All rights reserved\. 19 | // Use of this source code is governed by a BSD-style 20 | // license that can be found in the LICENSE file\.`) 21 | 22 | func TestHeaders(t *testing.T) { 23 | sfs := os.DirFS(".") 24 | fs.WalkDir(sfs, ".", func(path string, d fs.DirEntry, _ error) error { 25 | if d.IsDir() { 26 | if d.Name() == "testdata" { 27 | return fs.SkipDir 28 | } 29 | return nil 30 | } 31 | if !strings.HasSuffix(path, ".go") { 32 | return nil 33 | } 34 | f, err := sfs.Open(path) 35 | if err != nil { 36 | return err 37 | } 38 | defer f.Close() 39 | if !goHeader.MatchReader(bufio.NewReader(f)) { 40 | t.Errorf("%v: incorrect go header", path) 41 | } 42 | return nil 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /checks.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2023 The Go Authors. All rights reserved. 3 | # Use of this source code is governed by a BSD-style 4 | # license that can be found in the LICENSE file. 5 | 6 | # Not run by unit tests. Intended to help authors 7 | # during development. 8 | 9 | go version 10 | 11 | # Ensure that installed go binaries are on the path. 12 | # This bash expression follows the algorithm described at the top of 13 | # `go install help`: first try $GOBIN, then $GOPATH/bin, then $HOME/go/bin. 14 | go_install_dir=${GOBIN:-${GOPATH:-$HOME/go}/bin} 15 | PATH=$PATH:$go_install_dir 16 | 17 | source devtools/lib.sh 18 | 19 | # ensure_go_binary verifies that a binary exists in $PATH corresponding to the 20 | # given go-gettable URI. If no such binary exists, it is fetched via `go get`. 21 | ensure_go_binary() { 22 | local binary=$(basename $1) 23 | if ! [ -x "$(command -v $binary)" ]; then 24 | info "Installing: $1" 25 | # Install the binary in a way that doesn't affect our go.mod file. 26 | go install $1 27 | fi 28 | } 29 | 30 | # check_vet runs go vet on source files. 31 | check_vet() { 32 | runcmd go vet -all ./... 33 | } 34 | 35 | # check_staticcheck runs staticcheck on source files. 36 | check_staticcheck() { 37 | ensure_go_binary honnef.co/go/tools/cmd/staticcheck 38 | runcmd staticcheck ./... 39 | } 40 | 41 | # check_misspell runs misspell on source files. 42 | check_misspell() { 43 | ensure_go_binary github.com/client9/misspell/cmd/misspell 44 | runcmd misspell -error $(find . -name .git -prune \ 45 | -o -name .terraform -prune \ 46 | -o -type f -not -name modules.txt -not -name '*.svg' -not -name '*.ts.snap' -not -name '*.json') 47 | } 48 | 49 | # check_integration prints a warning if the environment 50 | # variable for integration testing is not set. 51 | check_integration() { 52 | if [[ "${GO_ECOSYSTEM_INTEGRATION_TESTING}" != "1" ]]; then 53 | warn "Running go test ./... will skip integration tests (GO_ECOSYSTEM_INTEGRATION_TESTING != 1)" 54 | fi 55 | } 56 | 57 | go_linters() { 58 | check_vet 59 | check_staticcheck 60 | check_misspell 61 | } 62 | 63 | go_modtidy() { 64 | runcmd go mod tidy 65 | } 66 | 67 | runchecks() { 68 | check_integration 69 | go_linters 70 | go_modtidy 71 | } 72 | 73 | usage() { 74 | cat <0 scan seconds", resp.Stats.ScanSeconds) 61 | } 62 | if resp.Stats.ScanMemory <= 0 { 63 | t.Errorf("got %d; want >0 scan memory", resp.Stats.ScanMemory) 64 | } 65 | }) 66 | 67 | // Errors 68 | for _, test := range []struct { 69 | name string 70 | args []string 71 | want string 72 | }{ 73 | { 74 | name: "too few args", 75 | args: []string{"testdata/module", vulndb}, 76 | want: "need four args", 77 | }, 78 | { 79 | name: "no vulndb", 80 | args: []string{govulncheckPath, govulncheck.FlagSource, module, "DNE"}, 81 | want: "URL missing path", 82 | }, 83 | { 84 | name: "no mode", 85 | args: []string{govulncheckPath, "unsupported mode", module, vulndb}, 86 | want: "invalid value", 87 | }, 88 | { 89 | name: "no mode", 90 | args: []string{govulncheckPath, govulncheck.FlagBinary, module, vulndb}, 91 | want: "binaries are only analyzed", 92 | }, 93 | { 94 | name: "no module", 95 | args: []string{govulncheckPath, govulncheck.FlagSource, "nosuchmodule", vulndb}, 96 | // Once govulncheck destinguishes this issue from no .mod file, 97 | // update want to reflect govulncheck's new output 98 | want: "no go.mod", 99 | }, 100 | } { 101 | t.Run(test.name, func(t *testing.T) { 102 | _, err := runTest(test.args) 103 | if err == nil { 104 | t.Fatal("got nil, want error") 105 | } 106 | if g, w := err.Error(), test.want; !strings.Contains(g, w) { 107 | t.Fatalf("error %q does not contain %q", g, w) 108 | } 109 | }) 110 | } 111 | } 112 | 113 | func runTest(args []string) (*govulncheck.AnalysisResponse, error) { 114 | var buf bytes.Buffer 115 | run(&buf, args) 116 | return govulncheck.UnmarshalAnalysisResponse(buf.Bytes()) 117 | } 118 | -------------------------------------------------------------------------------- /cmd/vulndbreqs/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // TODO(jba): delete when the worker is reliably computing 6 | // request counts. 7 | 8 | // Command vulndbreqs inserts and displays vuln DB request counts. 9 | package main 10 | 11 | import ( 12 | "context" 13 | "errors" 14 | "flag" 15 | "fmt" 16 | "log" 17 | 18 | "cloud.google.com/go/civil" 19 | "golang.org/x/pkgsite-metrics/internal" 20 | "golang.org/x/pkgsite-metrics/internal/bigquery" 21 | "golang.org/x/pkgsite-metrics/internal/config" 22 | "golang.org/x/pkgsite-metrics/internal/vulndbreqs" 23 | ) 24 | 25 | var date = flag.String("date", "", "date for compute") 26 | 27 | func main() { 28 | flag.Usage = func() { 29 | out := flag.CommandLine.Output() 30 | fmt.Fprintln(out, "usage:") 31 | fmt.Fprintln(out, "vulndbreqs add [DATE]") 32 | fmt.Fprintln(out, " calculate missing vuln DB counts and add to BigQuery") 33 | fmt.Fprintln(out, "vulndbreqs compute") 34 | fmt.Fprintln(out, " calculate and display vuln DB counts") 35 | fmt.Fprintln(out, "vulndbreqs show") 36 | fmt.Fprintln(out, " display vuln DB counts") 37 | flag.PrintDefaults() 38 | } 39 | 40 | flag.Parse() 41 | if err := run(context.Background()); err != nil { 42 | log.Fatal(err) 43 | } 44 | } 45 | 46 | func run(ctx context.Context) error { 47 | cfg, err := config.Init(ctx) 48 | if err != nil { 49 | return err 50 | } 51 | if cfg.ProjectID == "" { 52 | return errors.New("missing project ID (GOOGLE_CLOUD_PROJECT environment variable)") 53 | } 54 | client, err := bigquery.NewClientCreate(ctx, cfg.ProjectID, vulndbreqs.DatasetName) 55 | if err != nil { 56 | return err 57 | } 58 | defer client.Close() 59 | 60 | keyName := "projects/" + cfg.ProjectID + "/secrets/vulndb-hmac-key" 61 | var hmacKey []byte 62 | if flag.Arg(0) == "add" || flag.Arg(0) == "compute" { 63 | hk, err := internal.GetSecret(ctx, keyName) 64 | if err != nil { 65 | return err 66 | } 67 | hmacKey = []byte(hk) 68 | } 69 | 70 | switch flag.Arg(0) { 71 | case "add": 72 | err = doAdd(ctx, cfg.VulnDBBucketProjectID, client, hmacKey, flag.Arg(1)) 73 | case "compute": 74 | err = doCompute(ctx, cfg.VulnDBBucketProjectID, hmacKey) 75 | case "show": 76 | err = doShow(ctx, client) 77 | default: 78 | return fmt.Errorf("unknown command %q", flag.Arg(0)) 79 | } 80 | return err 81 | } 82 | 83 | func doAdd(ctx context.Context, projectID string, client *bigquery.Client, hmacKey []byte, date string) error { 84 | if date == "" { 85 | return vulndbreqs.ComputeAndStore(ctx, projectID, client, hmacKey) 86 | } 87 | d, err := civil.ParseDate(date) 88 | if err != nil { 89 | return err 90 | } 91 | return vulndbreqs.ComputeAndStoreDate(ctx, projectID, client, hmacKey, d) 92 | } 93 | 94 | func doCompute(ctx context.Context, projectID string, hmacKey []byte) error { 95 | d, err := civil.ParseDate(*date) 96 | if err != nil { 97 | return err 98 | } 99 | rcs, err := vulndbreqs.Compute(ctx, projectID, d, hmacKey) 100 | if err != nil { 101 | return err 102 | } 103 | for _, rc := range rcs { 104 | fmt.Printf("%s\t%d\t%s\n", rc.Date, rc.Count, rc.IP) 105 | } 106 | return nil 107 | } 108 | 109 | func doShow(ctx context.Context, client *bigquery.Client) error { 110 | counts, err := vulndbreqs.ReadRequestCountsFromBigQuery(ctx, client) 111 | if err != nil { 112 | return err 113 | } 114 | for _, c := range counts { 115 | fmt.Printf("%s\t%d\n", c.Date, c.Count) 116 | } 117 | return nil 118 | } 119 | -------------------------------------------------------------------------------- /cmd/worker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | # This Dockerfile expects the build context to be the repo root. 6 | 7 | # To test that the worker built with this Dockerfile runs, run 8 | # make test 9 | # from the repo root. 10 | 11 | # NOTE: don't put anything in /tmp here. It will work locally, 12 | # but Cloud Run mounts something else to /tmp, so anything 13 | # installed here will be shadowed. 14 | 15 | 16 | FROM golang:1.23.0 17 | 18 | LABEL maintainer="Go Ecosystem Team " 19 | 20 | #### Preliminaries 21 | 22 | WORKDIR / 23 | 24 | # Create some directories. 25 | 26 | # The worker binary and related files live here. 27 | RUN mkdir /app 28 | 29 | # When debugging the sandbox manually, run this as well: 30 | # RUN mkdir /tmp/modules 31 | 32 | # Where binaries live. 33 | # Mapped by the sandbox config to the same place inside the sandbox. 34 | # If you change this, you must also edit the bind mount in config.json.commented. 35 | # 36 | # We use an ARG command here to make a variable, but this is not intended to be 37 | # provided as a command-line argument to `docker build`. 38 | ARG BINARY_DIR=/app/binaries 39 | 40 | RUN mkdir $BINARY_DIR 41 | 42 | #### Sandbox setup 43 | 44 | # Install runsc. 45 | ADD https://storage.googleapis.com/gvisor/releases/release/20240930.0/x86_64/runsc /usr/local/bin/ 46 | RUN chmod a+rx /usr/local/bin/runsc 47 | 48 | # Set up for runsc. 49 | # runsc expects a directory called a "bundle" that contains a config.json 50 | # file and an OS filesystem. 51 | 52 | # Create the runsc bundle. 53 | WORKDIR /bundle 54 | 55 | # The root of the bundle filesystem. 56 | RUN mkdir rootfs 57 | 58 | # go-image.tar.gz is a complete Docker image of a Go installation in tar format. 59 | # Use it for the bundle's OS filesystem. 60 | COPY go-image.tar.gz . 61 | RUN tar --same-owner -pxzf go-image.tar.gz -C rootfs 62 | 63 | # Copy the downloaded copy of the vuln DB 64 | # into the /app dir similar to binaries. 65 | ARG VULNDB_DIR=/app/go-vulndb 66 | COPY go-vulndb $VULNDB_DIR 67 | 68 | COPY config.json . 69 | 70 | #### Building binaries 71 | 72 | # Set the working directory outside $GOPATH to ensure module mode is enabled. 73 | WORKDIR /src 74 | 75 | # Copy go.mods and go.sums into the container. 76 | # If they don't change, which is the common case, then docker can 77 | # cache these COPYs and the subsequent RUN. 78 | COPY go.mod go.sum checks.bash ./ 79 | 80 | # Copy the repo from local machine into Docker client’s current working 81 | # directory, so that we can use it to build the binary. 82 | # See .dockerignore at the repo root for excluded files. 83 | COPY . /src 84 | 85 | # Download the dependencies. 86 | RUN go mod download 87 | 88 | # Build the worker binary and put it in /app. 89 | RUN go build -mod=readonly -o /app/worker ./cmd/worker 90 | 91 | # TODO: install the latest version of govulncheck? 92 | # Build the version of govulncheck specified in the go.mod file. 93 | RUN go build -o $BINARY_DIR golang.org/x/vuln/cmd/govulncheck 94 | 95 | # Build the program that runs govulncheck inside the sandbox. 96 | RUN go build -mod=readonly -o $BINARY_DIR/govulncheck_sandbox ./cmd/govulncheck_sandbox 97 | 98 | # Build the program that runs govulncheck comparisons inside the sandbox 99 | RUN go build -mod=readonly -o $BINARY_DIR/govulncheck_compare ./cmd/govulncheck_compare 100 | 101 | # Build the sandbox runner program and put it in the bundle root. 102 | RUN go build -mod=readonly -o /bundle/rootfs/runner ./internal/sandbox/runner.go 103 | 104 | 105 | #### Worker setup 106 | 107 | WORKDIR /app 108 | 109 | ARG DOCKER_IMAGE 110 | ENV DOCKER_IMAGE=$DOCKER_IMAGE 111 | 112 | ARG BQ_DATASET 113 | ENV GO_ECOSYSTEM_BIGQUERY_DATASET=$BQ_DATASET 114 | 115 | ARG SERVICE_ID 116 | ENV GO_ECOSYSTEM_SERVICE_ID=$SERVICE_ID 117 | 118 | ENV GO_ECOSYSTEM_BINARY_DIR=$BINARY_DIR 119 | 120 | ENV GO_ECOSYSTEM_VULNDB_DIR=$VULNDB_DIR 121 | 122 | CMD ["./worker"] 123 | -------------------------------------------------------------------------------- /cmd/worker/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Command worker runs the go-metrics worker server. 6 | package main 7 | 8 | import ( 9 | "context" 10 | "flag" 11 | "fmt" 12 | "net/http" 13 | "os" 14 | "os/signal" 15 | "syscall" 16 | "time" 17 | 18 | "golang.org/x/exp/slog" 19 | "golang.org/x/pkgsite-metrics/internal/config" 20 | "golang.org/x/pkgsite-metrics/internal/log" 21 | "golang.org/x/pkgsite-metrics/internal/worker" 22 | ) 23 | 24 | var ( 25 | workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally") 26 | devMode = flag.Bool("dev", false, "enable developer mode (reload templates on each page load, serve non-minified JS/CSS, etc.)") 27 | port = flag.String("port", config.GetEnv("PORT", "8080"), "port to listen to") 28 | dataset = flag.String("dataset", "", "dataset (overrides GO_ECOSYSTEM_BIGQUERY_DATASET env var); use 'disable' for no BQ") 29 | insecure = flag.Bool("insecure", false, "bypass sandbox in order to compare with old code") 30 | // flag used in call to safehtml/template.TrustedSourceFromFlag 31 | _ = flag.String("static", "static", "path to folder containing static files served") 32 | ) 33 | 34 | func main() { 35 | flag.Usage = func() { 36 | out := flag.CommandLine.Output() 37 | fmt.Fprintln(out, "usage:") 38 | fmt.Fprintln(out, "worker FLAGS") 39 | fmt.Fprintln(out, " run as a server, listening at the PORT env var") 40 | flag.PrintDefaults() 41 | } 42 | 43 | flag.Parse() 44 | ctx := context.Background() 45 | var h slog.Handler 46 | if config.OnCloudRun() || *devMode { 47 | h = log.NewGoogleCloudHandler() 48 | } else { 49 | h = log.NewLineHandler(os.Stderr) 50 | } 51 | slog.SetDefault(slog.New(h)) 52 | if err := runServer(ctx); err != nil { 53 | log.Error(ctx, "failed to start the server", err) 54 | // Give the log message a chance to be captured (?). 55 | time.Sleep(5 * time.Second) 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | func runServer(ctx context.Context) error { 61 | cfg, err := config.Init(ctx) 62 | if err != nil { 63 | return err 64 | } 65 | cfg.LocalQueueWorkers = *workers 66 | cfg.DevMode = *devMode 67 | if *dataset != "" { 68 | cfg.BigQueryDataset = *dataset 69 | } 70 | cfg.Insecure = *insecure 71 | cfg.Dump(os.Stdout) 72 | log.Infof(ctx, "config: project=%s, dataset=%s", cfg.ProjectID, cfg.BigQueryDataset) 73 | 74 | s, err := worker.NewServer(ctx, cfg) 75 | if err != nil { 76 | return err 77 | } 78 | go monitor(ctx, s) 79 | 80 | addr := ":" + *port 81 | log.Infof(ctx, "Listening on addr http://localhost%s", addr) 82 | return fmt.Errorf("listening: %v", http.ListenAndServe(addr, nil)) 83 | } 84 | 85 | // monitor measures details of server execution from 86 | // the moment is starts listening to the moment it 87 | // gets a SIGTERM signal. 88 | func monitor(ctx context.Context, s *worker.Server) { 89 | start := time.Now() 90 | signals := make(chan os.Signal, 1) 91 | signal.Notify(signals, syscall.SIGTERM) 92 | <-signals 93 | log.Infof(ctx, "server stopped listening after: %v\n%s", time.Since(start), s.Info()) 94 | } 95 | -------------------------------------------------------------------------------- /config.json.commented: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | # 5 | # This file is JSON with comments. 6 | # A comment is any line whose first non-whitespace character is #. 7 | # A sed script in the Makefile and in deploy/worker.yaml removes 8 | # the comments to produce valid JSON. 9 | # 10 | # This is a bundle config file for runsc, as specified by the 11 | # Open Container Initiative: see 12 | # https://github.com/opencontainers/runtime-spec/blob/main/config.md. 13 | # Most of this file is generated by "runsc spec"; see 14 | # https://gvisor.dev/docs/user_guide/quick_start/oci. 15 | # The few important tweaks are commented. 16 | { 17 | "ociVersion": "1.0.0", 18 | "process": { 19 | "user": { 20 | "uid": 0, 21 | "gid": 0 22 | }, 23 | "args": [ 24 | # This is the command that "runsc run" will execute in the sandbox. 25 | # See the internal/sandbox package. 26 | # runsc will pipe the stdout and stderr to its caller, 27 | # and will exit with the same return code. 28 | "/runner" 29 | ], 30 | "env": [ 31 | "PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 32 | "TERM=xterm" 33 | ], 34 | "cwd": "/", 35 | "capabilities": { 36 | "bounding": [ 37 | "CAP_AUDIT_WRITE", 38 | "CAP_KILL", 39 | "CAP_NET_BIND_SERVICE" 40 | ], 41 | "effective": [ 42 | "CAP_AUDIT_WRITE", 43 | "CAP_KILL", 44 | "CAP_NET_BIND_SERVICE" 45 | ], 46 | "inheritable": [ 47 | "CAP_AUDIT_WRITE", 48 | "CAP_KILL", 49 | "CAP_NET_BIND_SERVICE" 50 | ], 51 | "permitted": [ 52 | "CAP_AUDIT_WRITE", 53 | "CAP_KILL", 54 | "CAP_NET_BIND_SERVICE" 55 | ] 56 | }, 57 | "rlimits": [ 58 | { 59 | "type": "RLIMIT_NOFILE", 60 | "hard": 1048576, 61 | "soft": 1048576 62 | } 63 | ] 64 | }, 65 | "root": { 66 | "path": "rootfs", 67 | # The filesystem must be writeable so 68 | # the go command can write to its caches. 69 | "readonly": false 70 | }, 71 | "hostname": "runsc", 72 | "mounts": [ 73 | { 74 | "destination": "/proc", 75 | "type": "proc", 76 | "source": "proc" 77 | }, 78 | { 79 | "destination": "/dev", 80 | "type": "tmpfs", 81 | "source": "tmpfs" 82 | }, 83 | { 84 | "destination": "/sys", 85 | "type": "sysfs", 86 | "source": "sysfs", 87 | "options": [ 88 | "nosuid", 89 | "noexec", 90 | "nodev", 91 | "ro" 92 | ] 93 | }, 94 | # Bind mounts. These let us map directories inside the sandbox 95 | # (the destination) to directories outside (the source). 96 | # If the source doesn't exist, you'll get the (obscure) error 97 | # "cannot read client sync file". 98 | # If the destination already exists, that's not an error, but the 99 | # files in that directory will be hidden to code running inside the 100 | # sandbox. 101 | { 102 | # Mount /app/binaries inside the sandbox to 103 | # the same directory outside. 104 | "destination": "/app/binaries", 105 | "type": "none", 106 | "source": "/app/binaries", 107 | "options": ["bind"] 108 | }, 109 | { 110 | # Mount /app/go-vulndb inside the sandbox to 111 | # the same directory outside. 112 | "destination": "/app/go-vulndb", 113 | "type": "none", 114 | "source": "/app/go-vulndb", 115 | "options": ["bind"] 116 | }, 117 | { 118 | # Mount /tmp/modules inside the sandbox to 119 | # the same directory outside. 120 | "destination": "/tmp/modules", 121 | "type": "none", 122 | "source": "/tmp/modules", 123 | "options": ["bind"] 124 | } 125 | ], 126 | "linux": { 127 | "namespaces": [ 128 | { 129 | "type": "pid" 130 | }, 131 | { 132 | "type": "network" 133 | }, 134 | { 135 | "type": "ipc" 136 | }, 137 | { 138 | "type": "uts" 139 | }, 140 | { 141 | "type": "mount" 142 | } 143 | ] 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /deploy/worker.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | # This is a Cloud Build config file for the go-ecosystem worker. 6 | # Invoke locally from the command line using devtools/deploy.sh. 7 | # It can also be configured to run from a trigger, by supplying the _ENV 8 | # substitution. 9 | 10 | substitutions: 11 | _ENV: '' 12 | _BQ_DATASET: '' 13 | 14 | steps: 15 | - id: Lock 16 | name: golang:1.23.0 17 | entrypoint: bash 18 | args: 19 | - -ec 20 | - | 21 | if [[ "$COMMIT_SHA" = '' ]]; then 22 | echo "no COMMIT_SHA, not locking" 23 | exit 0 24 | fi 25 | go run golang.org/x/website/cmd/locktrigger@latest \ 26 | -project $PROJECT_ID -build $BUILD_ID -repo https://go.googlesource.com/pkgsite-metrics 27 | 28 | - id: Test 29 | # Run tests. Do this early, to avoid wasting time if they fail. 30 | name: golang:1.23.0 31 | entrypoint: bash 32 | args: 33 | - -ec 34 | - go test ./... 35 | 36 | - id: Prepare 37 | name: gcr.io/cloud-builders/gcloud 38 | entrypoint: bash 39 | args: 40 | - -ec 41 | - | 42 | # Determine the image name and save for later steps. 43 | if [[ "$SHORT_SHA" = '' ]]; then 44 | echo >&2 "missing SHORT_SHA; use --substitutions on command line" 45 | exit 1 46 | fi 47 | if [[ "$_ENV" = '' ]]; then 48 | echo >&2 "missing _ENV; use --substitutions on command line" 49 | exit 1 50 | fi 51 | if [[ "$_BQ_DATASET" = '' ]]; then 52 | echo >&2 "missing _BQ_DATASET; use --substitutions on command line" 53 | exit 1 54 | fi 55 | 56 | tag=$(date +%Y%m%dt%H%M%S)-$SHORT_SHA 57 | image=gcr.io/$PROJECT_ID/${_ENV}-ecosystem-worker:$tag 58 | echo "image is $image" 59 | echo $image > /workspace/image.txt 60 | 61 | # Convert the commented config.json file to valid json. 62 | sed '/^[ \t]*#/d' config.json.commented > /workspace/config.json 63 | 64 | # Download the vuln DB from its bucket to a local directory, and remember 65 | # its last-modified time in a file. 66 | gsutil -m -q cp -r gs://go-vulndb /workspace 67 | gsutil stat gs://go-vulndb/index.json | \ 68 | awk '$$1 == "Update" { for (i = 4; i <= NF; i++) printf("%s ", $$i); printf("\n"); }' \ 69 | > /workspace/go-vulndb/LAST_MODIFIED 70 | # Download a tarball of a docker Go image. 71 | gsutil cp gs://go-ecosystem/go-image.tar.gz /workspace 72 | 73 | - id: Build 74 | # Build the docker image. 75 | # 76 | # The files we put in /workspace in the previous step need to be 77 | # in the repo root so they get uploaded to the Docker daemon. 78 | # However it turns out that /workspace is in fact the same directory, 79 | # so no copying is necessary. 80 | name: gcr.io/cloud-builders/docker 81 | entrypoint: bash 82 | args: 83 | - -ec 84 | - | 85 | image=$(cat /workspace/image.txt) 86 | docker build -t $image -f cmd/worker/Dockerfile . \ 87 | --build-arg DOCKER_IMAGE=$image \ 88 | --build-arg BQ_DATASET=${_BQ_DATASET} \ 89 | --build-arg SERVICE_ID=${_ENV}-ecosystem-worker 90 | docker push $image 91 | 92 | - id: Deploy 93 | name: gcr.io/cloud-builders/gcloud 94 | entrypoint: bash 95 | args: 96 | - -ec 97 | - | 98 | image=$(cat /workspace/image.txt) 99 | service=${_ENV}-ecosystem-worker 100 | args="--project $PROJECT_ID --region us-central1" 101 | gcloud beta run deploy $args $service --image $image --execution-environment=gen2 102 | # If there was a rollback, `gcloud run deploy` will create a revision but 103 | # not point traffic to it. The following command ensures that the new revision 104 | # will get traffic. 105 | latestTraffic=$(gcloud run services $args describe $service \ 106 | --format='value(status.traffic.latestRevision)') 107 | if [[ $latestTraffic != True ]]; then 108 | gcloud run services $args update-traffic $service --to-latest 109 | fi 110 | -------------------------------------------------------------------------------- /devtools/bqerrors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Go Authors. All rights reserved. 4 | # Use of this source code is governed by a BSD-style 5 | # license that can be found in the LICENSE file. 6 | 7 | # List counts of errors by date in BigQuery tables. 8 | 9 | set -e 10 | 11 | source devtools/lib.sh || { echo "Are you at repo root?"; exit 1; } 12 | 13 | usage() { 14 | die "usage: $0 DATASET" 15 | } 16 | 17 | bq_error_query() { 18 | local -r table=$1 19 | local q=" 20 | select date(created_at) as date, error_category, count(*) as count 21 | from $table 22 | group by 1, 2 23 | order by 1 desc" 24 | bq query $q 25 | } 26 | 27 | main() { 28 | local dataset=$1 29 | if [[ $dataset == '' ]]; then 30 | usage 31 | fi 32 | local -r project=$(tfvar prod_project) 33 | if [[ $project = '' ]]; then 34 | die "missing TF_VAR_prod_project" 35 | fi 36 | 37 | bq_error_query $project.$dataset.govulncheck 38 | } 39 | 40 | 41 | main "$@" 42 | -------------------------------------------------------------------------------- /devtools/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2022 The Go Authors. All rights reserved. 4 | # Use of this source code is governed by a BSD-style 5 | # license that can be found in the LICENSE file. 6 | 7 | # Deploy the go-ecosystem worker to Cloud Run, using Cloud Build. 8 | 9 | set -e 10 | 11 | source devtools/lib.sh || { echo "Are you at repo root?"; exit 1; } 12 | 13 | usage() { 14 | die "usage: $0 [-n] (dev | prod) BIGQUERY_DATASET" 15 | } 16 | 17 | # Report whether the current repo's workspace has no uncommitted files. 18 | clean_workspace() { 19 | [[ $(git status --porcelain) == '' ]] 20 | } 21 | 22 | main() { 23 | local prefix= 24 | if [[ $1 = '-n' ]]; then 25 | prefix='echo dryrun: ' 26 | shift 27 | fi 28 | 29 | local env=$1 30 | 31 | case $env in 32 | dev|prod);; 33 | *) usage;; 34 | esac 35 | 36 | local dataset=$2 37 | if [[ $dataset = '' ]]; then 38 | usage 39 | fi 40 | 41 | if which grants > /dev/null; then 42 | local allowed=false 43 | while read _ _ ok _; do 44 | if [[ $ok = OK ]]; then 45 | allowed=true 46 | fi 47 | done < <(grants check $GO_ECOSYSTEM_DEPLOY_GROUPS) 48 | if ! $allowed; then 49 | die "You need a grant for one of: $GO_ECOSYSTEM_DEPLOY_GROUPS" 50 | fi 51 | fi 52 | 53 | local -r project=$(tfvar ${env}_project) 54 | if [[ $project = '' ]]; then 55 | die "no ${env}_project in terraform.tfvars" 56 | fi 57 | local -r commit=$(git rev-parse --short HEAD) 58 | local unclean 59 | if ! clean_workspace; then 60 | unclean="-unclean" 61 | fi 62 | 63 | $prefix gcloud builds submit \ 64 | --project $project \ 65 | --config deploy/worker.yaml \ 66 | --substitutions SHORT_SHA=${commit}${unclean},_ENV=$env,_BQ_DATASET=$dataset 67 | } 68 | 69 | main "$@" 70 | -------------------------------------------------------------------------------- /devtools/lib.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | # Library of useful bash functions and variables. 6 | 7 | RED=; GREEN=; YELLOW=; NORMAL=; 8 | MAXWIDTH=0 9 | 10 | if tput setaf 1 >& /dev/null; then 11 | RED=$(tput setaf 1) 12 | GREEN=$(tput setaf 2) 13 | YELLOW=$(tput setaf 3) 14 | NORMAL=$(tput sgr0) 15 | MAXWIDTH=$(( $(tput cols) - 2 )) 16 | fi 17 | 18 | EXIT_CODE=0 19 | 20 | info() { echo -e "${GREEN}$*${NORMAL}" 1>&2; } 21 | warn() { echo -e "${YELLOW}$*${NORMAL}" 1>&2; } 22 | err() { echo -e "${RED}$*${NORMAL}" 1>&2; EXIT_CODE=1; } 23 | 24 | die() { 25 | err "$@" 26 | exit 1 27 | } 28 | 29 | dryrun=false 30 | 31 | # runcmd prints an info log describing the command that is about to be run, and 32 | # then runs it. It sets EXIT_CODE to non-zero if the command fails, but does not exit 33 | # the script. 34 | runcmd() { 35 | msg="$*" 36 | if $dryrun; then 37 | echo -e "${YELLOW}dryrun${GREEN}\$ $msg${NORMAL}" 38 | return 0 39 | fi 40 | # Truncate command logging for narrow terminals. 41 | # Account for the 2 characters of '$ '. 42 | if [[ $MAXWIDTH -gt 0 && ${#msg} -gt $MAXWIDTH ]]; then 43 | msg="${msg::$(( MAXWIDTH - 3 ))}..." 44 | fi 45 | 46 | echo -e "$*\n" 1>&2; 47 | "$@" || err "command failed" 48 | } 49 | 50 | # tfvar NAME returns the value of the terraform variable NAME. 51 | tfvar() { 52 | local v=TF_VAR_$1 53 | echo "${!v}" 54 | } 55 | 56 | worker_url() { 57 | local -r env=$1 58 | echo https://${env}-${GO_ECOSYSTEM_WORKER_URL_SUFFIX} 59 | } 60 | 61 | impersonation_service_account() { 62 | local -r env=$1 63 | local -r project=$(tfvar ${env}_project) 64 | case $env in 65 | prod|dev) echo impersonate@${project}.iam.gserviceaccount.com;; 66 | *) die "usage: $0 (dev | prod) ...";; 67 | esac 68 | } 69 | 70 | impersonation_token() { 71 | local -r env=$1 72 | gcloud --impersonate-service-account "$(impersonation_service_account "$env")" \ 73 | auth print-identity-token \ 74 | --include-email 75 | } 76 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module golang.org/x/pkgsite-metrics 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | cloud.google.com/go v0.110.7 7 | cloud.google.com/go/bigquery v1.54.0 8 | cloud.google.com/go/cloudtasks v1.12.1 9 | cloud.google.com/go/errorreporting v0.3.0 10 | cloud.google.com/go/firestore v1.12.0 11 | cloud.google.com/go/logging v1.8.1 12 | cloud.google.com/go/secretmanager v1.11.1 13 | cloud.google.com/go/storage v1.32.0 14 | github.com/GoogleCloudPlatform/opentelemetry-operations-go v1.0.0 15 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.26.0 16 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.0.0 17 | github.com/client9/misspell v0.3.4 18 | github.com/google/go-cmp v0.6.0 19 | github.com/google/safehtml v0.1.0 20 | github.com/jba/slog v0.0.0-20230225143746-b07e7e61ec27 21 | github.com/lib/pq v1.10.7 22 | go.opencensus.io v0.24.0 23 | go.opentelemetry.io/otel v1.11.2 24 | go.opentelemetry.io/otel/sdk v1.4.0 25 | golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 26 | golang.org/x/exp/event v0.0.0-20220218215828-6cf2b201936e 27 | golang.org/x/mod v0.25.0 28 | golang.org/x/net v0.41.0 29 | golang.org/x/oauth2 v0.30.0 30 | golang.org/x/sync v0.15.0 31 | golang.org/x/tools v0.34.0 32 | golang.org/x/vuln v1.1.4 33 | google.golang.org/api v0.132.0 34 | google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 35 | google.golang.org/grpc v1.56.2 36 | google.golang.org/protobuf v1.31.0 37 | honnef.co/go/tools v0.4.3 38 | mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 39 | ) 40 | 41 | require ( 42 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 43 | cloud.google.com/go/iam v1.1.0 // indirect 44 | cloud.google.com/go/longrunning v0.5.1 // indirect 45 | cloud.google.com/go/monitoring v1.15.1 // indirect 46 | cloud.google.com/go/trace v1.10.1 // indirect 47 | github.com/BurntSushi/toml v1.2.1 // indirect 48 | github.com/andybalholm/brotli v1.0.4 // indirect 49 | github.com/apache/arrow/go/v12 v12.0.0 // indirect 50 | github.com/apache/thrift v0.16.0 // indirect 51 | github.com/go-logr/logr v1.2.3 // indirect 52 | github.com/go-logr/stdr v1.2.2 // indirect 53 | github.com/goccy/go-json v0.9.11 // indirect 54 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 55 | github.com/golang/protobuf v1.5.3 // indirect 56 | github.com/golang/snappy v0.0.4 // indirect 57 | github.com/google/flatbuffers v2.0.8+incompatible // indirect 58 | github.com/google/s2a-go v0.1.4 // indirect 59 | github.com/google/uuid v1.3.0 // indirect 60 | github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect 61 | github.com/googleapis/gax-go/v2 v2.12.0 // indirect 62 | github.com/klauspost/asmfmt v1.3.2 // indirect 63 | github.com/klauspost/compress v1.15.9 // indirect 64 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 65 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect 66 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect 67 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 68 | github.com/zeebo/xxh3 v1.0.2 // indirect 69 | go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect 70 | go.opentelemetry.io/otel/metric v0.27.0 // indirect 71 | go.opentelemetry.io/otel/sdk/export/metric v0.26.0 // indirect 72 | go.opentelemetry.io/otel/sdk/metric v0.26.0 // indirect 73 | go.opentelemetry.io/otel/trace v1.11.2 // indirect 74 | golang.org/x/crypto v0.39.0 // indirect 75 | golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect 76 | golang.org/x/sys v0.33.0 // indirect 77 | golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect 78 | golang.org/x/text v0.26.0 // indirect 79 | golang.org/x/time v0.12.0 // indirect 80 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect 81 | google.golang.org/appengine v1.6.7 // indirect 82 | google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect 83 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect 84 | ) 85 | -------------------------------------------------------------------------------- /internal/analysis/analysis_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package analysis 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/google/go-cmp/cmp" 11 | ) 12 | 13 | func TestJSONTreeToDiagnostics(t *testing.T) { 14 | in := JSONTree{ 15 | "pkg1": { 16 | "a": { 17 | Diagnostics: []JSONDiagnostic{ 18 | {Category: "c1", Posn: "pos1", Message: "m1"}, 19 | {Category: "c2", Posn: "pos2", Message: "m2"}, 20 | }, 21 | }, 22 | "b": { 23 | Diagnostics: []JSONDiagnostic{{Category: "c3", Posn: "pos3", Message: "m3"}}, 24 | }, 25 | }, 26 | "pkg2": { 27 | "c": { 28 | Error: &jsonError{Err: "fail"}, 29 | }, 30 | }, 31 | } 32 | got := JSONTreeToDiagnostics(in) 33 | want := []*Diagnostic{ 34 | {PackageID: "pkg1", AnalyzerName: "a", Category: "c1", Position: "pos1", Message: "m1"}, 35 | {PackageID: "pkg1", AnalyzerName: "a", Category: "c2", Position: "pos2", Message: "m2"}, 36 | {PackageID: "pkg1", AnalyzerName: "b", Category: "c3", Position: "pos3", Message: "m3"}, 37 | {PackageID: "pkg2", AnalyzerName: "c", Error: "fail"}, 38 | } 39 | if diff := cmp.Diff(want, got); diff != "" { 40 | t.Errorf("mismatch (-want, +got)\n%s", diff) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /internal/bigquery/bigquery_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package bigquery 6 | 7 | import ( 8 | "context" 9 | "strings" 10 | "testing" 11 | 12 | bq "cloud.google.com/go/bigquery" 13 | test "golang.org/x/pkgsite-metrics/internal/testing" 14 | ) 15 | 16 | func TestIsNotFoundError(t *testing.T) { 17 | test.NeedsIntegrationEnv(t) 18 | 19 | client, err := bq.NewClient(context.Background(), "go-ecosystem") 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | dataset := client.Dataset("nope") 24 | _, err = dataset.Metadata(context.Background()) 25 | if !isNotFoundError(err) { 26 | t.Errorf("got false, want true for %v", err) 27 | } 28 | } 29 | 30 | func TestPartitionQuery(t *testing.T) { 31 | // Remove newlines and extra white 32 | clean := func(s string) string { 33 | return strings.Join(strings.Fields(s), " ") 34 | } 35 | 36 | for i, test := range []struct { 37 | q PartitionQuery 38 | want string 39 | }{ 40 | { 41 | PartitionQuery{ 42 | From: "full.table", 43 | Columns: "*", 44 | PartitionOn: "p", 45 | OrderBy: "o", 46 | }, 47 | `SELECT * EXCEPT (rownum) 48 | FROM ( SELECT *, ROW_NUMBER() OVER ( PARTITION BY p ORDER BY o ) AS rownum 49 | FROM full.table ) WHERE rownum = 1`, 50 | }, 51 | { 52 | PartitionQuery{ 53 | From: "full.table", 54 | Columns: "a, b, c", 55 | PartitionOn: "p", 56 | OrderBy: "o", 57 | Where: "name = 'foo' AND args = 'bar baz'", 58 | }, 59 | `SELECT * EXCEPT (rownum) 60 | FROM ( SELECT a, b, c, ROW_NUMBER() OVER ( PARTITION BY p ORDER BY o ) AS rownum 61 | FROM full.table 62 | WHERE name = 'foo' AND args = 'bar baz' 63 | ) WHERE rownum = 1`, 64 | }, 65 | } { 66 | got := clean(test.q.String()) 67 | want := clean(test.want) 68 | if got != want { 69 | t.Errorf("#%d:\ngot %s\nwant %s", i, got, want) 70 | } 71 | } 72 | } 73 | 74 | func TestSchemaString(t *testing.T) { 75 | type nest struct { 76 | N []byte 77 | M float64 78 | } 79 | 80 | type s struct { 81 | A string 82 | B int 83 | C []bool 84 | D nest 85 | } 86 | const want = "A,req:STRING;B,req:INTEGER;C,rep:BOOLEAN;D,req:(M,req:FLOAT;N,req:BYTES)" 87 | schema, err := InferSchema(s{}) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | got := SchemaString(schema) 92 | if got != want { 93 | t.Errorf("\ngot %q\nwant %q", got, want) 94 | } 95 | 96 | // The order of fields should not matter 97 | type p struct { 98 | A string 99 | D nest 100 | C []bool 101 | B int 102 | } 103 | 104 | schema, err = InferSchema(p{}) 105 | if err != nil { 106 | t.Fatal(err) 107 | } 108 | got = SchemaString(schema) 109 | if got != want { 110 | t.Errorf("\ngot %q\nwant %q", got, want) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /internal/buildbinary/bin.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package buildbinary 6 | 7 | import ( 8 | "fmt" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "time" 13 | 14 | "golang.org/x/pkgsite-metrics/internal/derrors" 15 | ) 16 | 17 | type BinaryInfo struct { 18 | BinaryPath string 19 | ImportPath string 20 | BuildTime time.Duration 21 | Error error 22 | } 23 | 24 | // FindAndBuildBinaries finds and builds all possible binaries from a given module. 25 | func FindAndBuildBinaries(modulePath string) (binaries []*BinaryInfo, err error) { 26 | defer derrors.Wrap(&err, "FindAndBuildBinaries") 27 | buildTargets, err := findBinaries(modulePath) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | for i, target := range buildTargets { 33 | path, buildTime, err := runBuild(modulePath, target, i) 34 | b := &BinaryInfo{ 35 | BinaryPath: path, 36 | ImportPath: target, 37 | BuildTime: buildTime, 38 | } 39 | if err != nil { 40 | b.Error = err 41 | } 42 | binaries = append(binaries, b) 43 | } 44 | return binaries, nil 45 | } 46 | 47 | // runBuild takes a given module and import path and attempts to build a binary 48 | func runBuild(modulePath, importPath string, i int) (binaryPath string, buildTime time.Duration, err error) { 49 | binName := fmt.Sprintf("bin%d", i) 50 | cmd := exec.Command("go", "build", "-C", modulePath, "-o", binName, importPath) 51 | start := time.Now() 52 | if err = cmd.Run(); err != nil { 53 | return "", 0, err 54 | } 55 | buildTime = time.Since(start) 56 | binaryPath = filepath.Join(modulePath, binName) 57 | return binaryPath, buildTime, nil 58 | } 59 | 60 | // findBinaries finds all packages that compile to binaries in a given directory 61 | // and returns a list of those package's import paths. 62 | func findBinaries(dir string) (buildTargets []string, err error) { 63 | // Running go list with the given arguments only prints the import paths of 64 | // packages with package "main", that is packages that could potentially 65 | // be built into binaries. 66 | cmd := exec.Command("go", "list", "-f", `{{ if eq .Name "main" }} {{ .ImportPath }} {{end}}`, "./...") 67 | cmd.Dir = dir 68 | out, err := cmd.Output() 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | return strings.Fields(string(out)), nil 74 | } 75 | -------------------------------------------------------------------------------- /internal/buildbinary/bin_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package buildbinary 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "github.com/google/go-cmp/cmp/cmpopts" 14 | ) 15 | 16 | const ( 17 | localTestData = "../testdata" 18 | ) 19 | 20 | func less(a, b string) bool { 21 | return a < b 22 | } 23 | 24 | func TestFindBinaries(t *testing.T) { 25 | tests := []struct { 26 | name string 27 | dir string 28 | want []string 29 | wantErr bool 30 | }{ 31 | { 32 | name: "local test", 33 | dir: filepath.Join(localTestData, "module"), 34 | want: []string{"golang.org/vuln"}, 35 | wantErr: false, 36 | }, 37 | { 38 | name: "multiple test", 39 | dir: filepath.Join(localTestData, "multipleBinModule"), 40 | want: []string{"example.com/test", "example.com/test/multipleBinModule", "example.com/test/p1"}, 41 | wantErr: false, 42 | }, 43 | { 44 | name: "error test", 45 | dir: "non-existing-module", 46 | wantErr: true, 47 | }, 48 | } 49 | for _, tt := range tests { 50 | t.Run(tt.name, func(t *testing.T) { 51 | got, err := findBinaries(tt.dir) 52 | if (err != nil) != tt.wantErr { 53 | t.Fatalf("got error=%v, wantErr=%v", err, tt.wantErr) 54 | } 55 | 56 | if diff := cmp.Diff(tt.want, got, cmpopts.SortSlices(less)); diff != "" { 57 | t.Errorf("mismatch (-want, +got):%s", diff) 58 | } 59 | }) 60 | } 61 | } 62 | 63 | func TestRunBuild(t *testing.T) { 64 | if testing.Short() { 65 | t.Skip("skipping test that uses internet in short mode") 66 | } 67 | 68 | tests := []struct { 69 | name string 70 | modulePath string 71 | importPath string 72 | want string 73 | wantErr bool 74 | }{ 75 | { 76 | name: "local test", 77 | modulePath: filepath.Join(localTestData, "module"), 78 | importPath: "golang.org/vuln", 79 | want: filepath.Join(localTestData, "module", "bin1"), 80 | }, 81 | { 82 | name: "multiple binaries", 83 | modulePath: filepath.Join(localTestData, "multipleBinModule"), 84 | importPath: "example.com/test/multipleBinModule", 85 | want: filepath.Join(localTestData, "multipleBinModule", "bin1"), 86 | }, 87 | } 88 | for _, tt := range tests { 89 | t.Run(tt.name, func(t *testing.T) { 90 | got, _, err := runBuild(tt.modulePath, tt.importPath, 1) 91 | defer os.Remove(got) 92 | if (err != nil) != tt.wantErr { 93 | t.Fatalf("got error=%v; wantErr=%v", err, tt.wantErr) 94 | } 95 | 96 | if diff := cmp.Diff(tt.want, got); diff != "" { 97 | t.Errorf("mismatch (-want, +got):%s", diff) 98 | } 99 | _, err = os.Stat(got) 100 | if err != nil && os.IsNotExist(err) { 101 | t.Errorf("did not produce the expected binary") 102 | } 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/buildtest/buildtest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package buildtest provides support for running "go build" 6 | // and similar build/installation commands in tests. 7 | package buildtest 8 | 9 | import ( 10 | "fmt" 11 | "os" 12 | "os/exec" 13 | "path/filepath" 14 | "runtime" 15 | "strings" 16 | "testing" 17 | ) 18 | 19 | var unsupportedGoosGoarch = map[string]bool{ 20 | "darwin/386": true, 21 | "darwin/arm": true, 22 | } 23 | 24 | // GoBuild runs "go build" on dir using the additional environment variables in 25 | // envVarVals, which should be an alternating list of variables and values. 26 | // It returns the path to the resulting binary. 27 | func GoBuild(t *testing.T, dir, tags string, envVarVals ...string) (binaryPath string) { 28 | switch runtime.GOOS { 29 | case "android", "js", "ios": 30 | t.Skipf("skipping on OS without 'go build' %s", runtime.GOOS) 31 | } 32 | 33 | if len(envVarVals)%2 != 0 { 34 | t.Fatal("last args should be alternating variables and values") 35 | } 36 | var env []string 37 | if len(envVarVals) > 0 { 38 | env = os.Environ() 39 | for i := 0; i < len(envVarVals); i += 2 { 40 | env = append(env, fmt.Sprintf("%s=%s", envVarVals[i], envVarVals[i+1])) 41 | } 42 | } 43 | 44 | gg := lookupEnv("GOOS", env, runtime.GOOS) + "/" + lookupEnv("GOARCH", env, runtime.GOARCH) 45 | if unsupportedGoosGoarch[gg] { 46 | t.Skipf("skipping unsupported GOOS/GOARCH pair %s", gg) 47 | } 48 | 49 | abs, err := filepath.Abs(dir) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | tmpDir := t.TempDir() 54 | binaryPath = filepath.Join(tmpDir, filepath.Base(abs)) 55 | var exeSuffix string 56 | if runtime.GOOS == "windows" { 57 | exeSuffix = ".exe" 58 | } 59 | // Make sure we use the same version of go that is running this test. 60 | goCommandPath := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix) 61 | if _, err := os.Stat(goCommandPath); err != nil { 62 | t.Fatal(err) 63 | } 64 | args := []string{"build", "-o", binaryPath + exeSuffix} 65 | if tags != "" { 66 | args = append(args, "-tags", tags) 67 | } 68 | cmd := exec.Command(goCommandPath, args...) 69 | cmd.Dir = dir 70 | cmd.Env = env 71 | cmd.Stdout = os.Stdout 72 | cmd.Stderr = os.Stderr 73 | if err := cmd.Run(); err != nil { 74 | t.Fatal(err) 75 | } 76 | return binaryPath + exeSuffix 77 | } 78 | 79 | // lookEnv looks for name in env, a list of "VAR=VALUE" strings. It returns 80 | // the value if name is found, and defaultValue if it is not. 81 | func lookupEnv(name string, env []string, defaultValue string) string { 82 | for _, vv := range env { 83 | i := strings.IndexByte(vv, '=') 84 | if i < 0 { 85 | // malformed env entry; just ignore it 86 | continue 87 | } 88 | if name == vv[:i] { 89 | return vv[i+1:] 90 | } 91 | } 92 | return defaultValue 93 | } 94 | 95 | // BuildGovulncheck builds the version of govulncheck specified in 96 | // the go.mod file of this repo into the tmpDir. If the installation 97 | // is successful, returns the full path to the binary. Otherwise, 98 | // returns the error. It uses the Go caches as defined by go env. 99 | func BuildGovulncheck(tmpDir string) (string, error) { 100 | cmd := exec.Command("go", "build", "-o", tmpDir, "golang.org/x/vuln/cmd/govulncheck") 101 | _, err := cmd.CombinedOutput() 102 | if err != nil { 103 | return "", err 104 | } 105 | return filepath.Join(tmpDir, "govulncheck"), nil 106 | } 107 | -------------------------------------------------------------------------------- /internal/buildtest/buildtest_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package buildtest 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestBuildGovulncheck(t *testing.T) { 12 | if _, err := BuildGovulncheck(t.TempDir()); err != nil { 13 | t.Fatal(err) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /internal/ejson2csv/ejson2csv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package ejson2csv 6 | 7 | import ( 8 | "encoding/csv" 9 | "encoding/json" 10 | "io" 11 | "strconv" 12 | ) 13 | 14 | func must(err error) { 15 | if err != nil { 16 | panic(err) 17 | } 18 | } 19 | 20 | // sliceOf converts string/int args into a 5-element slice of string, 21 | // reusing the input slice (pointer) s. 22 | func sliceOf(s *[]string, args ...any) []string { 23 | *s = (*s)[:0] 24 | for _, x := range args { 25 | switch x := x.(type) { 26 | case int: 27 | *s = append(*s, strconv.FormatInt(int64(x), 10)) 28 | case string: 29 | *s = append(*s, x) 30 | } 31 | } 32 | for len(*s) < 5 { 33 | *s = append(*s, "") 34 | } 35 | return *s 36 | } 37 | 38 | // Process converts JSON read from r into CSV written to w, filtered 39 | // according to the various flags. The default is to include only 40 | // diagnostic messages (all false), errors specifies errors, 41 | // others specifies all lines that are neither error nor diagnostics, 42 | // and all means all. One limits the output to the first line (error, 43 | // diagnostic, or neither) from each module in the JSON stream. 44 | func Process(r io.Reader, w io.Writer, errors, others, all, one bool) { 45 | var stuff any 46 | 47 | buf, err := io.ReadAll(r) 48 | must(err) 49 | 50 | err = json.Unmarshal(buf, &stuff) 51 | must(err) 52 | 53 | slice := stuff.([]any) 54 | 55 | out := csv.NewWriter(w) 56 | 57 | var line []string 58 | line = sliceOf(&line, "ModulePath", "mpIndex", "Message/error", "meIndex", "Position") 59 | out.Write(line) 60 | 61 | outer: 62 | for i, a := range slice { 63 | ma := a.(map[string]any) 64 | 65 | if s, ok := ma["Diagnostics"].([]any); ok { 66 | // sawDiagnostic indicates non-empty error/message 67 | // this should always be true here, but just in case, track it. 68 | sawDiagnostic := false 69 | 70 | for j, d := range s { 71 | md := d.(map[string]any) 72 | if m, ok := md["Error"].(string); ok && m != "" { 73 | // error messages print if errors or all. 74 | if errors || all { 75 | out.Write(sliceOf(&line, ma["ModulePath"], i, m, j, md["Position"])) 76 | if one { 77 | continue outer 78 | } 79 | } 80 | sawDiagnostic = true 81 | } 82 | if m, ok := md["Message"].(string); ok && m != "" { 83 | // diagnostic messages print if present and either all or not-errors-and-not-others 84 | if !errors && !others || all { 85 | out.Write(sliceOf(&line, ma["ModulePath"], i, m, j, md["Position"])) 86 | if one { 87 | continue outer 88 | } 89 | } 90 | sawDiagnostic = true 91 | } 92 | } 93 | if sawDiagnostic { 94 | continue outer 95 | } 96 | } 97 | // Here if no diagnostic message or error lines were printed. 98 | if others || all { 99 | out.Write(sliceOf(&line, ma["ModulePath"], i)) 100 | } 101 | } 102 | out.Flush() 103 | } 104 | -------------------------------------------------------------------------------- /internal/ejson2csv/ejson2csv_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package ejson2csv_test 6 | 7 | import ( 8 | "bytes" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | 13 | "golang.org/x/pkgsite-metrics/internal/ejson2csv" 14 | ) 15 | 16 | func must(err error) { 17 | if err != nil { 18 | panic(err) 19 | } 20 | } 21 | 22 | func run(pb *bytes.Buffer, errors, others, all, one bool) { 23 | in, err := os.Open(filepath.Join("testdata", "sample.json")) 24 | must(err) 25 | ejson2csv.Process(in, pb, errors, others, all, one) 26 | } 27 | 28 | var nl []byte = []byte{'\n'} 29 | 30 | func expect(t *testing.T, pb *bytes.Buffer, count int) { 31 | if got := bytes.Count(pb.Bytes(), nl); got != count { 32 | t.Errorf("Expected %d newlines, got %d", count, got) 33 | } 34 | } 35 | 36 | func TestEmpty(t *testing.T) { 37 | var b bytes.Buffer 38 | run(&b, false, false, false, false) 39 | expect(t, &b, 5) 40 | } 41 | 42 | func TestEmptyOne(t *testing.T) { 43 | var b bytes.Buffer 44 | run(&b, false, false, false, true) 45 | expect(t, &b, 3) 46 | } 47 | 48 | func TestError(t *testing.T) { 49 | var b bytes.Buffer 50 | run(&b, true, false, false, false) 51 | expect(t, &b, 37) 52 | } 53 | 54 | func TestErrorOne(t *testing.T) { 55 | var b bytes.Buffer 56 | run(&b, true, false, false, true) 57 | expect(t, &b, 4) 58 | } 59 | func TestOther(t *testing.T) { 60 | var b bytes.Buffer 61 | run(&b, false, true, false, false) 62 | expect(t, &b, 18) 63 | } 64 | 65 | func TestOtherOne(t *testing.T) { 66 | var b bytes.Buffer 67 | run(&b, false, true, false, true) 68 | expect(t, &b, 18) 69 | } 70 | 71 | func TestErrorOther(t *testing.T) { 72 | var b bytes.Buffer 73 | run(&b, true, true, false, false) 74 | expect(t, &b, 54) 75 | } 76 | 77 | func TestErrorOtherOne(t *testing.T) { 78 | var b bytes.Buffer 79 | run(&b, true, true, false, true) 80 | expect(t, &b, 21) 81 | } 82 | 83 | func TestAll(t *testing.T) { 84 | var b bytes.Buffer 85 | run(&b, false, false, true, false) 86 | expect(t, &b, 58) 87 | } 88 | 89 | func TestAllOne(t *testing.T) { 90 | var b bytes.Buffer 91 | run(&b, false, false, true, true) 92 | expect(t, &b, 23) 93 | } 94 | -------------------------------------------------------------------------------- /internal/fstore/fstore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package fstore provides general support for Firestore. 6 | // Its main feature is separate namespaces, to mimic separate 7 | // databases for different purposes (prod, dev, test, etc.). 8 | package fstore 9 | 10 | import ( 11 | "context" 12 | "errors" 13 | 14 | "cloud.google.com/go/firestore" 15 | "golang.org/x/pkgsite-metrics/internal/derrors" 16 | "google.golang.org/grpc/codes" 17 | "google.golang.org/grpc/status" 18 | ) 19 | 20 | const namespaceCollection = "Namespaces" 21 | 22 | // A Namespace is a top-level collection for partitioning a Firestore 23 | // database into separate segments. 24 | type Namespace struct { 25 | client *firestore.Client 26 | name string 27 | doc *firestore.DocumentRef 28 | } 29 | 30 | // OpenNamespace creates a new Firestore client whose collections will be located in the given namespace. 31 | func OpenNamespace(ctx context.Context, projectID, name string) (_ *Namespace, err error) { 32 | defer derrors.Wrap(&err, "OpenNamespace(%q, %q)", projectID, name) 33 | 34 | if name == "" { 35 | return nil, errors.New("empty namespace") 36 | } 37 | client, err := firestore.NewClient(ctx, projectID) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &Namespace{ 42 | client: client, 43 | name: name, 44 | doc: client.Collection(namespaceCollection).Doc(name), 45 | }, nil 46 | } 47 | 48 | // Name returns the Namespace's name. 49 | func (ns *Namespace) Name() string { return ns.name } 50 | 51 | // Client returns the underlying Firestore client. 52 | func (ns *Namespace) Client() *firestore.Client { return ns.client } 53 | 54 | // Close closes the underlying client. 55 | func (ns *Namespace) Close() error { return ns.client.Close() } 56 | 57 | // Collection returns a reference to the named collection in the namespace. 58 | func (ns *Namespace) Collection(name string) *firestore.CollectionRef { 59 | return ns.doc.Collection(name) 60 | } 61 | 62 | // Get gets the DocumentRef and decodes the result to a value of type T. 63 | func Get[T any](ctx context.Context, dr *firestore.DocumentRef) (_ *T, err error) { 64 | defer derrors.Wrap(&err, "fstore.Get(%q)", dr.Path) 65 | docsnap, err := dr.Get(ctx) 66 | if err != nil { 67 | return nil, convertError(err) 68 | } 69 | return Decode[T](docsnap) 70 | } 71 | 72 | // Set sets the DocumentRef to the value. 73 | func Set[T any](ctx context.Context, dr *firestore.DocumentRef, value *T) (err error) { 74 | defer derrors.Wrap(&err, "firestore.Set(%q)", dr.Path) 75 | _, err = dr.Set(ctx, value) 76 | return convertError(err) 77 | } 78 | 79 | // Decode decodes a DocumentSnapshot into a value of type T. 80 | func Decode[T any](ds *firestore.DocumentSnapshot) (*T, error) { 81 | var t T 82 | if err := ds.DataTo(&t); err != nil { 83 | return nil, convertError(err) 84 | } 85 | return &t, nil 86 | } 87 | 88 | // convertError converts err into one of this module's error kinds 89 | // if possible. 90 | func convertError(err error) error { 91 | serr, ok := status.FromError(err) 92 | if !ok { 93 | return err 94 | } 95 | switch serr.Code() { 96 | case codes.NotFound: 97 | return derrors.NotFound 98 | case codes.InvalidArgument: 99 | return derrors.InvalidArgument 100 | default: 101 | return err 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /internal/goenv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package internal contains mostly utility functions. 6 | package internal 7 | 8 | import ( 9 | "encoding/json" 10 | "os/exec" 11 | ) 12 | 13 | // GoEnv returns the key-value map of `go env`. 14 | func GoEnv() (map[string]string, error) { 15 | out, err := exec.Command("go", "env", "-json").Output() 16 | if err != nil { 17 | return nil, err 18 | } 19 | env := make(map[string]string) 20 | if err := json.Unmarshal(out, &env); err != nil { 21 | return nil, err 22 | } 23 | return env, nil 24 | } 25 | -------------------------------------------------------------------------------- /internal/goenv_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package internal 6 | 7 | import ( 8 | "testing" 9 | 10 | test "golang.org/x/pkgsite-metrics/internal/testing" 11 | ) 12 | 13 | func TestGoEnv(t *testing.T) { 14 | test.NeedsGoEnv(t) 15 | 16 | for _, key := range []string{"GOVERSION", "GOROOT", "GOPATH", "GOMODCACHE"} { 17 | if m, err := GoEnv(); m[key] == "" { 18 | t.Errorf("want something for go env %s; got nothing", key) 19 | } else if err != nil { 20 | t.Errorf("unexpected error for go env %s: %v", key, err) 21 | } 22 | } 23 | } 24 | 25 | func TestGoEnvNonVariable(t *testing.T) { 26 | test.NeedsGoEnv(t) 27 | 28 | key := "NOT_A_GO_ENV_VARIABLE" 29 | if m, err := GoEnv(); m[key] != "" { 30 | t.Errorf("expected nothing for go env %s; got %s", key, m[key]) 31 | } else if err != nil { 32 | t.Errorf("unexpected error for go env %s: %v", key, err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/govulncheck/govulncheck_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build unix 6 | 7 | package govulncheck 8 | 9 | import ( 10 | "os/exec" 11 | "syscall" 12 | ) 13 | 14 | func init() { 15 | getMemoryUsage = func(c *exec.Cmd) uint64 { 16 | return uint64(c.ProcessState.SysUsage().(*syscall.Rusage).Maxrss) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /internal/govulncheck/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package govulncheck 6 | 7 | import ( 8 | "golang.org/x/pkgsite-metrics/internal/govulncheckapi" 9 | "golang.org/x/pkgsite-metrics/internal/osv" 10 | ) 11 | 12 | // NewMetricsHandler returns a handler that returns all findings. 13 | // For use in the ecosystem metrics pipeline. 14 | func NewMetricsHandler() *MetricsHandler { 15 | return &MetricsHandler{ 16 | osvs: make(map[string]*osv.Entry), 17 | } 18 | } 19 | 20 | type MetricsHandler struct { 21 | findings []*govulncheckapi.Finding 22 | osvs map[string]*osv.Entry 23 | } 24 | 25 | func (h *MetricsHandler) Config(c *govulncheckapi.Config) error { 26 | return nil 27 | } 28 | 29 | func (h *MetricsHandler) Progress(p *govulncheckapi.Progress) error { 30 | return nil 31 | } 32 | 33 | func (h *MetricsHandler) SBOM(sbom *govulncheckapi.SBOM) error { 34 | return nil 35 | } 36 | 37 | func (h *MetricsHandler) OSV(e *osv.Entry) error { 38 | h.osvs[e.ID] = e 39 | return nil 40 | } 41 | 42 | func (h *MetricsHandler) Finding(finding *govulncheckapi.Finding) error { 43 | h.findings = append(h.findings, finding) 44 | return nil 45 | } 46 | 47 | func (h *MetricsHandler) Findings() []*govulncheckapi.Finding { 48 | return h.findings 49 | } 50 | 51 | func (h *MetricsHandler) OSVs() map[string]*osv.Entry { 52 | return h.osvs 53 | } 54 | -------------------------------------------------------------------------------- /internal/govulncheckapi/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // The govulncheckapi package is copied from x/vuln/internal/govulncheck 6 | // and matches the output structure of govulncheck when ran in -json mode. 7 | package govulncheckapi 8 | 9 | import ( 10 | "encoding/json" 11 | "io" 12 | 13 | "golang.org/x/pkgsite-metrics/internal/osv" 14 | ) 15 | 16 | // Handler handles messages to be presented in a vulnerability scan output 17 | // stream. 18 | type Handler interface { 19 | // Config communicates introductory message to the user. 20 | Config(config *Config) error 21 | 22 | // SBOM shows information about what govulncheck is scanning. 23 | SBOM(sbom *SBOM) error 24 | 25 | // Progress is called to display a progress message. 26 | Progress(progress *Progress) error 27 | 28 | // OSV is invoked for each osv Entry in the stream. 29 | OSV(entry *osv.Entry) error 30 | 31 | // Finding is called for each vulnerability finding in the stream. 32 | Finding(finding *Finding) error 33 | } 34 | 35 | // HandleJSON reads the json from the supplied stream and hands the decoded 36 | // output to the handler. 37 | func HandleJSON(from io.Reader, to Handler) error { 38 | dec := json.NewDecoder(from) 39 | for dec.More() { 40 | msg := Message{} 41 | // decode the next message in the stream 42 | if err := dec.Decode(&msg); err != nil { 43 | return err 44 | } 45 | // dispatch the message 46 | var err error 47 | if msg.Config != nil { 48 | err = to.Config(msg.Config) 49 | } 50 | if msg.Progress != nil { 51 | err = to.Progress(msg.Progress) 52 | } 53 | if msg.OSV != nil { 54 | err = to.OSV(msg.OSV) 55 | } 56 | if msg.Finding != nil { 57 | err = to.Finding(msg.Finding) 58 | } 59 | if err != nil { 60 | return err 61 | } 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /internal/jobs/firestore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jobs 6 | 7 | import ( 8 | "context" 9 | "time" 10 | 11 | "cloud.google.com/go/firestore" 12 | "golang.org/x/pkgsite-metrics/internal/derrors" 13 | "golang.org/x/pkgsite-metrics/internal/fstore" 14 | "google.golang.org/api/iterator" 15 | ) 16 | 17 | const jobCollection = "Jobs" 18 | 19 | type DB struct { 20 | ns *fstore.Namespace 21 | } 22 | 23 | // NewDB creates a new database client for jobs. 24 | func NewDB(ctx context.Context, projectID, namespace string) (_ *DB, err error) { 25 | ns, err := fstore.OpenNamespace(ctx, projectID, namespace) 26 | if err != nil { 27 | return nil, err 28 | } 29 | return &DB{ns}, nil 30 | } 31 | 32 | // CreateJob creates a new job. It returns an error if a job with the same ID already exists. 33 | func (d *DB) CreateJob(ctx context.Context, j *Job) (err error) { 34 | id := j.ID() 35 | defer derrors.Wrap(&err, "job.DB.CreateJob(%s)", id) 36 | _, err = d.jobRef(id).Create(ctx, j) 37 | return err 38 | } 39 | 40 | // DeleteJob deletes the job with the given ID. It does not return an error if the job doesn't exist. 41 | func (d *DB) DeleteJob(ctx context.Context, id string) (err error) { 42 | defer derrors.Wrap(&err, "job.DB.DeleteJob(%s)", id) 43 | _, err = d.jobRef(id).Delete(ctx) 44 | return err 45 | } 46 | 47 | // GetJob retrieves the job with the given ID. It returns an error if the job does not exist. 48 | func (d *DB) GetJob(ctx context.Context, id string) (_ *Job, err error) { 49 | defer derrors.Wrap(&err, "job.DB.GetJob(%s)", id) 50 | return fstore.Get[Job](ctx, d.jobRef(id)) 51 | } 52 | 53 | // UpdateJob gets the job with the given ID, which must exist, then calls f on 54 | // it, then writes it back to the database. These actions occur atomically. 55 | // If f returns an error, that error is returned and no update occurs. 56 | func (d *DB) UpdateJob(ctx context.Context, id string, f func(*Job) error) (err error) { 57 | defer derrors.Wrap(&err, "job.DB.UpdateJob(%s)", id) 58 | return d.ns.Client().RunTransaction(ctx, func(ctx context.Context, tx *firestore.Transaction) error { 59 | docref := d.jobRef(id) 60 | docsnap, err := tx.Get(docref) 61 | if err != nil { 62 | return err 63 | } 64 | j, err := fstore.Decode[Job](docsnap) 65 | if err != nil { 66 | return err 67 | } 68 | if err := f(j); err != nil { 69 | return err 70 | } 71 | return tx.Set(docref, j) 72 | }, 73 | firestore.MaxAttempts(firestore.DefaultTransactionMaxAttempts*5)) 74 | } 75 | 76 | // Increment value named name by n. 77 | func (d *DB) Increment(ctx context.Context, id, name string, n int) (err error) { 78 | defer derrors.Wrap(&err, "job.DB.Increment(%s)", id) 79 | docref := d.jobRef(id) 80 | _, err = docref.Update(ctx, []firestore.Update{ 81 | {Path: name, Value: firestore.Increment(n)}, // name will incremented by n 82 | }) 83 | return err 84 | } 85 | 86 | // ListJobs calls f on each job in the DB, most recently started first. 87 | // f is also passed the time that the job was last updated. 88 | // If f returns a non-nil error, the iteration stops and returns that error. 89 | func (d *DB) ListJobs(ctx context.Context, f func(_ *Job, lastUpdate time.Time) error) (err error) { 90 | defer derrors.Wrap(&err, "job.DB.ListJobs()") 91 | 92 | q := d.ns.Collection(jobCollection).OrderBy("StartedAt", firestore.Desc) 93 | iter := q.Documents(ctx) 94 | defer iter.Stop() 95 | for { 96 | docsnap, err := iter.Next() 97 | if err == iterator.Done { 98 | break 99 | } 100 | if err != nil { 101 | return err 102 | } 103 | job, err := fstore.Decode[Job](docsnap) 104 | if err != nil { 105 | return err 106 | } 107 | if err := f(job, docsnap.UpdateTime); err != nil { 108 | return err 109 | } 110 | } 111 | return nil 112 | } 113 | 114 | // jobRef returns the DocumentRef for a job with the given ID. 115 | func (d *DB) jobRef(id string) *firestore.DocumentRef { 116 | return d.ns.Collection(jobCollection).Doc(id) 117 | } 118 | -------------------------------------------------------------------------------- /internal/jobs/firestore_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jobs 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "testing" 11 | "time" 12 | 13 | "github.com/google/go-cmp/cmp" 14 | test "golang.org/x/pkgsite-metrics/internal/testing" 15 | ) 16 | 17 | var project = flag.String("project", "", "GCP project for Firestore") 18 | 19 | func TestDB(t *testing.T) { 20 | test.NeedsIntegrationEnv(t) 21 | if *project == "" { 22 | t.Skip("missing -project") 23 | } 24 | ctx := context.Background() 25 | db, err := NewDB(ctx, *project, "testing") 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | must := func(err error) { 31 | t.Helper() 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | } 36 | 37 | tm := time.Date(2001, 02, 03, 4, 5, 6, 0, time.UTC) 38 | job := NewJob("user", tm, "analysis/enqueue?min=10", "bin", "", "no args") 39 | 40 | // Make sure the job doesn't exist. Delete doesn't fail 41 | // in that case. 42 | must(db.DeleteJob(ctx, job.ID())) 43 | 44 | // Create a new job. 45 | must(db.CreateJob(ctx, job)) 46 | 47 | // Get it and make sure it's the same. 48 | got, err := db.GetJob(ctx, job.ID()) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | if !cmp.Equal(got, job) { 53 | t.Errorf("got\n%+v\nwant\n%+v", got, job) 54 | } 55 | 56 | // Update it. 57 | must(db.UpdateJob(ctx, job.ID(), func(j *Job) error { 58 | j.NumStarted++ 59 | j.NumSucceeded++ 60 | return nil 61 | })) 62 | 63 | job.NumStarted = 1 64 | job.NumSucceeded = 1 65 | got, err = db.GetJob(ctx, job.ID()) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | if !cmp.Equal(got, job) { 70 | t.Errorf("got\n%+v\nwant\n%+v", got, job) 71 | } 72 | 73 | // Create another job, then list both. 74 | job2 := NewJob("user2", tm.Add(24*time.Hour), "url2", "bin", "", "xxx") 75 | must(db.DeleteJob(ctx, job2.ID())) 76 | must(db.CreateJob(ctx, job2)) 77 | 78 | var got2 []*Job 79 | must(db.ListJobs(ctx, func(j *Job, _ time.Time) error { 80 | got2 = append(got2, j) 81 | return nil 82 | })) 83 | // Jobs listed in reverse start-time order. 84 | want2 := []*Job{job2, job} 85 | if diff := cmp.Diff(want2, got2); diff != "" { 86 | t.Errorf("mismatch (-want, +got)\n%s", diff) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /internal/jobs/job.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package jobs supports jobs, which are collections of enqueued tasks. 6 | package jobs 7 | 8 | import ( 9 | "time" 10 | ) 11 | 12 | // A Job is a set of related scan tasks enqueued at the same time. 13 | type Job struct { 14 | User string 15 | StartedAt time.Time 16 | URL string // The URL that initiated the job. 17 | Binary string // Name of binary. 18 | BinaryVersion string // Hex-encoded hash of binary. 19 | BinaryArgs string // The args to the binary. 20 | Canceled bool // The job was canceled. 21 | // Counts of tasks. 22 | NumEnqueued int // Written by enqueue endpoint. 23 | NumStarted int // Incremented at the start of a scan. 24 | NumSkipped int // Previously run, stored in BigQuery. 25 | NumFailed int // The HTTP request failed (status != 200) 26 | NumErrored int // The HTTP request succeeded, but the scan resulted in an error. 27 | NumSucceeded int 28 | } 29 | 30 | // NewJob creates a new Job. 31 | func NewJob(user string, start time.Time, url, binaryName, binaryVersion, binaryArgs string) *Job { 32 | return &Job{ 33 | User: user, 34 | StartedAt: start, 35 | URL: url, 36 | Binary: binaryName, 37 | BinaryVersion: binaryVersion, 38 | BinaryArgs: binaryArgs, 39 | } 40 | } 41 | 42 | const startTimeFormat = "060102-150405" // YYMMDD-HHMMSS, UTC 43 | 44 | // ID returns a unique identifier for a job which can serve as a database key. 45 | func (j *Job) ID() string { 46 | return j.User + "-" + j.StartedAt.In(time.UTC).Format(startTimeFormat) 47 | } 48 | 49 | func (j *Job) NumFinished() int { 50 | return j.NumSkipped + j.NumFailed + j.NumErrored + j.NumSucceeded 51 | } 52 | -------------------------------------------------------------------------------- /internal/log/cloud_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package log 6 | 7 | import ( 8 | "os" 9 | "time" 10 | 11 | "golang.org/x/exp/slog" 12 | ) 13 | 14 | // NewGoogleCloudHandler returns a Handler that outputs JSON for the Google 15 | // Cloud logging service. 16 | // See https://cloud.google.com/logging/docs/agent/logging/configuration#special-fields 17 | // for treatment of special fields. 18 | func NewGoogleCloudHandler() slog.Handler { 19 | return slog.HandlerOptions{ReplaceAttr: gcpReplaceAttr, Level: slog.LevelDebug}. 20 | NewJSONHandler(os.Stderr) 21 | } 22 | 23 | func gcpReplaceAttr(groups []string, a slog.Attr) slog.Attr { 24 | switch a.Key { 25 | case "time": 26 | if a.Value.Kind() == slog.KindTime { 27 | a.Value = slog.StringValue(a.Value.Time().Format(time.RFC3339)) 28 | } 29 | case "msg": 30 | a.Key = "message" 31 | case "level": 32 | a.Key = "severity" 33 | case "traceID": 34 | a.Key = "logging.googleapis.com/trace" 35 | } 36 | return a 37 | } 38 | -------------------------------------------------------------------------------- /internal/log/line_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package log 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "fmt" 11 | "io" 12 | "sync" 13 | 14 | "github.com/jba/slog/withsupport" 15 | "golang.org/x/exp/slog" 16 | ) 17 | 18 | // LineHandler is a slog.Handler that writes log events one per line 19 | // in an easy-to-read format: 20 | // 21 | // time level message label1=value1 label2=value2 ... 22 | type LineHandler struct { 23 | mu sync.Mutex 24 | w io.Writer 25 | gora *withsupport.GroupOrAttrs 26 | } 27 | 28 | func NewLineHandler(w io.Writer) *LineHandler { 29 | return &LineHandler{w: w} 30 | } 31 | 32 | func (h *LineHandler) Enabled(ctx context.Context, level slog.Level) bool { 33 | return true 34 | } 35 | 36 | func (h *LineHandler) WithGroup(name string) slog.Handler { 37 | return &LineHandler{w: h.w, gora: h.gora.WithGroup(name)} 38 | } 39 | func (h *LineHandler) WithAttrs(as []slog.Attr) slog.Handler { 40 | return &LineHandler{w: h.w, gora: h.gora.WithAttrs(as)} 41 | } 42 | 43 | func (h *LineHandler) Handle(ctx context.Context, r slog.Record) error { 44 | var buf bytes.Buffer 45 | if !r.Time.IsZero() { 46 | fmt.Fprintf(&buf, "%s ", r.Time.Format("2006/01/02 15:04:05")) 47 | } 48 | fmt.Fprintf(&buf, "%-5s %s", r.Level, r.Message) 49 | 50 | prefix := "" 51 | for _, ga := range h.gora.Collect() { 52 | if ga.Group != "" { 53 | prefix += ga.Group + "." 54 | } else { 55 | for _, a := range ga.Attrs { 56 | writeAttr(&buf, prefix, a) 57 | } 58 | } 59 | } 60 | r.Attrs(func(a slog.Attr) { writeAttr(&buf, prefix, a) }) 61 | buf.WriteByte('\n') 62 | h.mu.Lock() 63 | defer h.mu.Unlock() 64 | _, err := h.w.Write(buf.Bytes()) 65 | return err 66 | } 67 | 68 | func writeAttr(w io.Writer, prefix string, a slog.Attr) { 69 | if a.Value.Kind() == slog.KindGroup { 70 | if a.Key != "" { 71 | prefix = a.Key + "." 72 | } 73 | for _, g := range a.Value.Group() { 74 | writeAttr(w, prefix, g) 75 | } 76 | } else if a.Key != "" { 77 | fmt.Fprintf(w, " %s%s=", prefix, a.Key) 78 | if a.Value.Kind() == slog.KindString { 79 | fmt.Fprintf(w, "%q", a.Value) 80 | } else { 81 | fmt.Fprintf(w, "%v", a.Value) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /internal/log/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package log implements logging. 6 | package log 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | 12 | "golang.org/x/exp/slog" 13 | ) 14 | 15 | type loggerKey struct{} 16 | 17 | // NewContext adds the logger to the context. 18 | func NewContext(ctx context.Context, l *slog.Logger) context.Context { 19 | return context.WithValue(ctx, loggerKey{}, l) 20 | } 21 | 22 | // FromContext retrieves a logger from the context. If there is none, 23 | // it returns the default logger. 24 | func FromContext(ctx context.Context) *slog.Logger { 25 | if l, ok := ctx.Value(loggerKey{}).(*slog.Logger); ok { 26 | return l 27 | } 28 | return slog.Default() 29 | } 30 | 31 | func Debug(ctx context.Context, msg string, args ...any) { FromContext(ctx).Debug(msg, args...) } 32 | func Info(ctx context.Context, msg string, args ...any) { FromContext(ctx).Info(msg, args...) } 33 | func Warn(ctx context.Context, msg string, args ...any) { FromContext(ctx).Warn(msg, args...) } 34 | func Error(ctx context.Context, msg string, err error, args ...any) { 35 | FromContext(ctx).Error(msg, err, args...) 36 | } 37 | 38 | func Logf(ctx context.Context, level slog.Level, format string, args ...any) { 39 | l := FromContext(ctx) 40 | if l.Enabled(ctx, level) { 41 | l.Log(ctx, level, fmt.Sprintf(format, args...)) 42 | } 43 | } 44 | 45 | func Debugf(ctx context.Context, format string, args ...any) { 46 | Logf(ctx, slog.LevelDebug, format, args...) 47 | } 48 | func Infof(ctx context.Context, format string, args ...any) { 49 | Logf(ctx, slog.LevelInfo, format, args...) 50 | } 51 | func Warnf(ctx context.Context, format string, args ...any) { 52 | Logf(ctx, slog.LevelWarn, format, args...) 53 | } 54 | 55 | func Errorf(ctx context.Context, err error, format string, args ...any) { 56 | level := slog.LevelError 57 | l := FromContext(ctx) 58 | if l.Enabled(ctx, level) { 59 | l.Log(ctx, level, fmt.Sprintf(format, args...), slog.ErrorKey, err) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /internal/modules/modules.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package modules assists in working with modules, e.g., 6 | // downloading a module via a Go proxy client. 7 | package modules 8 | 9 | import ( 10 | "archive/zip" 11 | "context" 12 | "fmt" 13 | "io" 14 | "os" 15 | "path/filepath" 16 | "strings" 17 | 18 | "golang.org/x/pkgsite-metrics/internal/derrors" 19 | "golang.org/x/pkgsite-metrics/internal/log" 20 | "golang.org/x/pkgsite-metrics/internal/proxy" 21 | ) 22 | 23 | // Download fetches module at version via proxyClient and unzips the module 24 | // into dir. 25 | func Download(ctx context.Context, module, version, dir string, proxyClient *proxy.Client) error { 26 | zipr, err := proxyClient.Zip(ctx, module, version) 27 | if err != nil { 28 | return fmt.Errorf("%v: %w", err, derrors.ProxyError) 29 | } 30 | log.Debugf(ctx, "writing module zip: %s@%s", module, version) 31 | stripPrefix := module + "@" + version + "/" 32 | if err := writeZip(zipr, dir, stripPrefix); err != nil { 33 | return fmt.Errorf("%v: %w", err, derrors.ScanModuleOSError) 34 | } 35 | return nil 36 | } 37 | 38 | func writeZip(r *zip.Reader, destination, stripPrefix string) error { 39 | for _, f := range r.File { 40 | name := strings.TrimPrefix(f.Name, stripPrefix) 41 | fpath := filepath.Join(destination, name) 42 | if !strings.HasPrefix(fpath, filepath.Clean(destination)+string(os.PathSeparator)) { 43 | return fmt.Errorf("%s is an illegal filepath", fpath) 44 | } 45 | 46 | // Do not include vendor directory. They currently contain only modules.txt, 47 | // not the dependencies. This makes package loading fail. Starting with go1.24, 48 | // there likely won't be any vendor directories at all. 49 | if vendored(name) { 50 | continue 51 | } 52 | 53 | if f.FileInfo().IsDir() { 54 | if err := os.MkdirAll(fpath, os.ModePerm); err != nil { 55 | return err 56 | } 57 | continue 58 | } 59 | if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { 60 | return err 61 | } 62 | outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) 63 | if err != nil { 64 | return err 65 | } 66 | rc, err := f.Open() 67 | if err != nil { 68 | return err 69 | } 70 | if _, err := io.Copy(outFile, rc); err != nil { 71 | return err 72 | } 73 | if err := outFile.Close(); err != nil { 74 | return err 75 | } 76 | if err := rc.Close(); err != nil { 77 | return err 78 | } 79 | } 80 | return nil 81 | } 82 | 83 | func vendored(path string) bool { 84 | return path == "vendor" || strings.HasPrefix(path, "vendor"+string(os.PathSeparator)) 85 | } 86 | -------------------------------------------------------------------------------- /internal/modules/modules_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package modules 6 | 7 | import ( 8 | "archive/zip" 9 | "bytes" 10 | "os" 11 | "path/filepath" 12 | "testing" 13 | ) 14 | 15 | func TestWriteZip(t *testing.T) { 16 | // Create an in-memory zipped test module. 17 | buf := new(bytes.Buffer) 18 | w := zip.NewWriter(buf) 19 | var files = []struct { 20 | Name, Body string 21 | }{ 22 | {filepath.Join("golang.org@v0.0.0", "README"), "This is a readme."}, 23 | {filepath.Join("golang.org@v0.0.0", "main"), "package main"}, 24 | {filepath.Join("golang.org@v0.0.0", "vendor", "modules.txt"), "# golang.org v1.1.1"}, 25 | {filepath.Join("golang.org@v0.0.0", "vendorius"), "This is some file with vendor in its name"}, 26 | } 27 | for _, file := range files { 28 | f, err := w.Create(file.Name) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | _, err = f.Write([]byte(file.Body)) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | } 37 | err := w.Close() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | // Create a zip.Reader for the module. 43 | br := bytes.NewReader(buf.Bytes()) 44 | r, err := zip.NewReader(br, int64(len(buf.Bytes()))) 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | 49 | tempDir := t.TempDir() 50 | if err := writeZip(r, tempDir, "golang.org@v0.0.0/"); err != nil { 51 | t.Error(err) 52 | } 53 | // make sure there are no vendor files 54 | fs, err := os.ReadDir(tempDir) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | for _, f := range fs { 59 | if f.IsDir() && f.Name() == "vendor" { 60 | t.Errorf("found unexpected vendor file or dir: %s", f.Name()) 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /internal/observe/observe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package observe provides metric and tracing support for Go servers. 6 | // It uses OpenTelemetry and the golang.org/x/exp/events package. 7 | package observe 8 | 9 | import ( 10 | "context" 11 | "net/http" 12 | 13 | "golang.org/x/exp/event" 14 | "golang.org/x/pkgsite-metrics/internal/derrors" 15 | 16 | mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" 17 | texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" 18 | gcppropagator "github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator" 19 | "go.opentelemetry.io/otel/propagation" 20 | sdktrace "go.opentelemetry.io/otel/sdk/trace" 21 | eotel "golang.org/x/exp/event/otel" 22 | ) 23 | 24 | // An Observer handles tracing and metrics exporting. 25 | type Observer struct { 26 | ctx context.Context 27 | tracerProvider *sdktrace.TracerProvider 28 | traceHandler *eotel.TraceHandler 29 | metricHandler *eotel.MetricHandler 30 | propagator propagation.TextMapPropagator 31 | } 32 | 33 | // NewObserver creates an Observer. 34 | // The context is used to flush traces in AfterRequest, so it should be longer-lived 35 | // than any request context. 36 | // (We don't want to use the request context because we still want traces even if 37 | // it is canceled or times out.) 38 | func NewObserver(ctx context.Context, projectID, serverName string) (_ *Observer, err error) { 39 | defer derrors.Wrap(&err, "NewObserver(%q, %q)", projectID, serverName) 40 | 41 | exporter, err := texporter.New(texporter.WithProjectID(projectID)) 42 | if err != nil { 43 | return nil, err 44 | } 45 | // Create exporter (collector embedded with the exporter). 46 | controller, err := mexporter.NewExportPipeline([]mexporter.Option{ 47 | mexporter.WithProjectID(projectID), 48 | }) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | tp := sdktrace.NewTracerProvider( 54 | // Enable tracing if there is no incoming request, or if the incoming 55 | // request is sampled. 56 | sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), 57 | sdktrace.WithBatcher(exporter)) 58 | return &Observer{ 59 | ctx: ctx, 60 | tracerProvider: tp, 61 | traceHandler: eotel.NewTraceHandler(tp.Tracer(serverName)), 62 | metricHandler: eotel.NewMetricHandler(controller.Meter(serverName)), 63 | // The propagator extracts incoming trace IDs so that we can connect our trace spans 64 | // to the incoming ones constructed by Cloud Run. 65 | propagator: propagation.NewCompositeTextMapPropagator( 66 | propagation.TraceContext{}, 67 | propagation.Baggage{}, 68 | gcppropagator.New()), 69 | }, nil 70 | } 71 | 72 | // Observe adds metrics and tracing to an http.Handler. 73 | func (o *Observer) Observe(h http.Handler) http.Handler { 74 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 75 | if o == nil { 76 | h.ServeHTTP(w, r.WithContext(r.Context())) 77 | return 78 | } 79 | exporter := event.NewExporter(o, nil) 80 | ctx := event.WithExporter(r.Context(), exporter) 81 | ctx = o.propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) 82 | defer o.tracerProvider.ForceFlush(o.ctx) 83 | h.ServeHTTP(w, r.WithContext(ctx)) 84 | }) 85 | } 86 | 87 | // Event implements event.Handler. 88 | func (o *Observer) Event(ctx context.Context, ev *event.Event) context.Context { 89 | ctx = o.traceHandler.Event(ctx, ev) 90 | return o.metricHandler.Event(ctx, ev) 91 | } 92 | -------------------------------------------------------------------------------- /internal/osv/review_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package osv 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | ) 11 | 12 | // ReviewStatus encodes the review status of a 13 | // report: unknown, reviewed, and unreviewed. 14 | type ReviewStatus int 15 | 16 | const ( 17 | ReviewStatusUnknown ReviewStatus = iota 18 | ReviewStatusUnreviewed 19 | ReviewStatusReviewed 20 | ) 21 | 22 | var statusStrs = []string{ 23 | ReviewStatusUnknown: "", 24 | ReviewStatusUnreviewed: "UNREVIEWED", 25 | ReviewStatusReviewed: "REVIEWED", 26 | } 27 | 28 | func (r ReviewStatus) String() string { 29 | if !r.IsValid() { 30 | return fmt.Sprintf("INVALID(%d)", r) 31 | } 32 | return statusStrs[r] 33 | } 34 | 35 | func ReviewStatusValues() []string { 36 | return statusStrs[1:] 37 | } 38 | 39 | func (r ReviewStatus) IsValid() bool { 40 | return int(r) >= 0 && int(r) < len(statusStrs) 41 | } 42 | 43 | func ToReviewStatus(s string) (ReviewStatus, bool) { 44 | for stat, str := range statusStrs { 45 | if s == str { 46 | return ReviewStatus(stat), true 47 | } 48 | } 49 | return 0, false 50 | } 51 | 52 | func (r ReviewStatus) MarshalJSON() ([]byte, error) { 53 | if !r.IsValid() { 54 | return nil, fmt.Errorf("MarshalJSON: unrecognized review status: %d", r) 55 | } 56 | return json.Marshal(r.String()) 57 | } 58 | 59 | func (r *ReviewStatus) UnmarshalJSON(b []byte) error { 60 | var s string 61 | if err := json.Unmarshal(b, &s); err != nil { 62 | return err 63 | } 64 | if rs, ok := ToReviewStatus(s); ok { 65 | *r = rs 66 | return nil 67 | } 68 | return fmt.Errorf("UnmarshalJSON: unrecognized review status: %s", s) 69 | } 70 | -------------------------------------------------------------------------------- /internal/pkgsitedb/db.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !plan9 6 | 7 | // Package pkgsitedb provides functionality for connecting to the pkgsite 8 | // database. 9 | package pkgsitedb 10 | 11 | import ( 12 | "context" 13 | "database/sql" 14 | "fmt" 15 | "regexp" 16 | 17 | _ "github.com/lib/pq" 18 | 19 | "golang.org/x/pkgsite-metrics/internal" 20 | "golang.org/x/pkgsite-metrics/internal/config" 21 | "golang.org/x/pkgsite-metrics/internal/derrors" 22 | "golang.org/x/pkgsite-metrics/internal/scan" 23 | ) 24 | 25 | // Open creates a connection to the pkgsite database. 26 | func Open(ctx context.Context, cfg *config.Config) (_ *sql.DB, err error) { 27 | defer derrors.Wrap(&err, "Open") 28 | password, err := internal.GetSecret(ctx, cfg.PkgsiteDBSecret) 29 | if err != nil { 30 | return nil, err 31 | } 32 | connString := fmt.Sprintf( 33 | "user='%s' password='%s' host='%s' port=%s dbname='%s' sslmode='disable'", 34 | cfg.PkgsiteDBUser, password, cfg.PkgsiteDBHost, cfg.PkgsiteDBPort, cfg.PkgsiteDBName) 35 | defer derrors.Wrap(&err, "openPkgsiteDB, connString=%q", redactPassword(connString)) 36 | db, err := sql.Open("postgres", connString) 37 | if err != nil { 38 | return nil, err 39 | } 40 | if err := db.PingContext(ctx); err != nil { 41 | return nil, err 42 | } 43 | return db, nil 44 | } 45 | 46 | var passwordRegexp = regexp.MustCompile(`password=\S+`) 47 | 48 | func redactPassword(dbinfo string) string { 49 | return passwordRegexp.ReplaceAllLiteralString(dbinfo, "password=REDACTED") 50 | } 51 | 52 | // ModuleSpecs retrieves all modules that contain packages that are 53 | // imported by minImportedByCount or more packages. 54 | // It looks for the information in the search_documents table of the given pkgsite DB. 55 | func ModuleSpecs(ctx context.Context, db *sql.DB, minImportedByCount int) (specs []scan.ModuleSpec, err error) { 56 | defer derrors.Wrap(&err, "moduleSpecsFromDB") 57 | query := ` 58 | SELECT module_path, version, max(imported_by_count) 59 | FROM search_documents 60 | GROUP BY module_path, version 61 | HAVING max(imported_by_count) >= $1 62 | ORDER by max(imported_by_count) desc` 63 | rows, err := db.QueryContext(ctx, query, minImportedByCount) 64 | if err != nil { 65 | return nil, err 66 | } 67 | defer rows.Close() 68 | for rows.Next() { 69 | var spec scan.ModuleSpec 70 | if err := rows.Scan(&spec.Path, &spec.Version, &spec.ImportedBy); err != nil { 71 | return nil, err 72 | } 73 | specs = append(specs, spec) 74 | } 75 | if err := rows.Err(); err != nil { 76 | return nil, err 77 | } 78 | return specs, nil 79 | } 80 | -------------------------------------------------------------------------------- /internal/pkgsitedb/db_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build plan9 6 | 7 | package pkgsitedb 8 | 9 | import ( 10 | "context" 11 | "database/sql" 12 | "errors" 13 | 14 | "golang.org/x/pkgsite-metrics/internal/config" 15 | "golang.org/x/pkgsite-metrics/internal/scan" 16 | ) 17 | 18 | var errDoesNotCompile = errors.New("github.com/lib/pq does not compile on plan9") 19 | 20 | func Open(ctx context.Context, cfg *config.Config) (_ *sql.DB, err error) { 21 | return nil, errDoesNotCompile 22 | } 23 | 24 | func ModuleSpecs(ctx context.Context, db *sql.DB, minImportedByCount int) (specs []scan.ModuleSpec, err error) { 25 | return nil, errDoesNotCompile 26 | } 27 | -------------------------------------------------------------------------------- /internal/pkgsitedb/db_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !plan9 6 | 7 | package pkgsitedb 8 | 9 | import ( 10 | // imported to register the postgres database driver 11 | "context" 12 | "database/sql" 13 | "flag" 14 | "fmt" 15 | "net/url" 16 | "strings" 17 | "testing" 18 | 19 | _ "github.com/lib/pq" 20 | ) 21 | 22 | // dbInfo is -db flag used to test against a a local database (host 127.0.0.1). 23 | var dbInfo = flag.String("db", "", 24 | "DB info for testing in the form 'name=NAME&port=PORT&user=USER&password=PW'") 25 | 26 | func TestModuleSpecs(t *testing.T) { 27 | if *dbInfo == "" { 28 | t.Skip("missing -db") 29 | } 30 | info := map[string]string{} 31 | for _, kv := range strings.Split(*dbInfo, "&") { 32 | k, v, ok := strings.Cut(kv, "=") 33 | if !ok { 34 | t.Fatalf("%q is not in the form 'key=value'", kv) 35 | } 36 | info[k] = v 37 | } 38 | 39 | const host = "127.0.0.1" 40 | 41 | ctx := context.Background() 42 | dbinfo := fmt.Sprintf("postgres://%s/%s?sslmode=disable&user=%s&password=%s&port=%s&timezone=UTC", 43 | host, info["name"], url.QueryEscape(info["user"]), url.QueryEscape(info["password"]), 44 | url.QueryEscape(info["port"])) 45 | db, err := sql.Open("postgres", dbinfo) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | defer db.Close() 50 | if err := db.PingContext(ctx); err != nil { 51 | t.Fatal(err) 52 | } 53 | got, err := ModuleSpecs(ctx, db, 1000) 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | t.Logf("got %d module specs from %s", len(got), info["name"]) 58 | if got, want := len(got), 100; got < want { 59 | t.Errorf("got %d results, expected at least %d", got, want) 60 | } 61 | for _, g := range got { 62 | fmt.Printf("%s %s\n", g.Path, g.Version) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /internal/proxy/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package proxy 6 | 7 | import ( 8 | "archive/zip" 9 | "sync" 10 | ) 11 | 12 | type modver struct { 13 | Path string 14 | Version string 15 | } 16 | 17 | // cache caches proxy info, mod and zip calls. 18 | type cache struct { 19 | mu sync.Mutex 20 | 21 | infoCache map[modver]*VersionInfo 22 | modCache map[modver][]byte 23 | 24 | // One-element zip cache, to avoid a double download. 25 | // See TestFetchAndUpdateStateCacheZip in internal/worker/fetch_test.go. 26 | zipKey modver 27 | zipReader *zip.Reader 28 | } 29 | 30 | func (c *cache) getInfo(modulePath, version string) *VersionInfo { 31 | if c == nil { 32 | return nil 33 | } 34 | c.mu.Lock() 35 | defer c.mu.Unlock() 36 | return c.infoCache[modver{Path: modulePath, Version: version}] 37 | } 38 | 39 | func (c *cache) putInfo(modulePath, version string, v *VersionInfo) { 40 | if c == nil { 41 | return 42 | } 43 | c.mu.Lock() 44 | defer c.mu.Unlock() 45 | if c.infoCache == nil { 46 | c.infoCache = map[modver]*VersionInfo{} 47 | } 48 | c.infoCache[modver{Path: modulePath, Version: version}] = v 49 | } 50 | 51 | func (c *cache) getMod(modulePath, version string) []byte { 52 | if c == nil { 53 | return nil 54 | } 55 | c.mu.Lock() 56 | defer c.mu.Unlock() 57 | return c.modCache[modver{Path: modulePath, Version: version}] 58 | } 59 | 60 | func (c *cache) putMod(modulePath, version string, b []byte) { 61 | if c == nil { 62 | return 63 | } 64 | c.mu.Lock() 65 | defer c.mu.Unlock() 66 | if c.modCache == nil { 67 | c.modCache = map[modver][]byte{} 68 | } 69 | c.modCache[modver{Path: modulePath, Version: version}] = b 70 | } 71 | 72 | func (c *cache) getZip(modulePath, version string) *zip.Reader { 73 | if c == nil { 74 | return nil 75 | } 76 | c.mu.Lock() 77 | defer c.mu.Unlock() 78 | if c.zipKey == (modver{Path: modulePath, Version: version}) { 79 | return c.zipReader 80 | } 81 | return nil 82 | } 83 | 84 | func (c *cache) putZip(modulePath, version string, r *zip.Reader) { 85 | if c == nil { 86 | return 87 | } 88 | c.mu.Lock() 89 | defer c.mu.Unlock() 90 | c.zipKey = modver{Path: modulePath, Version: version} 91 | c.zipReader = r 92 | } 93 | -------------------------------------------------------------------------------- /internal/proxy/proxytest/module.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package proxytest supports testing with the proxy. 6 | package proxytest 7 | 8 | import "fmt" 9 | 10 | // Module represents a module version used by the proxy server. 11 | type Module struct { 12 | ModulePath string 13 | Version string 14 | Files map[string]string 15 | NotCached bool // if true, behaves like it's uncached 16 | zip []byte 17 | } 18 | 19 | // ChangePath returns a copy of m with a different module path. 20 | func (m *Module) ChangePath(modulePath string) *Module { 21 | m2 := *m 22 | m2.ModulePath = modulePath 23 | return &m2 24 | } 25 | 26 | // ChangeVersion returns a copy of m with a different version. 27 | func (m *Module) ChangeVersion(version string) *Module { 28 | m2 := *m 29 | m2.Version = version 30 | return &m2 31 | } 32 | 33 | // AddFile returns a copy of m with an additional file. It 34 | // panics if the filename is already present. 35 | func (m *Module) AddFile(filename, contents string) *Module { 36 | return m.setFile(filename, &contents, false) 37 | } 38 | 39 | // DeleteFile returns a copy of m with filename removed. 40 | // It panics if filename is not present. 41 | func (m *Module) DeleteFile(filename string) *Module { 42 | return m.setFile(filename, nil, true) 43 | } 44 | 45 | // ReplaceFile returns a copy of m with different contents for filename. 46 | // It panics if filename is not present. 47 | func (m *Module) ReplaceFile(filename, contents string) *Module { 48 | return m.setFile(filename, &contents, true) 49 | } 50 | 51 | func (m *Module) setFile(filename string, contents *string, mustExist bool) *Module { 52 | _, ok := m.Files[filename] 53 | if mustExist && !ok { 54 | panic(fmt.Sprintf("%s@%s does not have a file named %s", m.ModulePath, m.Version, filename)) 55 | } 56 | if !mustExist && ok { 57 | panic(fmt.Sprintf("%s@%s already has a file named %s", m.ModulePath, m.Version, filename)) 58 | } 59 | m2 := *m 60 | if m.Files != nil { 61 | m2.Files = map[string]string{} 62 | for k, v := range m.Files { 63 | m2.Files[k] = v 64 | } 65 | } 66 | if contents == nil { 67 | delete(m2.Files, filename) 68 | } else { 69 | m2.Files[filename] = *contents 70 | } 71 | return &m2 72 | } 73 | -------------------------------------------------------------------------------- /internal/proxy/proxytest/proxytest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package proxytest 6 | 7 | import ( 8 | "fmt" 9 | "path/filepath" 10 | "strings" 11 | "testing" 12 | 13 | "golang.org/x/mod/modfile" 14 | "golang.org/x/pkgsite-metrics/internal/proxy" 15 | "golang.org/x/pkgsite-metrics/internal/testing/testhelper" 16 | "golang.org/x/tools/txtar" 17 | ) 18 | 19 | // SetupTestClient creates a fake module proxy for testing using the given test 20 | // version information. 21 | // 22 | // It returns a function for tearing down the proxy after the test is completed 23 | // and a Client for interacting with the test proxy. 24 | func SetupTestClient(t *testing.T, modules []*Module) (*proxy.Client, func()) { 25 | t.Helper() 26 | s := NewServer(modules) 27 | client, serverClose, err := NewClientForServer(s) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | return client, serverClose 32 | } 33 | 34 | // NewClientForServer starts serving proxyMux locally. It returns a client to the 35 | // server and a function to shut down the server. 36 | func NewClientForServer(s *Server) (*proxy.Client, func(), error) { 37 | // override client.httpClient to skip TLS verification 38 | httpClient, prox, serverClose := testhelper.SetupTestClientAndServer(s.mux) 39 | client, err := proxy.New(prox.URL) 40 | if err != nil { 41 | return nil, nil, err 42 | } 43 | client.HTTPClient = httpClient 44 | return client, serverClose, nil 45 | } 46 | 47 | // LoadTestModules reads the modules in the given directory. Each file in that 48 | // directory with a .txtar extension should be named "path@version" and should 49 | // be in txtar format (golang.org/x/tools/txtar). The path part of the filename 50 | // will be preceded by "example.com/" and colons will be replaced by slashes to 51 | // form a full module path. The file contents are used verbatim except that some 52 | // variables beginning with "$" are substituted with predefined strings. 53 | // 54 | // LoadTestModules panics if there is an error reading any of the files. 55 | func LoadTestModules(dir string) []*Module { 56 | files, err := filepath.Glob(filepath.Join(dir, "*.txtar")) 57 | if err != nil { 58 | panic(err) 59 | } 60 | var ms []*Module 61 | for _, f := range files { 62 | m, err := readTxtarModule(f) 63 | if err != nil { 64 | panic(err) 65 | } 66 | ms = append(ms, m) 67 | } 68 | return ms 69 | } 70 | 71 | var testModuleReplacer = strings.NewReplacer( 72 | "$MITLicense", testhelper.MITLicense, 73 | "$BSD0License", testhelper.BSD0License, 74 | ) 75 | 76 | func readTxtarModule(filename string) (*Module, error) { 77 | modver := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)) 78 | i := strings.IndexRune(modver, '@') 79 | if i < 0 { 80 | return nil, fmt.Errorf("%s: filename missing '@'", modver) 81 | } 82 | modulePath, version := "example.com/"+modver[:i], modver[i+1:] 83 | modulePath = strings.ReplaceAll(modulePath, ":", "/") 84 | if modulePath == "" || version == "" { 85 | return nil, fmt.Errorf("%s: empty module path or version", filename) 86 | } 87 | m := &Module{ 88 | ModulePath: modulePath, 89 | Version: version, 90 | Files: map[string]string{}, 91 | } 92 | ar, err := txtar.ParseFile(filename) 93 | if err != nil { 94 | return nil, err 95 | } 96 | for _, f := range ar.Files { 97 | if f.Name == "go.mod" { 98 | // Overwrite the pregenerated module path if one is specified in 99 | // the go.mod file. 100 | m.ModulePath = modfile.ModulePath(f.Data) 101 | } 102 | m.Files[f.Name] = strings.TrimSpace(testModuleReplacer.Replace(string(f.Data))) 103 | } 104 | return m, nil 105 | } 106 | 107 | // FindModule returns the module in mods with the given path and version, or nil 108 | // if there isn't one. An empty version argument matches any version. 109 | func FindModule(mods []*Module, path, version string) *Module { 110 | for _, m := range mods { 111 | if m.ModulePath == path && (version == "" || m.Version == version) { 112 | return m 113 | } 114 | } 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /internal/proxy/testdata/basic@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A simple module with a single package, which is at the module root. 2 | 3 | -- go.mod -- 4 | module example.com/basic 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- file1.go -- 13 | // Package basic is a sample package. 14 | package basic 15 | 16 | import "time" 17 | 18 | // Version is the same as the module version. 19 | const Version = "v1.0.0" 20 | 21 | // F is a function. 22 | func F(t time.Time, s string) (T, u) { 23 | x := 3 24 | x = C 25 | } 26 | 27 | // G is new in v1.1.0. 28 | func G() int { 29 | return 3 30 | } 31 | 32 | -- file2.go -- 33 | package basic 34 | 35 | var V = Version 36 | 37 | type T int 38 | 39 | type u int 40 | 41 | -- example_test.go -- 42 | package basic_test 43 | 44 | // Example for the package. 45 | func Example() { 46 | fmt.Println("hello") 47 | // Output: hello 48 | } 49 | 50 | // A function example. 51 | func ExampleF() { 52 | basic.F() 53 | } 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /internal/proxy/testdata/basic@v1.1.0.txtar: -------------------------------------------------------------------------------- 1 | A simple module with a single package, which is at the module root. 2 | 3 | -- go.mod -- 4 | module example.com/basic 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- file1.go -- 13 | // Package basic is a sample package. 14 | package basic 15 | 16 | import "time" 17 | 18 | // Version is the same as the module version. 19 | const Version = "v1.1.0" 20 | 21 | // F is a function. 22 | func F(t time.Time, s string) (T, u) { 23 | x := 3 24 | x = C 25 | } 26 | 27 | // G is new in v1.1.0. 28 | func G() int { 29 | return 3 30 | } 31 | 32 | -- file2.go -- 33 | package basic 34 | 35 | var V = Version 36 | 37 | type T int 38 | 39 | type u int 40 | 41 | -- example_test.go -- 42 | package basic_test 43 | 44 | // Example for the package. 45 | func Example() { 46 | fmt.Println("hello") 47 | // Output: hello 48 | } 49 | 50 | // A function example. 51 | func ExampleF() { 52 | basic.F() 53 | } 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /internal/proxy/testdata/build-constraints@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module with files that have build constraints. 2 | 3 | -- go.mod -- 4 | module example.com/build-constraints 5 | 6 | -- LICENSE -- 7 | $BSD0License 8 | 9 | -- cpu/cpu.go -- 10 | // Package cpu implements processor feature detection 11 | // used by the Go standard library. 12 | package cpu 13 | 14 | -- cpu/cpu_arm.go -- 15 | package cpu 16 | 17 | nconst CacheLinePadSize = 1 18 | 19 | -- cpu/cpu_arm64.go -- 20 | package cpu 21 | 22 | const CacheLinePadSize = 2 23 | 24 | -- cpu/cpu_x86.go -- 25 | // +build 386 amd64 amd64p32 26 | 27 | package cpu 28 | 29 | const CacheLinePadSize = 3 30 | 31 | -- ignore/ignore.go -- 32 | // +build ignore 33 | 34 | package ignore 35 | -------------------------------------------------------------------------------- /internal/proxy/testdata/deprecated@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module that is deprecated, according to its latest go.mod file 2 | (not this one). 3 | 4 | -- go.mod -- 5 | module example.com/deprecated 6 | 7 | -- LICENSE -- 8 | $MITLicense 9 | 10 | -- file.go -- 11 | // Package pkg is a sample package. 12 | package pkg 13 | 14 | // Version is the same as the module version. 15 | const Version = "v1.0.0" 16 | -------------------------------------------------------------------------------- /internal/proxy/testdata/deprecated@v1.1.0.txtar: -------------------------------------------------------------------------------- 1 | A module that is deprecated, according to its latest go.mod file. 2 | 3 | -- go.mod -- 4 | // Deprecated: use something else 5 | module example.com/deprecated 6 | 7 | -- LICENSE -- 8 | $MITLicense 9 | 10 | -- file.go -- 11 | // Package pkg is a sample package. 12 | package pkg 13 | 14 | // Version is the same as the module version. 15 | const Version = "v1.1.0" 16 | -------------------------------------------------------------------------------- /internal/proxy/testdata/generics@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module that uses generics. 2 | 3 | -- go.mod -- 4 | module example.com/generics 5 | 6 | go 1.18 7 | 8 | -- LICENSE -- 9 | $MITLicense 10 | 11 | -- file.go -- 12 | 13 | // Package generics uses generics. 14 | package generics 15 | 16 | import "constraints" 17 | 18 | func Min[T constraints.Ordered](a, b T) T { 19 | if a < b { 20 | return a 21 | } 22 | return b 23 | } 24 | 25 | type List[T any] struct { 26 | Val T 27 | Next *List[T] 28 | } 29 | 30 | -------------------------------------------------------------------------------- /internal/proxy/testdata/multi@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module with two packages, foo and bar. 2 | 3 | -- go.mod -- 4 | module example.com/multi 5 | 6 | go 1.13 7 | 8 | -- LICENSE -- 9 | $BSD0License 10 | 11 | -- README.md -- 12 | README file for testing. 13 | 14 | -- foo/LICENSE.md -- 15 | $MITLicense 16 | 17 | -- foo/foo.go -- 18 | // package foo 19 | package foo 20 | 21 | import ( 22 | "fmt" 23 | 24 | "example.com/multi/bar" 25 | ) 26 | 27 | // FooBar returns the string "foo bar". 28 | func FooBar() string { 29 | return fmt.Sprintf("foo %s", bar.Bar()) 30 | } 31 | 32 | -- bar/LICENSE -- 33 | $MITLicense 34 | 35 | -- bar/README -- 36 | Another README file for testing. 37 | 38 | -- bar/bar.go -- 39 | // package bar 40 | package bar 41 | 42 | // Bar returns the string "bar". 43 | func Bar() string { 44 | return "bar" 45 | } 46 | -------------------------------------------------------------------------------- /internal/proxy/testdata/nonredist@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module with multiple packages, one of which is not redistributable. 2 | 3 | -- go.mod -- 4 | module example.com/nonredist 5 | 6 | go 1.13 7 | 8 | -- LICENSE -- 9 | $BSD0License 10 | 11 | -- README.md -- 12 | README file for testing. 13 | 14 | -- bar/LICENSE -- 15 | $MITLicense 16 | 17 | -- bar/bar.go -- 18 | // package bar 19 | package bar 20 | 21 | // Bar returns the string "bar". 22 | func Bar() string { 23 | return "bar" 24 | } 25 | 26 | 27 | -- bar/baz/COPYING -- 28 | $MITLicense 29 | -- bar/baz/baz.go -- 30 | // package baz 31 | package baz 32 | 33 | // Baz returns the string "baz". 34 | func Baz() string { 35 | return "baz" 36 | } 37 | 38 | -- unk/README.md -- 39 | README file will be removed before DB insert. 40 | 41 | -- unk/LICENSE.md -- 42 | An unknown license. 43 | 44 | -- unk/unk.go -- 45 | // package unk 46 | package unk 47 | 48 | import ( 49 | "fmt" 50 | 51 | "example.com/nonredist/bar" 52 | ) 53 | 54 | // FooBar returns the string "foo bar". 55 | func FooBar() string { 56 | return fmt.Sprintf("foo %s", bar.Bar()) 57 | } 58 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | // Hello returns a greeting. 12 | func Hello() string { 13 | return "Hello, world." 14 | } 15 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.1.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | // Hello returns a greeting. 12 | func Hello() string { 13 | return "Hello, world." 14 | } 15 | 16 | // Glass returns a useful phrase for world travelers. 17 | func Glass() string { 18 | // See http://www.oocities.org/nodotus/hbglass.html. 19 | return "I can eat glass and it doesn't hurt me." 20 | } 21 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.2.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | // Hello returns a greeting. 12 | func Hello() string { 13 | return "Hello, world." 14 | } 15 | 16 | // Glass returns a useful phrase for world travelers. 17 | func Glass() string { 18 | // See http://www.oocities.org/nodotus/hbglass.html. 19 | return "I can eat glass and it doesn't hurt me." 20 | } 21 | 22 | // Go returns a Go proverb. 23 | func Go() string { 24 | return "Don't communicate by sharing memory, share memory by communicating." 25 | } 26 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.3.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | // Hello returns a greeting. 12 | func Hello() string { 13 | return "Hello, world." 14 | } 15 | 16 | // Glass returns a useful phrase for world travelers. 17 | func Glass() string { 18 | // See http://www.oocities.org/nodotus/hbglass.html. 19 | return "I can eat glass and it doesn't hurt me." 20 | } 21 | 22 | // Go returns a Go proverb. 23 | func Go() string { 24 | return "Don't communicate by sharing memory, share memory by communicating." 25 | } 26 | 27 | // Opt returns an optimization truth. 28 | func Opt() string { 29 | // Wisdom from ken. 30 | return "If a program is too slow, it must have a loop." 31 | } 32 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.4.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | import "rsc.io/sampler" 12 | 13 | // Hello returns a greeting. 14 | func Hello() string { 15 | return sampler.Hello() 16 | } 17 | 18 | // Glass returns a useful phrase for world travelers. 19 | func Glass() string { 20 | // See http://www.oocities.org/nodotus/hbglass.html. 21 | return "I can eat glass and it doesn't hurt me." 22 | } 23 | 24 | // Go returns a Go proverb. 25 | func Go() string { 26 | return "Don't communicate by sharing memory, share memory by communicating." 27 | } 28 | 29 | // Opt returns an optimization truth. 30 | func Opt() string { 31 | // Wisdom from ken. 32 | return "If a program is too slow, it must have a loop." 33 | } 34 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v1.5.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | import "rsc.io/sampler" 12 | 13 | // Hello returns a greeting. 14 | func Hello() string { 15 | return sampler.Hello() 16 | } 17 | 18 | // Glass returns a useful phrase for world travelers. 19 | func Glass() string { 20 | // See http://www.oocities.org/nodotus/hbglass.html. 21 | return "I can eat glass and it doesn't hurt me." 22 | } 23 | 24 | // Go returns a Go proverb. 25 | func Go() string { 26 | return "Don't communicate by sharing memory, share memory by communicating." 27 | } 28 | 29 | // Opt returns an optimization truth. 30 | func Opt() string { 31 | // Wisdom from ken. 32 | return "If a program is too slow, it must have a loop." 33 | } 34 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v3.0.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote/v3 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | import "rsc.io/sampler" 12 | 13 | // Hello returns a greeting. 14 | func HelloV3() string { 15 | return sampler.Hello() 16 | } 17 | 18 | // Glass returns a useful phrase for world travelers. 19 | func GlassV3() string { 20 | // See http://www.oocities.org/nodotus/hbglass.html. 21 | return "I can eat glass and it doesn't hurt me." 22 | } 23 | 24 | // Go returns a Go proverb. 25 | func GoV3() string { 26 | return "Don't communicate by sharing memory, share memory by communicating." 27 | } 28 | 29 | // Opt returns an optimization truth. 30 | func OptV3() string { 31 | // Wisdom from ken. 32 | return "If a program is too slow, it must have a loop." 33 | } 34 | -------------------------------------------------------------------------------- /internal/proxy/testdata/quote@v3.1.0.txtar: -------------------------------------------------------------------------------- 1 | -- go.mod -- 2 | module rsc.io/quote/v3 3 | 4 | -- LICENSE -- 5 | $MITLicense 6 | 7 | -- quote.go -- 8 | // Package quote collects pithy sayings. 9 | package quote // import "rsc.io/quote" 10 | 11 | import "rsc.io/sampler" 12 | 13 | // Hello returns a greeting. 14 | func HelloV3() string { 15 | return sampler.Hello() 16 | } 17 | 18 | // Concurrency returns a Go proverb about concurrency. 19 | func Concurrency() string { 20 | return "Concurrency is not parallelism." 21 | } 22 | 23 | // Glass returns a useful phrase for world travelers. 24 | func GlassV3() string { 25 | // See http://www.oocities.org/nodotus/hbglass.html. 26 | return "I can eat glass and it doesn't hurt me." 27 | } 28 | 29 | // Go returns a Go proverb. 30 | func GoV3() string { 31 | return "Don't communicate by sharing memory, share memory by communicating." 32 | } 33 | 34 | // Opt returns an optimization truth. 35 | func OptV3() string { 36 | // Wisdom from ken. 37 | return "If a program is too slow, it must have a loop." 38 | } 39 | -------------------------------------------------------------------------------- /internal/proxy/testdata/retractions@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module with some versions retracted. 2 | This is not the latest version, so the retract directive is ignored. 3 | 4 | -- go.mod -- 5 | module example.com/retractions 6 | 7 | retract v1.0.0 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- file.go -- 13 | // Package pkg is a sample package. 14 | package pkg 15 | 16 | // Version is the same as the module version. 17 | const Version = "v1.0.0" 18 | -------------------------------------------------------------------------------- /internal/proxy/testdata/retractions@v1.1.0.txtar: -------------------------------------------------------------------------------- 1 | A module with some versions retracted. 2 | This is not the latest version. 3 | 4 | -- go.mod -- 5 | module example.com/retractions 6 | 7 | -- LICENSE -- 8 | $MITLicense 9 | 10 | -- file.go -- 11 | // Package pkg is a sample package. 12 | package pkg 13 | 14 | // Version is the same as the module version. 15 | const Version = "v1.1.0" 16 | -------------------------------------------------------------------------------- /internal/proxy/testdata/retractions@v1.2.0.txtar: -------------------------------------------------------------------------------- 1 | A module with some versions retracted. 2 | This is the latest version. It retracts itself. 3 | 4 | -- go.mod -- 5 | module example.com/retractions 6 | 7 | retract ( 8 | v1.2.0 // bad 9 | v1.1.0 // worse 10 | ) 11 | 12 | -- LICENSE -- 13 | $MITLicense 14 | 15 | -- file.go -- 16 | // Package pkg is a sample package. 17 | package pkg 18 | 19 | // Version is the same as the module version. 20 | const Version = "v1.2.0" 21 | -------------------------------------------------------------------------------- /internal/proxy/testdata/single@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module with a single package that is below the module root. 2 | 3 | -- go.mod -- 4 | module example.com/single 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- pkg/file1.go -- 13 | // Package pkg is a sample package. 14 | package pkg 15 | 16 | import "time" 17 | 18 | // Version is the same as the module version. 19 | const Version = "v1.0.0" 20 | 21 | // F is a function. 22 | func F(t time.Time, s string) (T, u) { 23 | x := 3 24 | x = C 25 | } 26 | 27 | // G is new in v1.1.0. 28 | func G() int { 29 | return 3 30 | } 31 | 32 | -- pkg/file2.go -- 33 | package pkg 34 | 35 | var V = Version 36 | 37 | type T int 38 | 39 | type u int 40 | 41 | -- pkg/example_test.go -- 42 | package pkg_test 43 | 44 | // Example for the package. 45 | func Example() { 46 | fmt.Println("hello") 47 | // Output: hello 48 | } 49 | 50 | // A function example. 51 | func ExampleF() { 52 | pkg.F() 53 | } 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /internal/proxy/testdata/symbols@v1.0.0.txtar: -------------------------------------------------------------------------------- 1 | A module used for testing the symbols logic. 2 | 3 | -- go.mod -- 4 | module example.com/symbols 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- symbols.go -- 13 | package symbols 14 | 15 | // const 16 | const C = 1 17 | 18 | // const iota 19 | const ( 20 | AA = iota + 1 21 | _ 22 | BB 23 | CC 24 | ) 25 | 26 | type Num int 27 | 28 | const ( 29 | DD Num = iota 30 | _ 31 | EE 32 | FF 33 | ) 34 | 35 | // var 36 | var V = 2 37 | 38 | // Multiple variables on the same line. 39 | var A, B string 40 | 41 | // func 42 | func F() {} 43 | 44 | // type 45 | type T int 46 | 47 | // typeConstant 48 | const CT T = 3 49 | 50 | // typeVariable 51 | var VT T 52 | 53 | // multi-line var 54 | var ( 55 | ErrA = errors.New("error A") 56 | ErrB = errors.New("error B") 57 | ) 58 | 59 | // typeFunc 60 | func TF() T { return T(0) } 61 | 62 | // method 63 | // BUG(uid): this verifies that notes are rendered 64 | func (T) M() {} 65 | 66 | type S1 struct { 67 | F int // field 68 | } 69 | 70 | type S2 struct { 71 | S1 // embedded struct; should have an id 72 | } 73 | 74 | type I1 interface { 75 | M1() 76 | } 77 | 78 | type I2 interface { 79 | I1 // embedded interface; should not have an id 80 | } 81 | 82 | type ( 83 | Int int 84 | ) 85 | -------------------------------------------------------------------------------- /internal/proxy/testdata/symbols@v1.1.0.txtar: -------------------------------------------------------------------------------- 1 | A module used for testing the symbols logic. 2 | 3 | -- go.mod -- 4 | module example.com/symbols 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- symbols.go -- 13 | package symbols 14 | 15 | // const 16 | const C = 1 17 | 18 | // const iota 19 | const ( 20 | AA = iota + 1 21 | _ 22 | BB 23 | CC 24 | ) 25 | 26 | type Num int 27 | 28 | const ( 29 | DD Num = iota 30 | _ 31 | EE 32 | FF 33 | ) 34 | 35 | // var 36 | var V = 2 37 | 38 | // Multiple variables on the same line. 39 | var A, B string 40 | 41 | // func 42 | func F() {} 43 | 44 | // type 45 | type T int 46 | 47 | // typeConstant 48 | const CT T = 3 49 | 50 | // typeVariable 51 | var VT T 52 | 53 | // multi-line var 54 | var ( 55 | ErrA = errors.New("error A") 56 | ErrB = errors.New("error B") 57 | ) 58 | 59 | // typeFunc 60 | func TF() T { return T(0) } 61 | 62 | // method 63 | // BUG(uid): this verifies that notes are rendered 64 | func (T) M() {} 65 | 66 | type S1 struct { 67 | F int // field 68 | } 69 | 70 | type S2 struct { 71 | S1 // embedded struct; should have an id 72 | G int 73 | } 74 | 75 | type I1 interface { 76 | M1() 77 | } 78 | 79 | type I2 interface { 80 | I1 // embedded interface; should not have an id 81 | M2() 82 | } 83 | 84 | type ( 85 | Int int 86 | String bool 87 | ) 88 | 89 | -- hello/hello.go -- 90 | // +build linux darwin 91 | // +build amd64 92 | 93 | package hello 94 | 95 | // Hello returns a greeting. 96 | func Hello() string { 97 | return "Hello" 98 | } 99 | 100 | -- hello/hello_js.go -- 101 | // +build js,wasm 102 | 103 | package hello 104 | 105 | // HelloJS returns a greeting when the build context is js/wasm. 106 | func HelloJS() string { 107 | return "Hello" 108 | } 109 | 110 | -- multigoos/multigoos.go -- 111 | // +build darwin linux windows 112 | 113 | package multigoos 114 | 115 | // type FD is introduced for windows, linux and darwin at this version. 116 | type FD struct {} 117 | 118 | -- multigoos/multigoos_windows.go -- 119 | // +build windows 120 | 121 | package multigoos 122 | 123 | // Different signature from CloseOnExec for linux and darwin. 124 | func CloseOnExec(foo string) error { 125 | return nil 126 | } 127 | 128 | -- multigoos/multigoos_unix.go -- 129 | // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris 130 | 131 | package multigoos 132 | 133 | // Different signature from CloseOnExec for windows. 134 | func CloseOnExec(num int) (int, error) { 135 | return num, nil 136 | } 137 | 138 | -- duplicate/duplicate.go -- 139 | // +build linux darwin 140 | 141 | package duplicate 142 | 143 | // Unexported here, exported in v1.2.0. 144 | type tokenType int 145 | 146 | // Token types. 147 | const ( 148 | TokenShort tokenType = iota 149 | ) 150 | -------------------------------------------------------------------------------- /internal/proxy/testdata/symbols@v1.2.0.txtar: -------------------------------------------------------------------------------- 1 | A module used for testing the symbols logic. 2 | 3 | -- go.mod -- 4 | module example.com/symbols 5 | 6 | -- README.md -- 7 | This is the README for a test module. 8 | 9 | -- LICENSE -- 10 | $MITLicense 11 | 12 | -- symbols.go -- 13 | package symbols 14 | 15 | // const 16 | const C = 1 17 | 18 | // const iota 19 | const ( 20 | AA = iota + 1 21 | _ 22 | BB 23 | CC 24 | ) 25 | 26 | type Num int 27 | 28 | const ( 29 | DD Num = iota 30 | _ 31 | EE 32 | FF 33 | ) 34 | 35 | // var 36 | var V = 2 37 | 38 | // Multiple variables on the same line. 39 | var A, B string 40 | 41 | // func 42 | func F() {} 43 | 44 | // type 45 | type T int 46 | 47 | // typeConstant 48 | const CT T = 3 49 | 50 | // typeVariable 51 | var VT T 52 | 53 | // multi-line var 54 | var ( 55 | ErrA = errors.New("error A") 56 | ErrB = errors.New("error B") 57 | ) 58 | 59 | // typeFunc 60 | func TF() T { return T(0) } 61 | 62 | // method 63 | // BUG(uid): this verifies that notes are rendered 64 | func (T) M() {} 65 | 66 | type S1 struct { 67 | F int // field 68 | } 69 | 70 | type S2 struct { 71 | S1 // embedded struct; should have an id 72 | G int 73 | } 74 | 75 | type I1 interface { 76 | M1() 77 | } 78 | 79 | type I2 interface { 80 | I1 // embedded interface; should not have an id 81 | M2() 82 | } 83 | 84 | type ( 85 | Int int 86 | String bool 87 | ) 88 | 89 | -- hello/hello.go -- 90 | package hello 91 | 92 | // Hello returns a greeting. 93 | func Hello() string { 94 | return "Hello" 95 | } 96 | 97 | -- hello/hello_js.go -- 98 | // +build js,wasm 99 | 100 | package hello 101 | 102 | // HelloJS returns a greeting when the build context is js/wasm. 103 | func HelloJS() string { 104 | return "Hello" 105 | } 106 | 107 | -- multigoos/multigoos_windows.go -- 108 | // +build windows 109 | 110 | package multigoos 111 | 112 | func CloseOnExec(foo string) error { 113 | return nil 114 | } 115 | 116 | type FD struct {} 117 | 118 | // FD was introduced in v1.1.0 for linux, darwin and windows. 119 | // MyWindowsMethod is introduced only for windows in this version. 120 | func (*FD) MyWindowsMethod() { 121 | } 122 | 123 | -- multigoos/multigoos_unix.go -- 124 | // +build aix darwin dragonfly freebsd linux netbsd openbsd solaris 125 | 126 | package multigoos 127 | 128 | func CloseOnExec(num int) (int, error) { 129 | return num, nil 130 | } 131 | 132 | type FD struct {} 133 | 134 | // FD was introduced in v1.1.0 for linux, darwin and windows. 135 | // MyMethod is introduced only for darwin and linux in this version. 136 | func (*FD) MyMethod() { 137 | } 138 | 139 | -- multigoos/multigoos_js.go -- 140 | // +build js,wasm 141 | 142 | package multigoos 143 | 144 | func CloseOnExec(n int) { 145 | } 146 | 147 | -- duplicate/duplicate.go -- 148 | // +build linux darwin 149 | 150 | package duplicate 151 | 152 | type TokenType int 153 | 154 | // Token types. 155 | const ( 156 | TokenShort TokenType = iota 157 | ) 158 | 159 | -- duplicate/duplicate_windows.go -- 160 | // +build windows 161 | 162 | package duplicate 163 | 164 | // Constant here, type for JS, linux and darwin. 165 | const TokenType = 3 166 | 167 | -- duplicate/duplicate_js.go -- 168 | // +build js 169 | 170 | package duplicate 171 | 172 | // Exported here, unexported in v1.1.0. 173 | type TokenType struct { 174 | } 175 | 176 | func TokenShort() TokenType { return &TokenType{} } 177 | -------------------------------------------------------------------------------- /internal/queue/queue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package queue 6 | 7 | import ( 8 | "testing" 9 | 10 | taskspb "cloud.google.com/go/cloudtasks/apiv2/cloudtaskspb" 11 | "github.com/google/go-cmp/cmp" 12 | "golang.org/x/pkgsite-metrics/internal/config" 13 | "google.golang.org/protobuf/testing/protocmp" 14 | "google.golang.org/protobuf/types/known/durationpb" 15 | ) 16 | 17 | type testTask struct { 18 | name string 19 | path string 20 | params string 21 | } 22 | 23 | func (t *testTask) Name() string { return t.name } 24 | func (t *testTask) Path() string { return t.path } 25 | func (t *testTask) Params() string { return t.params } 26 | 27 | func TestNewTaskID(t *testing.T) { 28 | for _, test := range []struct { 29 | name, path, params string 30 | want string 31 | }{ 32 | { 33 | "m@v1.2", "path", "params", 34 | "m_v1_2-ns-31026413", 35 | }, 36 | { 37 | "µπΩ/github.com@v2.3.4-ß", "p", "", 38 | "_00b5_03c0_03a9_-github_com_v2_3_4-_00df-ns-148de9c5", 39 | }, 40 | } { 41 | tt := &testTask{test.name, test.path, test.params} 42 | got := newTaskID("ns", tt) 43 | if got != test.want { 44 | t.Errorf("%v: got %s, want %s", tt, got, test.want) 45 | } 46 | } 47 | } 48 | 49 | func TestNewTaskRequest(t *testing.T) { 50 | cfg := config.Config{ 51 | ProjectID: "Project", 52 | LocationID: "us-central1", 53 | QueueURL: "http://1.2.3.4:8000", 54 | ServiceAccount: "sa", 55 | } 56 | want := &taskspb.CreateTaskRequest{ 57 | Parent: "projects/Project/locations/us-central1/queues/queueID", 58 | Task: &taskspb.Task{ 59 | DispatchDeadline: durationpb.New(maxCloudTasksTimeout), 60 | MessageType: &taskspb.Task_HttpRequest{ 61 | HttpRequest: &taskspb.HttpRequest{ 62 | HttpMethod: taskspb.HttpMethod_POST, 63 | Url: "http://1.2.3.4:8000/test/scan/mod@v1.2.3?importedby=0&mode=test&insecure=true", 64 | AuthorizationHeader: &taskspb.HttpRequest_OidcToken{ 65 | OidcToken: &taskspb.OidcToken{ 66 | ServiceAccountEmail: "sa", 67 | }, 68 | }, 69 | }, 70 | }, 71 | }, 72 | } 73 | gcp, err := newGCP(&cfg, nil, "queueID") 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | opts := &Options{ 78 | Namespace: "test", 79 | TaskNameSuffix: "suf", 80 | } 81 | sreq := &testTask{ 82 | name: "name", 83 | path: "mod@v1.2.3", 84 | params: "importedby=0&mode=test&insecure=true", 85 | } 86 | got, err := gcp.newTaskRequest(sreq, opts) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | want.Task.Name = got.Task.Name 91 | if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { 92 | t.Errorf("mismatch (-want, +got):\n%s", diff) 93 | } 94 | 95 | opts.DisableProxyFetch = true 96 | want.Task.MessageType.(*taskspb.Task_HttpRequest).HttpRequest.Url += "&proxyfetch=off" 97 | got, err = gcp.newTaskRequest(sreq, opts) 98 | if err != nil { 99 | t.Fatal(err) 100 | } 101 | want.Task.Name = got.Task.Name 102 | if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { 103 | t.Errorf("mismatch (-want, +got):\n%s", diff) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/sandbox/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Go Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style 3 | # license that can be found in the LICENSE file. 4 | 5 | # Makefile for the sandbox package. 6 | # `make` will build and install binaries needed for the test, run the test as root, 7 | # then clean up. 8 | 9 | default: test clean 10 | 11 | test: /usr/local/bin/runsc testbundle 12 | sudo RUN_FROM_MAKE=1 $(shell which go) test -v 13 | 14 | 15 | # Release version must match the one in cmd/worker/Dockerfile. 16 | RUNSC_URL := https://storage.googleapis.com/gvisor/releases/release/20240930.0/$(shell uname -m) 17 | 18 | # This is an edited version of the commands at https://gvisor.dev/docs/user_guide/install. 19 | /usr/local/bin/runsc: 20 | wget $(RUNSC_URL)/runsc $(RUNSC_URL)/runsc.sha512 21 | sha512sum -c runsc.sha512 22 | rm -f *.sha512 23 | chmod a+rx runsc 24 | sudo mv runsc /usr/local/bin 25 | 26 | testbundle: testdata/bundle/rootfs/runner testdata/bundle/rootfs/printargs 27 | chmod o+rx testdata/bundle/rootfs 28 | 29 | testdata/bundle/rootfs/runner: runner.go 30 | go build -o $@ $< 31 | chmod o+rx $@ 32 | 33 | testdata/bundle/rootfs/printargs: testdata/printargs.go 34 | go build -o $@ $< 35 | chmod o+rx $@ 36 | 37 | clean: 38 | rm testdata/bundle/rootfs/runner 39 | rm testdata/bundle/rootfs/printargs 40 | 41 | .PHONY: clean testbundle 42 | 43 | -------------------------------------------------------------------------------- /internal/sandbox/runner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | // Package main defines a program that runs another program 8 | // provided on standard input, prints its standard output, then 9 | // terminates. It logs to stderr. 10 | // 11 | // The input is expected to be json content encoding an exec.Cmd 12 | // structure extended with a boolean AppendToEnv field. 13 | package main 14 | 15 | import ( 16 | "bytes" 17 | "encoding/json" 18 | "errors" 19 | "io" 20 | "log" 21 | "os" 22 | "os/exec" 23 | ) 24 | 25 | func main() { 26 | log.SetOutput(os.Stderr) 27 | log.SetPrefix("runner: ") 28 | log.Print("starting") 29 | in, err := io.ReadAll(os.Stdin) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | log.Printf("read %q", in) 34 | var cmd struct { 35 | exec.Cmd 36 | AppendToEnv bool 37 | } 38 | if err := json.Unmarshal(in, &cmd); err != nil { 39 | log.Fatal(err) 40 | } 41 | if cmd.AppendToEnv { 42 | cmd.Env = append(os.Environ(), cmd.Env...) 43 | } 44 | log.Printf("cmd: %+v", cmd) 45 | out, err := cmd.Output() 46 | if err != nil { 47 | s := err.Error() 48 | var eerr *exec.ExitError 49 | if errors.As(err, &eerr) { 50 | s += ": " + string(bytes.TrimSpace(eerr.Stderr)) 51 | } 52 | log.Fatalf("%v failed with %s", cmd.Args, s) 53 | } 54 | if _, err := os.Stdout.Write(out); err != nil { 55 | log.Fatal(err) 56 | } 57 | log.Print("succeeded") 58 | } 59 | -------------------------------------------------------------------------------- /internal/sandbox/sandbox_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sandbox 6 | 7 | import ( 8 | "errors" 9 | "os" 10 | "os/exec" 11 | "strings" 12 | "testing" 13 | 14 | "golang.org/x/pkgsite-metrics/internal/derrors" 15 | test "golang.org/x/pkgsite-metrics/internal/testing" 16 | ) 17 | 18 | func TestIntegration(t *testing.T) { 19 | test.NeedsIntegrationEnv(t) 20 | 21 | cmd := exec.Command("make") 22 | cmd.Stdout = os.Stdout 23 | cmd.Stderr = os.Stderr 24 | if err := cmd.Run(); err != nil { 25 | t.Fatal(err) 26 | } 27 | } 28 | 29 | // TestSandbox tests require a minimal bundle, in testdata/bundle. 30 | // The Makefile in this directory will build and install the binaries 31 | // needed for the test. See TestIntegration above. 32 | func TestSandbox(t *testing.T) { 33 | if os.Getenv("RUN_FROM_MAKE") != "1" { 34 | t.Skip("skipping; must run with 'make'.") 35 | } 36 | sb := New("testdata/bundle") 37 | sb.Runsc = "/usr/local/bin/runsc" // must match path in Makefile 38 | if err := sb.Validate(); err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | check := func(t *testing.T, cmd *Cmd, want string) { 43 | t.Helper() 44 | out, err := cmd.Output() 45 | if err != nil { 46 | t.Fatal(derrors.IncludeStderr(err)) 47 | } 48 | got := string(out) 49 | if got != want { 50 | t.Fatalf("got\n%q\nwant\n%q", got, want) 51 | } 52 | } 53 | 54 | t.Run("printargs", func(t *testing.T) { 55 | check(t, sb.Command("printargs", "a", "b"), `args: 56 | 0: "a" 57 | 1: "b"`) 58 | }) 59 | 60 | t.Run("space in arg", func(t *testing.T) { 61 | check(t, sb.Command("printargs", "a", "b c\td"), `args: 62 | 0: "a" 63 | 1: "b c\td"`) 64 | }) 65 | 66 | t.Run("replace env", func(t *testing.T) { 67 | cmd := sb.Command("printargs", "$HOME", "$FOO") 68 | cmd.Env = []string{"FOO=17"} 69 | check(t, cmd, `args: 70 | 0: "" 71 | 1: "17"`) 72 | }) 73 | t.Run("append to env", func(t *testing.T) { 74 | cmd := sb.Command("printargs", "$HOME", "$FOO") 75 | cmd.Env = []string{"FOO=17"} 76 | cmd.AppendToEnv = true 77 | check(t, cmd, `args: 78 | 0: "/" 79 | 1: "17"`) 80 | }) 81 | t.Run("no program", func(t *testing.T) { 82 | _, err := sb.Command("foo").Output() 83 | var eerr *exec.ExitError 84 | if !errors.As(err, &eerr) { 85 | t.Fatalf("got %T, wanted *exec.ExitError", err) 86 | } 87 | if g, w := eerr.ExitCode(), 1; g != w { 88 | t.Fatalf("got exit code %d, wanted %d", g, w) 89 | } 90 | if g, w := string(eerr.Stderr), "no such file"; !strings.Contains(g, w) { 91 | t.Fatalf("got\n%q\nwhich does not contain %q", g, w) 92 | } 93 | }) 94 | } 95 | 96 | func TestValidate(t *testing.T) { 97 | // Validate doesn't actually run the sandbox, so we can test it. 98 | t.Skip("fails in gcloud build") 99 | sb := New("testdata/bundle") 100 | sb.Runsc = "/usr/local/bin/runsc" 101 | if err := sb.Validate(); err != nil { 102 | t.Fatal(err) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ociVersion": "1.0.0", 3 | "process": { 4 | "user": { 5 | "uid": 0, 6 | "gid": 0 7 | }, 8 | "args": [ 9 | "/runner" 10 | ], 11 | "env": [ 12 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 13 | "TERM=xterm" 14 | ], 15 | "cwd": "/", 16 | "capabilities": { 17 | "bounding": [ 18 | "CAP_AUDIT_WRITE", 19 | "CAP_KILL", 20 | "CAP_NET_BIND_SERVICE" 21 | ], 22 | "effective": [ 23 | "CAP_AUDIT_WRITE", 24 | "CAP_KILL", 25 | "CAP_NET_BIND_SERVICE" 26 | ], 27 | "inheritable": [ 28 | "CAP_AUDIT_WRITE", 29 | "CAP_KILL", 30 | "CAP_NET_BIND_SERVICE" 31 | ], 32 | "permitted": [ 33 | "CAP_AUDIT_WRITE", 34 | "CAP_KILL", 35 | "CAP_NET_BIND_SERVICE" 36 | ], 37 | "ambient": [ 38 | "CAP_AUDIT_WRITE", 39 | "CAP_KILL", 40 | "CAP_NET_BIND_SERVICE" 41 | ] 42 | }, 43 | "rlimits": [ 44 | { 45 | "type": "RLIMIT_NOFILE", 46 | "hard": 1024, 47 | "soft": 1024 48 | } 49 | ] 50 | }, 51 | "root": { 52 | "path": "rootfs", 53 | "readonly": false 54 | }, 55 | "hostname": "runsc", 56 | "mounts": [ 57 | { 58 | "destination": "/proc", 59 | "type": "proc", 60 | "source": "proc" 61 | }, 62 | { 63 | "destination": "/dev", 64 | "type": "tmpfs", 65 | "source": "tmpfs", 66 | "options": [] 67 | }, 68 | { 69 | "destination": "/sys", 70 | "type": "sysfs", 71 | "source": "sysfs", 72 | "options": [ 73 | "nosuid", 74 | "noexec", 75 | "nodev", 76 | "ro" 77 | ] 78 | }, 79 | { 80 | "destination": "/tmp/foo", 81 | "type": "none", 82 | "source": "/", 83 | "options": ["bind"] 84 | } 85 | ], 86 | "linux": { 87 | "namespaces": [ 88 | { 89 | "type": "pid" 90 | }, 91 | { 92 | "type": "network" 93 | }, 94 | { 95 | "type": "ipc" 96 | }, 97 | { 98 | "type": "uts" 99 | }, 100 | { 101 | "type": "mount" 102 | } 103 | ] 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/rootfs/.dockerenv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/pkgsite-metrics/0f64c811394707791dd58eaece7af0ff7d811a0f/internal/sandbox/testdata/bundle/rootfs/.dockerenv -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/rootfs/etc/hostname: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/pkgsite-metrics/0f64c811394707791dd58eaece7af0ff7d811a0f/internal/sandbox/testdata/bundle/rootfs/etc/hostname -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/rootfs/etc/hosts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/pkgsite-metrics/0f64c811394707791dd58eaece7af0ff7d811a0f/internal/sandbox/testdata/bundle/rootfs/etc/hosts -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/rootfs/etc/mtab: -------------------------------------------------------------------------------- 1 | /proc/mounts -------------------------------------------------------------------------------- /internal/sandbox/testdata/bundle/rootfs/etc/resolv.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/pkgsite-metrics/0f64c811394707791dd58eaece7af0ff7d811a0f/internal/sandbox/testdata/bundle/rootfs/etc/resolv.conf -------------------------------------------------------------------------------- /internal/sandbox/testdata/printargs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build ignore 6 | 7 | // This program prints its arguments and exits. 8 | // If an argument begins with a "$", it prints 9 | // the value of the environment variable instead. 10 | // It is used for testing the sandbox package. 11 | package main 12 | 13 | import ( 14 | "fmt" 15 | "os" 16 | ) 17 | 18 | func main() { 19 | fmt.Printf("args:\n") 20 | for i, arg := range os.Args[1:] { 21 | val := arg 22 | if len(arg) > 0 && arg[0] == '$' { 23 | val = os.Getenv(arg[1:]) 24 | } 25 | fmt.Printf("%d: %q\n", i, val) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /internal/scan/testdata/modules.txt: -------------------------------------------------------------------------------- 1 | # test module corpus file 2 | 3 | m1 v1.0.0 18 4 | m2 v2.3.4 5 5 | m3 1 6 | -------------------------------------------------------------------------------- /internal/secrets.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package internal 6 | 7 | import ( 8 | "context" 9 | 10 | secretmanager "cloud.google.com/go/secretmanager/apiv1" 11 | smpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" 12 | "golang.org/x/pkgsite-metrics/internal/derrors" 13 | ) 14 | 15 | // GetSecret retrieves a secret from the GCP Secret Manager. 16 | // secretFullName should be of the form "projects/PROJECT/secrets/NAME". 17 | func GetSecret(ctx context.Context, secretFullName string) (_ string, err error) { 18 | defer derrors.Wrap(&err, "GetSecret(ctx, %q)", secretFullName) 19 | 20 | client, err := secretmanager.NewClient(ctx) 21 | if err != nil { 22 | return "", err 23 | } 24 | defer client.Close() 25 | result, err := client.AccessSecretVersion(ctx, &smpb.AccessSecretVersionRequest{ 26 | Name: secretFullName + "/versions/latest", 27 | }) 28 | if err != nil { 29 | return "", err 30 | } 31 | return string(result.Payload.Data), nil 32 | } 33 | -------------------------------------------------------------------------------- /internal/testdata/module/go.mod: -------------------------------------------------------------------------------- 1 | module golang.org/vuln 2 | 3 | go 1.18 4 | 5 | // This version has a vulnerability that is called. 6 | require golang.org/x/text v0.3.0 7 | -------------------------------------------------------------------------------- /internal/testdata/module/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 2 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 3 | -------------------------------------------------------------------------------- /internal/testdata/module/vuln.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "golang.org/x/text/language" 4 | 5 | func main() { 6 | language.Parse("") 7 | } 8 | -------------------------------------------------------------------------------- /internal/testdata/multipleBinModule/go.mod: -------------------------------------------------------------------------------- 1 | module example.com/test 2 | 3 | go 1.20 -------------------------------------------------------------------------------- /internal/testdata/multipleBinModule/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | fmt.Println("Hello World") 7 | } 8 | -------------------------------------------------------------------------------- /internal/testdata/multipleBinModule/multipleBinModule/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | ) 7 | 8 | func main() { 9 | x := math.Abs(-3) 10 | fmt.Print(x) 11 | } 12 | -------------------------------------------------------------------------------- /internal/testdata/multipleBinModule/p1/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // This package doesn't use any of the code in M, but instead 9 | // is used to build something or as a helper 10 | 11 | func main() { 12 | s := strings.Join([]string{"One", "Two"}, " ") 13 | fmt.Println(s) 14 | } 15 | -------------------------------------------------------------------------------- /internal/testdata/multipleBinModule/p2/file.go: -------------------------------------------------------------------------------- 1 | package p2 2 | 3 | import "fmt" 4 | 5 | func DoSmthn() { 6 | fmt.Print("test") 7 | } 8 | -------------------------------------------------------------------------------- /internal/testdata/vulndb/ID/GO-2020-0015.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "GO-2020-0015", 3 | "published": "2021-04-14T20:04:52Z", 4 | "modified": "2021-06-07T12:00:00Z", 5 | "aliases": [ 6 | "CVE-2020-14040", 7 | "GHSA-5rcv-m4m3-hfh7" 8 | ], 9 | "details": "An attacker could provide a single byte to a UTF16 decoder instantiated with\nUseBOM or ExpectBOM to trigger an infinite loop if the String function on\nthe Decoder is called, or the Decoder is passed to transform.String.\nIf used to parse user supplied input, this may be used as a denial of service\nvector.\n", 10 | "affected": [ 11 | { 12 | "package": { 13 | "name": "golang.org/x/text", 14 | "ecosystem": "Go" 15 | }, 16 | "ranges": [ 17 | { 18 | "type": "SEMVER", 19 | "events": [ 20 | { 21 | "introduced": "0" 22 | }, 23 | { 24 | "fixed": "0.3.3" 25 | } 26 | ] 27 | } 28 | ], 29 | "database_specific": { 30 | "url": "https://pkg.go.dev/vuln/GO-2020-0015" 31 | }, 32 | "ecosystem_specific": { 33 | "imports": [ 34 | { 35 | "path": "golang.org/x/text/encoding/unicode", 36 | "symbols": [ 37 | "bomOverride.Transform", 38 | "utf16Decoder.Transform" 39 | ] 40 | }, 41 | { 42 | "path": "golang.org/x/text/transform", 43 | "symbols": [ 44 | "Transform" 45 | ] 46 | } 47 | ] 48 | } 49 | } 50 | ], 51 | "references": [ 52 | { 53 | "type": "FIX", 54 | "url": "https://go.dev/cl/238238" 55 | }, 56 | { 57 | "type": "FIX", 58 | "url": "https://go.googlesource.com/text/+/23ae387dee1f90d29a23c0e87ee0b46038fbed0e" 59 | }, 60 | { 61 | "type": "WEB", 62 | "url": "https://go.dev/issue/39491" 63 | }, 64 | { 65 | "type": "WEB", 66 | "url": "https://groups.google.com/g/golang-announce/c/bXVeAmGOqz0" 67 | }, 68 | { 69 | "type": "WEB", 70 | "url": "https://nvd.nist.gov/vuln/detail/CVE-2020-14040" 71 | }, 72 | { 73 | "type": "WEB", 74 | "url": "https://github.com/advisories/GHSA-5rcv-m4m3-hfh7" 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /internal/testdata/vulndb/ID/GO-2021-0113.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "GO-2021-0113", 3 | "published": "2021-10-06T17:51:21Z", 4 | "modified": "2021-10-06T17:51:21Z", 5 | "aliases": [ 6 | "CVE-2021-38561" 7 | ], 8 | "details": "Due to improper index calculation, an incorrectly formatted language tag can cause Parse\nto panic via an out of bounds read. If Parse is used to process untrusted user inputs,\nthis may be used as a vector for a denial of service attack.\n", 9 | "affected": [ 10 | { 11 | "package": { 12 | "name": "golang.org/x/text", 13 | "ecosystem": "Go" 14 | }, 15 | "ranges": [ 16 | { 17 | "type": "SEMVER", 18 | "events": [ 19 | { 20 | "introduced": "0" 21 | }, 22 | { 23 | "fixed": "0.3.7" 24 | } 25 | ] 26 | } 27 | ], 28 | "database_specific": { 29 | "url": "https://pkg.go.dev/vuln/GO-2021-0113" 30 | }, 31 | "ecosystem_specific": { 32 | "imports": [ 33 | { 34 | "path": "golang.org/x/text/language", 35 | "symbols": [ 36 | "MatchStrings", 37 | "MustParse", 38 | "Parse", 39 | "ParseAcceptLanguage" 40 | ] 41 | } 42 | ] 43 | } 44 | } 45 | ], 46 | "references": [ 47 | { 48 | "type": "FIX", 49 | "url": "https://go.dev/cl/340830" 50 | }, 51 | { 52 | "type": "FIX", 53 | "url": "https://go.googlesource.com/text/+/383b2e75a7a4198c42f8f87833eefb772868a56f" 54 | }, 55 | { 56 | "type": "WEB", 57 | "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-38561" 58 | } 59 | ] 60 | } -------------------------------------------------------------------------------- /internal/testdata/vulndb/index/db.json: -------------------------------------------------------------------------------- 1 | {"modified":"2023-05-18T20:38:56Z"} 2 | -------------------------------------------------------------------------------- /internal/testdata/vulndb/index/modules.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "path": "golang.org/x/text", 4 | "vulns": [ 5 | { 6 | "id": "GO-2020-0015", 7 | "modified": "2021-06-07T12:00:00Z" 8 | }, 9 | { 10 | "id": "GO-2021-0113", 11 | "modified": "2021-10-06T17:51:21Z" 12 | } 13 | ] 14 | } 15 | ] -------------------------------------------------------------------------------- /internal/testdata/vulndb/index/vulns.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "GO-2020-0015", 4 | "modified": "2021-06-07T12:00:00Z", 5 | "aliases": [ 6 | "CVE-2020-14040", 7 | "GHSA-5rcv-m4m3-hfh7" 8 | ] 9 | }, 10 | { 11 | "id": "GO-2021-0113", 12 | "modified": "2021-10-06T17:51:21Z", 13 | "aliases": [ 14 | "CVE-2021-38561" 15 | ] 16 | } 17 | ] -------------------------------------------------------------------------------- /internal/testing/testenv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package testing provides testing utilities. 6 | package testing 7 | 8 | import ( 9 | "os" 10 | "os/exec" 11 | "testing" 12 | ) 13 | 14 | // NeedsGoEnv skips t if the current system can't get the environment with 15 | // “go env” in a subprocess. 16 | func NeedsGoEnv(t testing.TB) { 17 | t.Helper() 18 | 19 | if _, err := exec.LookPath("go"); err != nil { 20 | t.Skip("skipping test: can't run go env") 21 | } 22 | } 23 | 24 | // NeedsIntegrationEnv skips t if the underlying test satisfies integration 25 | // requirements. It must be executed in the non-short test mode with an 26 | // appropriate integration environment. 27 | func NeedsIntegrationEnv(t testing.TB) { 28 | t.Helper() 29 | 30 | if os.Getenv("GO_ECOSYSTEM_INTEGRATION_TESTING") != "1" { 31 | t.Skip("skipping; need local test environment with GCS permissions (set GO_ECOSYSTEM_INTEGRATION_TESTING=1)") 32 | } 33 | if testing.Short() { 34 | t.Skip("skipping; integration tests must be run in non-short mode") 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /internal/vulndb/vulndb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package vulndb provides functionality for manipulating 6 | // inputs and outputs of vulndb endpoint. 7 | package vulndb 8 | 9 | import ( 10 | "context" 11 | "time" 12 | 13 | "golang.org/x/pkgsite-metrics/internal/bigquery" 14 | "golang.org/x/pkgsite-metrics/internal/derrors" 15 | "golang.org/x/pkgsite-metrics/internal/osv" 16 | ) 17 | 18 | // Definitions for BigQuery. 19 | 20 | // SchemaVersion changes whenever the BigQuery vulndb schema changes. 21 | var SchemaVersion string 22 | 23 | func init() { 24 | s, err := bigquery.InferSchema(Entry{}) 25 | if err != nil { 26 | panic(err) 27 | } 28 | SchemaVersion = bigquery.SchemaVersion(s) 29 | bigquery.AddTable(TableName, s) 30 | } 31 | 32 | const ( 33 | // Vuln DB requests live in their own dataset that doesn't vary. 34 | // This is the same database the vulnbreqs endpoint uses. 35 | DatasetName = "vulndb" 36 | TableName = "vulndb" 37 | ) 38 | 39 | // Entry is a row stored in a table. It follows the core 40 | // structure of osv.Entry. 41 | type Entry struct { 42 | CreatedAt time.Time `bigquery:"created_at"` 43 | 44 | ModifiedTime time.Time `bigquery:"modified_time"` 45 | PublishedTime time.Time `bigquery:"published_time"` 46 | WithdrawnTime time.Time `bigquery:"withdrawn_time"` 47 | 48 | ID string `bigquery:"id"` 49 | 50 | // Modules can in principle have multiple entries 51 | // with the same path. 52 | Modules []Module `bigquery:"modules"` 53 | } 54 | 55 | func (e *Entry) SetUploadTime(t time.Time) { e.CreatedAt = t } 56 | 57 | // Module plays the role of osv.Affected. The latter also has 58 | // a Module field (among others), but we merge them into one 59 | // type to avoid nesting which can make the queries more complex. 60 | type Module struct { 61 | Path string `bigquery:"path"` 62 | // Ranges field plays the role of osv.Range type 63 | // where “SEMVER” range kind is assumed. 64 | Ranges []Range `bigquery:"ranges"` 65 | } 66 | 67 | // Range plays the role of osv.RangeEvent. That is, it is 68 | // a list of versions representing the ranges in which the 69 | // module is vulnerable. The events should be sorted, and 70 | // MUST represent non-overlapping ranges. 71 | type Range struct { 72 | Introduced string `bigquery:"introduced"` 73 | Fixed string `bigquery:"fixed"` 74 | } 75 | 76 | func Convert(oe *osv.Entry) *Entry { 77 | e := &Entry{ 78 | ID: oe.ID, 79 | ModifiedTime: oe.Modified, 80 | PublishedTime: oe.Published, 81 | Modules: modules(oe), 82 | } 83 | if oe.Withdrawn != nil { 84 | e.WithdrawnTime = *oe.Withdrawn 85 | } 86 | return e 87 | } 88 | 89 | func modules(oe *osv.Entry) []Module { 90 | var modules []Module 91 | for _, a := range oe.Affected { 92 | modules = append(modules, Module{ 93 | Path: a.Module.Path, 94 | Ranges: ranges(a), 95 | }) 96 | } 97 | return modules 98 | } 99 | 100 | func ranges(a osv.Affected) []Range { 101 | var rs []Range 102 | for _, r := range a.Ranges { 103 | for _, e := range r.Events { 104 | rs = append(rs, Range{ 105 | Introduced: e.Introduced, 106 | Fixed: e.Fixed, 107 | }) 108 | } 109 | } 110 | return rs 111 | } 112 | 113 | // ReadMostRecentDB returns entries from the table that reflect the 114 | // most recent state of the vulnerability database at c. 115 | func ReadMostRecentDB(ctx context.Context, c *bigquery.Client) (entries []*Entry, err error) { 116 | defer derrors.Wrap(&err, "ReadMostRecentDB") 117 | 118 | // The server does not create vulndb table since it lives 119 | // in a different dataset. Hence, one could get an error 120 | // accessing it if it was not created in the past. 121 | if _, err := c.CreateOrUpdateTable(ctx, TableName); err != nil { 122 | return nil, err 123 | } 124 | 125 | query := bigquery.PartitionQuery{ 126 | From: "`" + c.FullTableName(TableName) + "`", 127 | PartitionOn: "ID", 128 | OrderBy: "modified_time DESC", 129 | }.String() 130 | iter, err := c.Query(ctx, query) 131 | if err != nil { 132 | return nil, err 133 | } 134 | err = bigquery.ForEachRow(iter, func(e *Entry) bool { 135 | entries = append(entries, e) 136 | return true 137 | }) 138 | if err != nil { 139 | return nil, err 140 | } 141 | return entries, nil 142 | } 143 | -------------------------------------------------------------------------------- /internal/vulndb/vulndb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package vulndb 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | "time" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "golang.org/x/pkgsite-metrics/internal/bigquery" 14 | "golang.org/x/pkgsite-metrics/internal/osv" 15 | test "golang.org/x/pkgsite-metrics/internal/testing" 16 | ) 17 | 18 | func TestConvert(t *testing.T) { 19 | oe := &osv.Entry{ 20 | ID: "a", 21 | Affected: []osv.Affected{ 22 | {Module: osv.Module{Path: "example.mod/a"}, Ranges: []osv.Range{{Events: []osv.RangeEvent{{Introduced: "0"}, {Fixed: "0.9.0"}}}}}, 23 | {Module: osv.Module{Path: "a.example.mod/a"}, Ranges: []osv.Range{{Events: []osv.RangeEvent{{Introduced: "1.0.0"}, {Fixed: "2.0.0"}}}}}, 24 | }} 25 | want := &Entry{ 26 | ID: "a", 27 | Modules: []Module{ 28 | { 29 | Path: "example.mod/a", 30 | Ranges: []Range{{Introduced: "0"}, {Fixed: "0.9.0"}}, 31 | }, 32 | { 33 | Path: "a.example.mod/a", 34 | Ranges: []Range{{Introduced: "1.0.0"}, {Fixed: "2.0.0"}}, 35 | }, 36 | }, 37 | } 38 | got := Convert(oe) 39 | if diff := cmp.Diff(want, got); diff != "" { 40 | t.Fatalf("mismatch (-want, +got):\n%s", diff) 41 | } 42 | } 43 | 44 | func TestReadMostRecentDB(t *testing.T) { 45 | test.NeedsIntegrationEnv(t) 46 | 47 | ctx := context.Background() 48 | const projectID = "go-ecosystem" 49 | 50 | client, err := bigquery.NewClientForTesting(ctx, projectID, "read_recent_db") 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | defer client.Close() 55 | 56 | writeToBigQuery := func(es []*Entry) { 57 | if _, err := client.CreateOrUpdateTable(ctx, TableName); err != nil { 58 | t.Fatal(err) 59 | } 60 | if err := bigquery.UploadMany(ctx, client, TableName, es, 0); err != nil { 61 | t.Fatal(err) 62 | } 63 | } 64 | 65 | lmt := time.Now() 66 | es := []*Entry{ 67 | {ID: "A"}, 68 | {ID: "A", ModifiedTime: lmt}, 69 | {ID: "B", ModifiedTime: lmt}, 70 | } 71 | writeToBigQuery(es) 72 | 73 | got, err := ReadMostRecentDB(ctx, client) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | if len(got) != 2 { 78 | t.Fatalf("want 2 rows; got %d", len(got)) 79 | } 80 | for _, e := range got { 81 | // Ideally, we would check lmt != e.ModifiedTime but 82 | // unmarshaling time.Time introduces some nanosecond 83 | // level imprecision. Instead, we just check that it 84 | // is actually set. 85 | if e.ModifiedTime.IsZero() { 86 | t.Fatalf("want last modified time %v; got %v", lmt, e.ModifiedTime) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /internal/vulndbreqs/bq.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package vulndbreqs 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "time" 11 | 12 | "cloud.google.com/go/civil" 13 | "golang.org/x/pkgsite-metrics/internal/bigquery" 14 | "golang.org/x/pkgsite-metrics/internal/derrors" 15 | ) 16 | 17 | const ( 18 | // Vuln DB requests live in their own dataset that doesn't vary. 19 | DatasetName = "vulndb" 20 | RequestCountTableName = "requests" 21 | IPRequestCountTableName = "ip-requests" 22 | ) 23 | 24 | func init() { 25 | s, err := bigquery.InferSchema(RequestCount{}) 26 | if err != nil { 27 | panic(err) 28 | } 29 | bigquery.AddTable(RequestCountTableName, s) 30 | s, err = bigquery.InferSchema(IPRequestCount{}) 31 | if err != nil { 32 | panic(err) 33 | } 34 | bigquery.AddTable(IPRequestCountTableName, s) 35 | } 36 | 37 | // RequestCount holds the number of requests made on a date. 38 | type RequestCount struct { 39 | CreatedAt time.Time `bigquery:"created_at"` 40 | Date civil.Date `bigquery:"date"` // year-month-day without a timezone 41 | Count int `bigquery:"count"` 42 | } 43 | 44 | // SetUploadTime is used by Client.Upload. 45 | func (r *RequestCount) SetUploadTime(t time.Time) { r.CreatedAt = t } 46 | 47 | // IPRequestCount holds the number of requests for a single IP on a date. 48 | type IPRequestCount struct { 49 | CreatedAt time.Time `bigquery:"created_at"` 50 | Date civil.Date `bigquery:"date"` // year-month-day without a timezone 51 | IP string `bigquery:"ip"` // obfuscated IP address 52 | Count int `bigquery:"count"` 53 | } 54 | 55 | // SetUploadTime is used by Client.Upload. 56 | func (r *IPRequestCount) SetUploadTime(t time.Time) { r.CreatedAt = t } 57 | 58 | // writeToBigQuery writes request counts to BigQuery. 59 | func writeToBigQuery(ctx context.Context, client *bigquery.Client, rcs []*RequestCount, ircs []*IPRequestCount) (err error) { 60 | defer derrors.Wrap(&err, "vulndbreqs.writeToBigQuery") 61 | if _, err := client.CreateOrUpdateTable(ctx, RequestCountTableName); err != nil { 62 | return err 63 | } 64 | if err := bigquery.UploadMany(ctx, client, RequestCountTableName, rcs, 100); err != nil { 65 | return err 66 | } 67 | if _, err := client.CreateOrUpdateTable(ctx, IPRequestCountTableName); err != nil { 68 | return err 69 | } 70 | return bigquery.UploadMany(ctx, client, IPRequestCountTableName, ircs, 100) 71 | } 72 | 73 | // ReadRequestCountsFromBigQuery returns daily counts for requests to the vuln DB, most recent first. 74 | func ReadRequestCountsFromBigQuery(ctx context.Context, client *bigquery.Client) (_ []*RequestCount, err error) { 75 | defer derrors.Wrap(&err, "readFromBigQuery") 76 | // Select the most recently inserted row for each date. 77 | q := fmt.Sprintf("(%s) ORDER BY date DESC", bigquery.PartitionQuery{ 78 | From: "`" + client.FullTableName(RequestCountTableName) + "`", 79 | PartitionOn: "date", 80 | OrderBy: "created_at DESC", 81 | }) 82 | iter, err := client.Query(ctx, q) 83 | if err != nil { 84 | return nil, err 85 | } 86 | return bigquery.All[RequestCount](iter) 87 | } 88 | -------------------------------------------------------------------------------- /internal/vulndbreqs/bq_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package vulndbreqs 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | "time" 11 | 12 | "cloud.google.com/go/civil" 13 | "github.com/google/go-cmp/cmp" 14 | "github.com/google/go-cmp/cmp/cmpopts" 15 | "golang.org/x/exp/slices" 16 | "golang.org/x/pkgsite-metrics/internal/bigquery" 17 | test "golang.org/x/pkgsite-metrics/internal/testing" 18 | ) 19 | 20 | func TestBigQuery(t *testing.T) { 21 | test.NeedsIntegrationEnv(t) 22 | 23 | must := func(err error) { 24 | t.Helper() 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | } 29 | 30 | ctx := context.Background() 31 | const projectID = "go-ecosystem" 32 | 33 | client, err := bigquery.NewClientForTesting(ctx, projectID, "bigquery") 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | defer client.Close() 38 | 39 | date := func(y, m, d int) civil.Date { 40 | return civil.Date{Year: y, Month: time.Month(m), Day: d} 41 | } 42 | 43 | counts := []*IPRequestCount{ 44 | {Date: date(2022, 10, 1), IP: "A", Count: 1}, 45 | {Date: date(2022, 10, 3), IP: "B", Count: 3}, 46 | {Date: date(2022, 10, 4), IP: "C", Count: 4}, 47 | } 48 | must(writeToBigQuery(ctx, client, sumRequestCounts(counts), counts)) 49 | // Insert duplicates with a later time; we expect to get these, not the originals. 50 | time.Sleep(50 * time.Millisecond) 51 | for _, row := range counts { 52 | row.Count++ 53 | } 54 | want := sumRequestCounts(counts) 55 | must(writeToBigQuery(ctx, client, want, counts)) 56 | 57 | got, err := ReadRequestCountsFromBigQuery(ctx, client) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | slices.SortFunc(want, func(c1, c2 *RequestCount) bool { return c1.Date.After(c2.Date) }) 62 | if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(RequestCount{}, "CreatedAt")); diff != "" { 63 | t.Errorf("mismatch (-want, +got):\n%s", diff) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /internal/vulndbreqs/iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package vulndbreqs 6 | 7 | import ( 8 | "context" 9 | "time" 10 | 11 | "cloud.google.com/go/logging" 12 | "cloud.google.com/go/logging/logadmin" 13 | "golang.org/x/pkgsite-metrics/internal/log" 14 | "google.golang.org/api/iterator" 15 | "google.golang.org/grpc/codes" 16 | "google.golang.org/grpc/status" 17 | ) 18 | 19 | // entryIterator wraps a logadmin.EntryIterator to handle quota limits. 20 | // When it sees a ResourceExhausted error, it waits a few seconds to 21 | // get more quota. 22 | type entryIterator struct { 23 | ctx context.Context 24 | client *logadmin.Client 25 | filter string 26 | it *logadmin.EntryIterator 27 | count int 28 | token string 29 | } 30 | 31 | func newEntryIterator(ctx context.Context, client *logadmin.Client, filter string) *entryIterator { 32 | return &entryIterator{ctx: ctx, client: client, filter: filter} 33 | } 34 | 35 | func (it *entryIterator) Next() (*logging.Entry, error) { 36 | for { 37 | if it.it == nil { 38 | it.it = it.client.Entries(it.ctx, logadmin.Filter(it.filter)) 39 | pi := it.it.PageInfo() 40 | // Using a large page size results in fewer requests to the logging API. 41 | // 1000 is the maximum allowed. 42 | pi.MaxSize = 1000 43 | // If we remembered a page token, start the iterator with it. 44 | // See [google.golang.org/api/iterator.PageInfo]. 45 | if it.token != "" { 46 | pi.Token = it.token 47 | } 48 | it.count = 0 49 | } 50 | entry, err := it.it.Next() 51 | if err == iterator.Done { 52 | return nil, err 53 | } 54 | if s, ok := status.FromError(err); ok && s.Code() == codes.ResourceExhausted { 55 | // We ran out of quota. Wait a little and try again. 56 | log.Infof(it.ctx, "entryIterator: got ResourceExhausted after reading %d entries, sleeping...:\n%v", it.count, err) 57 | time.Sleep(10 * time.Second) 58 | log.Infof(it.ctx, "entryIterator: retrying") 59 | it.token = it.it.PageInfo().Token 60 | // We can't continue with this iterator, so create a new one at the 61 | // top of the loop. 62 | it.it = nil 63 | continue 64 | } 65 | if err != nil { 66 | return nil, err 67 | } 68 | it.count++ 69 | return entry, nil 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/worker/enqueue.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "sync" 10 | 11 | "golang.org/x/pkgsite-metrics/internal/config" 12 | "golang.org/x/pkgsite-metrics/internal/derrors" 13 | "golang.org/x/pkgsite-metrics/internal/log" 14 | "golang.org/x/pkgsite-metrics/internal/pkgsitedb" 15 | "golang.org/x/pkgsite-metrics/internal/queue" 16 | "golang.org/x/pkgsite-metrics/internal/scan" 17 | ) 18 | 19 | const defaultMinImportedByCount = 10 20 | 21 | func readModules(ctx context.Context, cfg *config.Config, file string, minImpCount int) ([]scan.ModuleSpec, error) { 22 | if file != "" { 23 | log.Infof(ctx, "reading modules from file %s", file) 24 | return scan.ParseCorpusFile(file, minImpCount) 25 | } 26 | log.Infof(ctx, "reading modules from DB %s", cfg.PkgsiteDBName) 27 | return readFromDB(ctx, cfg, minImpCount) 28 | } 29 | 30 | func readFromDB(ctx context.Context, cfg *config.Config, minImportedByCount int) ([]scan.ModuleSpec, error) { 31 | db, err := pkgsitedb.Open(ctx, cfg) 32 | if err != nil { 33 | return nil, err 34 | } 35 | defer db.Close() 36 | return pkgsitedb.ModuleSpecs(ctx, db, minImportedByCount) 37 | } 38 | 39 | func enqueueTasks(ctx context.Context, tasks []queue.Task, q queue.Queue, opts *queue.Options) (err error) { 40 | defer derrors.Wrap(&err, "enqueueTasks") 41 | 42 | // Enqueue concurrently, because sequentially takes a while. 43 | const concurrentEnqueues = 20 44 | var ( 45 | mu sync.Mutex 46 | nEnqueued, nErrors int 47 | ) 48 | sem := make(chan struct{}, concurrentEnqueues) 49 | 50 | for _, sreq := range tasks { 51 | log.Infof(ctx, "enqueuing: %s?%s", sreq.Path(), sreq.Params()) 52 | sreq := sreq 53 | sem <- struct{}{} 54 | go func() { 55 | defer func() { <-sem }() 56 | enqueued, err := q.EnqueueScan(ctx, sreq, opts) 57 | mu.Lock() 58 | if err != nil { 59 | log.Errorf(ctx, err, "enqueuing") 60 | nErrors++ 61 | } else if enqueued { 62 | nEnqueued++ 63 | } 64 | mu.Unlock() 65 | }() 66 | } 67 | // Wait for goroutines to finish. 68 | for i := 0; i < concurrentEnqueues; i++ { 69 | sem <- struct{}{} 70 | } 71 | log.Infof(ctx, "Successfully scheduled modules to be fetched: %d modules enqueued, %d errors", nEnqueued, nErrors) 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /internal/worker/govulncheck.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | 14 | "golang.org/x/pkgsite-metrics/internal" 15 | "golang.org/x/pkgsite-metrics/internal/derrors" 16 | "golang.org/x/pkgsite-metrics/internal/govulncheck" 17 | "golang.org/x/pkgsite-metrics/internal/log" 18 | ) 19 | 20 | type GovulncheckServer struct { 21 | *Server 22 | workVersion *govulncheck.WorkVersion 23 | } 24 | 25 | func newGovulncheckServer(s *Server) *GovulncheckServer { 26 | return &GovulncheckServer{Server: s} 27 | } 28 | 29 | func (h *GovulncheckServer) getWorkVersion(ctx context.Context) (_ *govulncheck.WorkVersion, err error) { 30 | defer derrors.Wrap(&err, "GovulncheckServer.getWorkVersion") 31 | h.mu.Lock() 32 | defer h.mu.Unlock() 33 | 34 | if h.workVersion == nil { 35 | lmt, err := dbLastModified(h.cfg.VulnDBDir) 36 | if err != nil { 37 | return nil, err 38 | } 39 | goEnv, err := internal.GoEnv() 40 | if err != nil { 41 | return nil, err 42 | } 43 | h.workVersion = &govulncheck.WorkVersion{ 44 | GoVersion: goEnv["GOVERSION"], 45 | VulnDBLastModified: lmt, 46 | WorkerVersion: h.cfg.VersionID, 47 | SchemaVersion: govulncheck.SchemaVersion, 48 | } 49 | log.Infof(ctx, "govulncheck work version: %+v", h.workVersion) 50 | } 51 | return h.workVersion, nil 52 | } 53 | 54 | // dbLastModified computes the last modified time stamp of 55 | // vulnerability database rooted at vulnDB. 56 | // 57 | // Follows the logic of golang.org/x/internal/client/client.go:Client.LastModifiedTime. 58 | func dbLastModified(vulnDB string) (time.Time, error) { 59 | dbFile := filepath.Join(vulnDB, "index/db.json") 60 | b, err := os.ReadFile(dbFile) 61 | if err != nil { 62 | return time.Time{}, err 63 | } 64 | 65 | // dbMeta contains metadata about the database itself. 66 | // 67 | // Copy of golang.org/x/internal/client/schema.go:dbMeta. 68 | type dbMeta struct { 69 | // Modified is the time the database was last modified, calculated 70 | // as the most recent time any single OSV entry was modified. 71 | Modified time.Time `json:"modified"` 72 | } 73 | 74 | var dbm dbMeta 75 | if err := json.Unmarshal(b, &dbm); err != nil { 76 | return time.Time{}, err 77 | } 78 | 79 | return dbm.Modified, nil 80 | } 81 | -------------------------------------------------------------------------------- /internal/worker/govulncheck_enqueue.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "net/http" 12 | "sort" 13 | "strings" 14 | 15 | "golang.org/x/pkgsite-metrics/internal/config" 16 | "golang.org/x/pkgsite-metrics/internal/derrors" 17 | "golang.org/x/pkgsite-metrics/internal/govulncheck" 18 | "golang.org/x/pkgsite-metrics/internal/queue" 19 | "golang.org/x/pkgsite-metrics/internal/scan" 20 | ) 21 | 22 | // handleEnqueue enqueues multiple modules for a single govulncheck mode. 23 | func (h *GovulncheckServer) handleEnqueue(w http.ResponseWriter, r *http.Request) error { 24 | return h.enqueue(r, false) 25 | } 26 | 27 | // handleEnqueueAll enqueues multiple modules for all govulncheck modes. 28 | func (h *GovulncheckServer) handleEnqueueAll(w http.ResponseWriter, r *http.Request) error { 29 | return h.enqueue(r, true) 30 | } 31 | 32 | func (h *GovulncheckServer) enqueue(r *http.Request, allModes bool) error { 33 | ctx := r.Context() 34 | params := &govulncheck.EnqueueQueryParams{Min: defaultMinImportedByCount} 35 | if err := scan.ParseParams(r, params); err != nil { 36 | return fmt.Errorf("%w: %v", derrors.InvalidArgument, err) 37 | } 38 | modes, err := listModes(params.Mode, allModes) 39 | if err != nil { 40 | return fmt.Errorf("%w: %v", derrors.InvalidArgument, err) 41 | } 42 | tasks, err := createGovulncheckQueueTasks(ctx, h.cfg, params, modes) 43 | if err != nil { 44 | return err 45 | } 46 | return enqueueTasks(ctx, tasks, h.queue, 47 | &queue.Options{Namespace: "govulncheck", TaskNameSuffix: params.Suffix}) 48 | } 49 | 50 | // listModes lists all applicable modes depending on who called it. If enqueue did (allModes=false), 51 | // returns only valid modeParam. If enqueueAll did (allModes=true), returns modes that enqueueAll 52 | // supports, which are modes/{ModeCompare}. 53 | func listModes(modeParam string, allModes bool) ([]string, error) { 54 | if allModes { 55 | if modeParam != "" { 56 | return nil, errors.New("mode query param provided for enqueueAll") 57 | } 58 | var ms []string 59 | for k := range modes { 60 | // Don't add ModeCompare to enqueueAll (it's something we only want to run occasionally) 61 | if k != ModeCompare { 62 | ms = append(ms, k) 63 | } 64 | } 65 | sort.Strings(ms) // make deterministic for testing 66 | return ms, nil 67 | } 68 | mode, err := govulncheckMode(modeParam) 69 | if err != nil { 70 | return nil, err 71 | } 72 | return []string{mode}, nil 73 | } 74 | 75 | func createGovulncheckQueueTasks(ctx context.Context, cfg *config.Config, params *govulncheck.EnqueueQueryParams, modes []string) (_ []queue.Task, err error) { 76 | defer derrors.Wrap(&err, "createGovulncheckQueueTasks(%v)", modes) 77 | var ( 78 | tasks []queue.Task 79 | modspecs []scan.ModuleSpec 80 | ) 81 | for _, mode := range modes { 82 | if modspecs == nil { 83 | modspecs, err = readModules(ctx, cfg, params.File, params.Min) 84 | if err != nil { 85 | return nil, err 86 | } 87 | } 88 | reqs := moduleSpecsToGovulncheckScanRequests(modspecs, mode) 89 | for _, req := range reqs { 90 | if req.Module != "std" { // ignore the standard library 91 | tasks = append(tasks, req) 92 | } 93 | } 94 | } 95 | return tasks, nil 96 | } 97 | 98 | func moduleSpecsToGovulncheckScanRequests(modspecs []scan.ModuleSpec, mode string) []*govulncheck.Request { 99 | var sreqs []*govulncheck.Request 100 | for _, ms := range modspecs { 101 | sreqs = append(sreqs, &govulncheck.Request{ 102 | ModuleURLPath: scan.ModuleURLPath{ 103 | Module: ms.Path, 104 | Version: ms.Version, 105 | }, 106 | QueryParams: govulncheck.QueryParams{ 107 | ImportedBy: ms.ImportedBy, 108 | Mode: mode, 109 | }, 110 | }) 111 | } 112 | return sreqs 113 | } 114 | 115 | func govulncheckMode(mode string) (string, error) { 116 | if mode == "" { 117 | // ModeGovulncheck is the default mode. 118 | return ModeGovulncheck, nil 119 | } 120 | mode = strings.ToUpper(mode) 121 | if _, ok := modes[mode]; !ok { 122 | return "", fmt.Errorf("unsupported mode: %v", mode) 123 | } 124 | return mode, nil 125 | } 126 | -------------------------------------------------------------------------------- /internal/worker/govulncheck_enqueue_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "golang.org/x/pkgsite-metrics/internal/config" 14 | "golang.org/x/pkgsite-metrics/internal/govulncheck" 15 | "golang.org/x/pkgsite-metrics/internal/queue" 16 | "golang.org/x/pkgsite-metrics/internal/scan" 17 | ) 18 | 19 | func TestCreateQueueTasks(t *testing.T) { 20 | vreq := func(path, version, mode string, importedBy int) *govulncheck.Request { 21 | return &govulncheck.Request{ 22 | ModuleURLPath: scan.ModuleURLPath{Module: path, Version: version}, 23 | QueryParams: govulncheck.QueryParams{Mode: mode, ImportedBy: importedBy}, 24 | } 25 | } 26 | 27 | params := &govulncheck.EnqueueQueryParams{Min: 8, File: "testdata/modules.txt"} 28 | gotTasks, err := createGovulncheckQueueTasks(context.Background(), &config.Config{}, params, []string{ModeGovulncheck}) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | wantTasks := []queue.Task{ 34 | vreq("github.com/pkg/errors", "v0.9.1", ModeGovulncheck, 10), 35 | vreq("golang.org/x/net", "v0.4.0", ModeGovulncheck, 20), 36 | } 37 | if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(govulncheck.Request{})); diff != "" { 38 | t.Errorf("mismatch (-want, +got):\n%s", diff) 39 | } 40 | 41 | allModes, err := listModes("", true) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | gotTasks, err = createGovulncheckQueueTasks(context.Background(), &config.Config{}, params, allModes) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | // cfg.BinaryBucket is empty, so no binary-mode tasks are created. 50 | wantTasks = []queue.Task{ 51 | vreq("github.com/pkg/errors", "v0.9.1", ModeGovulncheck, 10), 52 | vreq("golang.org/x/net", "v0.4.0", ModeGovulncheck, 20), 53 | } 54 | 55 | if diff := cmp.Diff(wantTasks, gotTasks, cmp.AllowUnexported(govulncheck.Request{})); diff != "" { 56 | t.Errorf("mismatch (-want, +got):\n%s", diff) 57 | } 58 | } 59 | 60 | func TestListModes(t *testing.T) { 61 | for _, test := range []struct { 62 | param string 63 | all bool 64 | want []string 65 | wantErr bool 66 | }{ 67 | {"", true, []string{ModeGovulncheck}, false}, 68 | {"", false, []string{ModeGovulncheck}, false}, 69 | {"imports", true, nil, true}, 70 | } { 71 | t.Run(fmt.Sprintf("%q,%t", test.param, test.all), func(t *testing.T) { 72 | got, err := listModes(test.param, test.all) 73 | if err != nil && !test.wantErr { 74 | t.Fatal(err) 75 | } 76 | if err == nil && !cmp.Equal(got, test.want) { 77 | t.Errorf("got %v, want %v", got, test.want) 78 | } 79 | }) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /internal/worker/govulncheck_scan_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "io" 11 | "path/filepath" 12 | "runtime" 13 | "strings" 14 | "testing" 15 | 16 | "golang.org/x/pkgsite-metrics/internal/buildtest" 17 | "golang.org/x/pkgsite-metrics/internal/govulncheck" 18 | "golang.org/x/pkgsite-metrics/internal/govulncheckapi" 19 | ) 20 | 21 | func TestAsScanError(t *testing.T) { 22 | check := func(err error, want bool) { 23 | if got := errors.As(err, new(scanError)); got != want { 24 | t.Errorf("%T: got %t, want %t", err, got, want) 25 | } 26 | } 27 | check(io.EOF, false) 28 | check(scanError{io.EOF}, true) 29 | } 30 | 31 | func TestVulnsForMode(t *testing.T) { 32 | findings := []*govulncheckapi.Finding{ 33 | {Trace: []*govulncheckapi.Frame{{Module: "M1", Package: "P1", Function: "F1"}}}, 34 | {Trace: []*govulncheckapi.Frame{{Module: "M1", Package: "P1"}}}, 35 | {Trace: []*govulncheckapi.Frame{{Module: "M1"}}}, 36 | {Trace: []*govulncheckapi.Frame{{Module: "M2"}}}, 37 | } 38 | 39 | vulnsStr := func(vulns []*govulncheck.Vuln) string { 40 | var vs []string 41 | for _, v := range vulns { 42 | vs = append(vs, fmt.Sprintf("%s:%s", v.ModulePath, v.PackagePath)) 43 | } 44 | return strings.Join(vs, ", ") 45 | } 46 | 47 | for _, tc := range []struct { 48 | mode string 49 | want string 50 | }{ 51 | {scanModeSourceSymbol, "M1:P1"}, 52 | {scanModeSourcePackage, "M1:P1"}, 53 | {scanModeSourceModule, "M1:, M2:"}, 54 | } { 55 | tc := tc 56 | t.Run(tc.mode, func(t *testing.T) { 57 | vs := vulnsForScanMode(&govulncheck.AnalysisResponse{Findings: findings}, tc.mode) 58 | if got := vulnsStr(vs); got != tc.want { 59 | t.Errorf("got %s; want %s", got, tc.want) 60 | } 61 | }) 62 | } 63 | } 64 | 65 | func TestUnrecoverableError(t *testing.T) { 66 | for _, e := range []struct { 67 | ec string 68 | want bool 69 | }{ 70 | {"LOAD", true}, 71 | {"MISC", false}, 72 | {"BIGQUERY", false}, 73 | } { 74 | if got := unrecoverableError(e.ec); got != e.want { 75 | t.Errorf("want %t for %s; got %t", e.want, e.ec, got) 76 | } 77 | } 78 | } 79 | 80 | // TODO: can we have a test for sandbox? We do test the sandbox 81 | // and unmarshalling in cmd/govulncheck_sandbox, so what would be 82 | // left here is checking that runsc is initiated properly. It is 83 | // not clear how to do that here nor is it necessary. 84 | func TestRunScanModuleInsecure(t *testing.T) { 85 | if testing.Short() { 86 | t.Skip("skipping test that uses internet in short mode") 87 | } 88 | 89 | govulncheckPath, err := buildtest.BuildGovulncheck(t.TempDir()) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | 94 | vulndb, err := filepath.Abs("../testdata/vulndb") 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | s := &scanner{insecure: true, govulncheckPath: govulncheckPath, vulnDBDir: vulndb} 100 | 101 | response, err := s.runGovulncheckScanInsecure("../testdata/module", ModeGovulncheck) 102 | if err != nil { 103 | t.Fatal(err) 104 | } 105 | findings := response.Findings 106 | wantID := "GO-2021-0113" 107 | found := false 108 | for _, v := range findings { 109 | if v.OSV == wantID { 110 | found = true 111 | break 112 | } 113 | } 114 | if !found { 115 | t.Errorf("want %s, did not find it in %d vulns", wantID, len(findings)) 116 | } 117 | 118 | stats := response.Stats 119 | if got := stats.ScanSeconds; got <= 0 { 120 | t.Errorf("scan time not collected or negative: %v", got) 121 | } 122 | if got := stats.ScanMemory; got <= 0 && runtime.GOOS == "linux" { 123 | t.Errorf("scan memory not collected or negative: %v", got) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /internal/worker/jobs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Handlers for jobs. 6 | // 7 | // jobs/describe?jobid=xxx describe a job 8 | // jobs/list list all jobs 9 | // jobs/cancel?jobid=xxx cancel a job 10 | // jobs/results?jobid=xxx&errors={true|false} get job results 11 | 12 | package worker 13 | 14 | import ( 15 | "bytes" 16 | "context" 17 | "encoding/json" 18 | "errors" 19 | "fmt" 20 | "io" 21 | "net/http" 22 | "strings" 23 | "time" 24 | 25 | "golang.org/x/pkgsite-metrics/internal/analysis" 26 | "golang.org/x/pkgsite-metrics/internal/derrors" 27 | "golang.org/x/pkgsite-metrics/internal/jobs" 28 | ) 29 | 30 | func (s *Server) handleJobs(w http.ResponseWriter, r *http.Request) (err error) { 31 | defer derrors.Wrap(&err, "Server.handleJobs") 32 | ctx := r.Context() 33 | 34 | if s.jobDB == nil { 35 | return &serverError{err: errors.New("jobs DB not configured"), status: http.StatusNotImplemented} 36 | } 37 | 38 | jobID := r.FormValue("jobid") 39 | errs := r.FormValue("errors") // for results 40 | return s.processJobRequest(ctx, w, r.URL.Path, jobID, errs, s.jobDB) 41 | } 42 | 43 | type jobDB interface { 44 | CreateJob(ctx context.Context, j *jobs.Job) error 45 | GetJob(ctx context.Context, id string) (*jobs.Job, error) 46 | UpdateJob(ctx context.Context, id string, f func(*jobs.Job) error) error 47 | ListJobs(context.Context, func(*jobs.Job, time.Time) error) error 48 | } 49 | 50 | func (s *Server) processJobRequest(ctx context.Context, w io.Writer, path, jobID, errs string, db jobDB) error { 51 | path = strings.TrimPrefix(path, "/jobs/") 52 | switch path { 53 | case "describe": // describe one job 54 | if jobID == "" { 55 | return fmt.Errorf("missing jobid: %w", derrors.InvalidArgument) 56 | } 57 | job, err := db.GetJob(ctx, jobID) 58 | if err != nil { 59 | return err 60 | } 61 | return writeJSON(w, job) 62 | 63 | case "cancel": 64 | if jobID == "" { 65 | return fmt.Errorf("missing jobid: %w", derrors.InvalidArgument) 66 | } 67 | return db.UpdateJob(ctx, jobID, func(j *jobs.Job) error { 68 | j.Canceled = true 69 | return nil 70 | }) 71 | 72 | case "list": 73 | var joblist []*jobs.Job 74 | err := db.ListJobs(ctx, func(j *jobs.Job, _ time.Time) error { 75 | joblist = append(joblist, j) 76 | return nil 77 | }) 78 | if err != nil { 79 | return err 80 | } 81 | return writeJSON(w, joblist) 82 | 83 | case "results": 84 | if jobID == "" { 85 | return fmt.Errorf("missing jobid: %w", derrors.InvalidArgument) 86 | } 87 | job, err := db.GetJob(ctx, jobID) 88 | if err != nil { 89 | return err 90 | } 91 | if s.bqClient == nil { 92 | return errors.New("bq client is nil") 93 | } 94 | results, err := analysis.ReadResults(ctx, s.bqClient, job.Binary, job.BinaryVersion, job.BinaryArgs, errs) 95 | if err != nil { 96 | return err 97 | } 98 | return writeJSON(w, results) 99 | 100 | default: 101 | return fmt.Errorf("unknown path %q: %w", path, derrors.InvalidArgument) 102 | } 103 | } 104 | 105 | // writeJSON JSON-marshals v and writes it to w. 106 | // Marshal failures do not result in partial writes. 107 | func writeJSON(w io.Writer, v any) error { 108 | var buf bytes.Buffer 109 | enc := json.NewEncoder(&buf) 110 | enc.SetIndent("", " ") 111 | if err := enc.Encode(v); err != nil { 112 | return err 113 | } 114 | _, err := w.Write(buf.Bytes()) 115 | return err 116 | } 117 | -------------------------------------------------------------------------------- /internal/worker/jobs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "encoding/json" 11 | "fmt" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/google/go-cmp/cmp" 17 | "golang.org/x/exp/maps" 18 | "golang.org/x/exp/slices" 19 | "golang.org/x/pkgsite-metrics/internal/derrors" 20 | "golang.org/x/pkgsite-metrics/internal/jobs" 21 | ) 22 | 23 | func TestJobs(t *testing.T) { 24 | ctx := context.Background() 25 | db := &testJobDB{map[string]*jobs.Job{}} 26 | tm := time.Date(2023, 3, 11, 1, 2, 3, 0, time.UTC) 27 | job := jobs.NewJob("user", tm, "url", "bin", "", "args go here") 28 | if err := db.CreateJob(ctx, job); err != nil { 29 | t.Fatal(err) 30 | } 31 | s := &Server{} 32 | var buf bytes.Buffer 33 | if err := s.processJobRequest(ctx, &buf, "/jobs/describe", job.ID(), "false", db); err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | var got jobs.Job 38 | if err := json.Unmarshal(buf.Bytes(), &got); err != nil { 39 | t.Fatal(err) 40 | } 41 | if !cmp.Equal(&got, job) { 42 | t.Errorf("got\n%+v\nwant\n%+v", got, job) 43 | } 44 | 45 | if err := s.processJobRequest(ctx, &buf, "/jobs/cancel", job.ID(), "false", db); err != nil { 46 | t.Fatal(err) 47 | } 48 | 49 | got2, err := db.GetJob(ctx, job.ID()) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | if !got2.Canceled { 54 | t.Error("got canceled false, want true") 55 | } 56 | 57 | buf.Reset() 58 | if err := s.processJobRequest(ctx, &buf, "/jobs/list", "", "", db); err != nil { 59 | t.Fatal(err) 60 | } 61 | // Don't check for specific output, just make sure there's something 62 | // that mentions the job user. 63 | got3 := buf.String() 64 | if !strings.Contains(got3, job.User) { 65 | t.Errorf("got\n%q\nwhich does not contain the job user %q", got3, job.User) 66 | } 67 | } 68 | 69 | type testJobDB struct { 70 | jobs map[string]*jobs.Job 71 | } 72 | 73 | func (d *testJobDB) CreateJob(ctx context.Context, j *jobs.Job) error { 74 | id := j.ID() 75 | if _, ok := d.jobs[id]; ok { 76 | return fmt.Errorf("job with id %q exists", id) 77 | } 78 | d.jobs[id] = j 79 | return nil 80 | } 81 | 82 | func (d *testJobDB) DeleteJob(ctx context.Context, id string) error { 83 | delete(d.jobs, id) 84 | return nil 85 | } 86 | 87 | func (d *testJobDB) GetJob(ctx context.Context, id string) (*jobs.Job, error) { 88 | j, ok := d.jobs[id] 89 | if !ok { 90 | return nil, fmt.Errorf("job with id %q: %w", id, derrors.NotFound) 91 | } 92 | // Copy job so a client in the same process can't modify it. 93 | j2 := *j 94 | return &j2, nil 95 | } 96 | 97 | func (d *testJobDB) UpdateJob(ctx context.Context, id string, f func(*jobs.Job) error) error { 98 | j, err := d.GetJob(ctx, id) 99 | if err != nil { 100 | return err 101 | } 102 | if err := f(j); err != nil { 103 | return err 104 | } 105 | d.jobs[id] = j 106 | return nil 107 | } 108 | 109 | func (d *testJobDB) ListJobs(ctx context.Context, f func(*jobs.Job, time.Time) error) error { 110 | jobslice := maps.Values(d.jobs) 111 | // Sort by StartedAt descending. 112 | slices.SortFunc(jobslice, func(j1, j2 *jobs.Job) bool { 113 | return j1.StartedAt.After(j2.StartedAt) 114 | }) 115 | for _, j := range jobslice { 116 | if err := f(j, time.Time{}); err != nil { 117 | return err 118 | } 119 | } 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /internal/worker/scan_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "os" 12 | "testing" 13 | 14 | "golang.org/x/exp/slog" 15 | "golang.org/x/pkgsite-metrics/internal/derrors" 16 | "golang.org/x/pkgsite-metrics/internal/log" 17 | "golang.org/x/pkgsite-metrics/internal/proxy" 18 | test "golang.org/x/pkgsite-metrics/internal/testing" 19 | ) 20 | 21 | func TestPrepareModule(t *testing.T) { 22 | test.NeedsIntegrationEnv(t) 23 | ctx := context.Background() 24 | slog.SetDefault(slog.New(log.NewLineHandler(os.Stderr))) 25 | const insecure = true 26 | proxyClient, err := proxy.New("https://proxy.golang.org/cached-only") 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | for _, test := range []struct { 32 | modulePath, version string 33 | init bool 34 | want error 35 | }{ 36 | // Bad version; proxy should return an error. 37 | {"rsc.io/quote", "x", true, derrors.ProxyError}, 38 | // This module has a go.mod file... 39 | {"rsc.io/quote", "v1.0.0", false, nil}, 40 | // ...so it doesn't matter if we pass true for init. 41 | {"rsc.io/quote", "v1.0.0", true, nil}, 42 | // This module doesn't have a go.mod file... 43 | {"github.com/pkg/errors", "v0.9.1", false, derrors.BadModule}, 44 | // ... but passing init will make it work. 45 | {"github.com/pkg/errors", "v0.9.1", true, nil}, 46 | // This module has a dependency (github.com/decred/blake256) for which 47 | // the proxy returns 404 when fetch is disabled. 48 | {"github.com/decred/gominer", "v1.0.0", true, derrors.BadModule}, 49 | } { 50 | t.Run(fmt.Sprintf("%s@%s,%t", test.modulePath, test.version, test.init), func(t *testing.T) { 51 | dir := t.TempDir() 52 | args := prepareModuleArgs{ 53 | modulePath: test.modulePath, 54 | version: test.version, 55 | dir: dir, 56 | proxyClient: proxyClient, 57 | insecure: insecure, 58 | init: test.init, 59 | } 60 | err := prepareModule(ctx, args) 61 | if !errors.Is(err, test.want) { 62 | t.Errorf("got %v, want %v", err, test.want) 63 | } 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /internal/worker/testdata/analyzer/analyzer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "golang.org/x/tools/go/analysis/passes/findcall" 9 | "golang.org/x/tools/go/analysis/singlechecker" 10 | ) 11 | 12 | func main() { singlechecker.Main(findcall.Analyzer) } 13 | -------------------------------------------------------------------------------- /internal/worker/testdata/module/a.go: -------------------------------------------------------------------------------- 1 | package p 2 | 3 | func Fact(n int) int { 4 | if n == 0 { 5 | return 1 6 | } 7 | return n * Fact(n-1) 8 | } 9 | -------------------------------------------------------------------------------- /internal/worker/testdata/module/go.mod: -------------------------------------------------------------------------------- 1 | module test_module 2 | 3 | -------------------------------------------------------------------------------- /internal/worker/testdata/modules.txt: -------------------------------------------------------------------------------- 1 | std v1.19.4 2025760 2 | github.com/pkg/errors v0.9.1 10 3 | golang.org/x/net v0.4.0 20 4 | 5 | -------------------------------------------------------------------------------- /internal/worker/vulndb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "errors" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | "time" 15 | 16 | "fmt" 17 | "net/http" 18 | 19 | "cloud.google.com/go/storage" 20 | "google.golang.org/api/iterator" 21 | 22 | "golang.org/x/pkgsite-metrics/internal" 23 | "golang.org/x/pkgsite-metrics/internal/bigquery" 24 | "golang.org/x/pkgsite-metrics/internal/derrors" 25 | "golang.org/x/pkgsite-metrics/internal/log" 26 | "golang.org/x/pkgsite-metrics/internal/osv" 27 | "golang.org/x/pkgsite-metrics/internal/vulndb" 28 | "golang.org/x/pkgsite-metrics/internal/vulndbreqs" 29 | ) 30 | 31 | func (s *Server) handleComputeRequests(w http.ResponseWriter, r *http.Request) (err error) { 32 | defer derrors.Wrap(&err, "handleComputeRequests") 33 | 34 | ctx := r.Context() 35 | // Don't use the Server's BigQuery client: it's for the wrong 36 | // dataset. 37 | vClient, err := bigquery.NewClientCreate(ctx, s.cfg.ProjectID, vulndbreqs.DatasetName) 38 | if err != nil { 39 | return err 40 | } 41 | keyName := "projects/" + s.cfg.ProjectID + "/secrets/vulndb-hmac-key" 42 | hmacKey, err := internal.GetSecret(ctx, keyName) 43 | if err != nil { 44 | return err 45 | } 46 | err = vulndbreqs.ComputeAndStore(ctx, s.cfg.VulnDBBucketProjectID, vClient, []byte(hmacKey)) 47 | if err != nil { 48 | return err 49 | } 50 | fmt.Fprintf(w, "Successfully computed and stored request counts.\n") 51 | return nil 52 | } 53 | 54 | func (s *Server) handleVulnDB(w http.ResponseWriter, r *http.Request) (err error) { 55 | defer derrors.Wrap(&err, "handleVulnDB") 56 | 57 | ctx := r.Context() 58 | dbClient, err := bigquery.NewClientCreate(ctx, s.cfg.ProjectID, vulndb.DatasetName) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | c, err := storage.NewClient(ctx) 64 | if err != nil { 65 | return err 66 | } 67 | bucket := c.Bucket("go-vulndb") 68 | if bucket == nil { 69 | return errors.New("failed to create go-vulndb bucket") 70 | } 71 | 72 | lmts, err := lastModified(ctx, dbClient) 73 | if err != nil { 74 | return err 75 | } 76 | entries, err := vulndbEntries(ctx, bucket) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | for _, e := range entries { 82 | lmt, ok := lmts[e.ID] 83 | if ok && e.ModifiedTime.Equal(lmt) { 84 | // Skip adding the entry if nothing has changed in the meantime. 85 | log.Infof(ctx, "skipping entry %s, it has not been modified", e.ID) 86 | continue 87 | } 88 | if err = writeResult(ctx, false, w, dbClient, vulndb.TableName, e); err != nil { 89 | return err 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | 96 | func vulndbEntries(ctx context.Context, bucket *storage.BucketHandle) ([]*vulndb.Entry, error) { 97 | osvEntries, err := allVulnerabilities(ctx, bucket) 98 | if err != nil { 99 | return nil, err 100 | } 101 | var entries []*vulndb.Entry 102 | for _, oe := range osvEntries { 103 | entries = append(entries, vulndb.Convert(oe)) 104 | } 105 | return entries, nil 106 | } 107 | 108 | // gcsOSVPrefix is the directory under which .json 109 | // files with OSV entries are located. 110 | const gcsOSVPrefix = "ID" 111 | 112 | // allVulnerabilities fetches all osv.Entries from GCS bucket located at ID/*.json paths. 113 | func allVulnerabilities(ctx context.Context, bucket *storage.BucketHandle) ([]*osv.Entry, error) { 114 | var entries []*osv.Entry 115 | query := &storage.Query{Prefix: gcsOSVPrefix} 116 | it := bucket.Objects(ctx, query) 117 | for { 118 | attrs, err := it.Next() 119 | if err == iterator.Done { 120 | break 121 | } 122 | if err != nil { 123 | return nil, err 124 | } 125 | // Skip zip files and index.json. 126 | if !strings.HasSuffix(attrs.Name, ".json") || strings.HasSuffix(attrs.Name, "index.json") { 127 | continue 128 | } 129 | 130 | e, err := readEntry(ctx, bucket, attrs.Name) 131 | if err != nil { 132 | return nil, err 133 | } 134 | entries = append(entries, e) 135 | } 136 | return entries, nil 137 | } 138 | 139 | func readEntry(ctx context.Context, bucket *storage.BucketHandle, gcsPath string) (*osv.Entry, error) { 140 | localPath := filepath.Join(os.TempDir(), "binary") 141 | if err := copyToLocalFile(localPath, false, gcsPath, gcsOpenFileFunc(ctx, bucket)); err != nil { 142 | return nil, err 143 | } 144 | js, err := os.ReadFile(localPath) 145 | if err != nil { 146 | return nil, err 147 | } 148 | var entry osv.Entry 149 | if err := json.Unmarshal(js, &entry); err != nil { 150 | return nil, err 151 | } 152 | return &entry, nil 153 | } 154 | 155 | func lastModified(ctx context.Context, c *bigquery.Client) (map[string]time.Time, error) { 156 | es, err := vulndb.ReadMostRecentDB(ctx, c) 157 | if err != nil { 158 | return nil, err 159 | } 160 | m := make(map[string]time.Time) 161 | for _, e := range es { 162 | m[e.ID] = e.ModifiedTime 163 | } 164 | return m, nil 165 | } 166 | -------------------------------------------------------------------------------- /internal/worker/vulndb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package worker 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "cloud.google.com/go/storage" 12 | test "golang.org/x/pkgsite-metrics/internal/testing" 13 | ) 14 | 15 | func TestIntegrationAllVulns(t *testing.T) { 16 | test.NeedsIntegrationEnv(t) 17 | 18 | ctx := context.Background() 19 | c, err := storage.NewClient(ctx) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | bucket := c.Bucket("go-vulndb") 24 | if bucket == nil { 25 | t.Fatal("failed to create go-vulndb bucket") 26 | } 27 | es, err := allVulnerabilities(ctx, bucket) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | if len(es) == 0 { 32 | t.Fatal("want some vulnerabilities; got none") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /terraform/README.md: -------------------------------------------------------------------------------- 1 | # Terraform configuration for pkgsite-metrics 2 | 3 | ## External variables 4 | 5 | Some inputs to this config are not checked into the repo. 6 | You can provide them on the `terraform` command line, 7 | create a `terraform.tfvars` file in this directory, 8 | or use environment variables beginning with `TF_VAR_`. 9 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // This file includes the tools the pkgsite-metrics depend on. 6 | // It is never built as it is hidden behind the tools build tag, 7 | // but the modules containing the imported paths below become a 8 | // requirement of the pkgsite-metrics module. That means go mod 9 | // download will fetch them and all of their dependencies. This 10 | // in turn means we can build these dependencies without further 11 | // interaction with the proxy or making any network requests in 12 | // general. This is useful, for instance, for testing in CI 13 | // integrations where network connection is limited. 14 | 15 | //go:build tools 16 | 17 | package main 18 | 19 | import ( 20 | _ "github.com/client9/misspell/cmd/misspell" 21 | _ "golang.org/x/vuln/cmd/govulncheck" 22 | _ "honnef.co/go/tools/cmd/staticcheck" 23 | _ "mvdan.cc/unparam" 24 | ) 25 | --------------------------------------------------------------------------------