├── .gitignore ├── cmd ├── doc.go ├── cmd_suite_test.go ├── defaults.go ├── buildinfo.go ├── run_test.go ├── crashd.go └── run.go ├── archiver ├── doc.go └── tarrer.go ├── k8s ├── doc.go ├── k8s.go ├── k8s_suite_test.go ├── secret.go ├── bastion.go ├── main_test.go ├── cluster.go ├── kube_config.go ├── container.go ├── nodes.go ├── container_logger.go ├── object_writer.go ├── client_test.go ├── search_result.go ├── executor.go ├── search_params.go ├── result_writer.go └── search_params_test.go ├── testing ├── kind-cluster-docker.yaml ├── kindcluster_test.go └── sshserver.go ├── CONTRIBUTING.md ├── starlark ├── doc.go ├── starlark_exec_test.go ├── starlark_suite_test.go ├── prog_avail_local.go ├── os_builtins.go ├── run_local.go ├── hostlist_provider.go ├── log.go ├── kube_nodes_provider_test.go ├── prog_avail_local_test.go ├── run_local_test.go ├── set_defaults.go ├── main_test.go ├── capture_local.go ├── resources_kube_nodes_provider_test.go ├── hostlist_provider_test.go ├── archive.go ├── archive_test.go ├── kube_get.go ├── ssh_config_test.go ├── kube_nodes_provider.go ├── kube_port_forward.go ├── kube_exec.go ├── set_defaults_test.go ├── crashd_config.go ├── capv_provider.go ├── resources.go ├── ssh_config.go ├── log_test.go ├── kube_exec_test.go ├── kube_config_test.go ├── crashd_config_test.go ├── capa_provider.go ├── kube_config.go └── copy_to.go ├── util ├── util_suite_test.go ├── path.go ├── path_test.go ├── args.go └── args_test.go ├── main.go ├── examples ├── script-args.crsh ├── pod-logs.crsh ├── kube_exec.crsh ├── host-list-provider.crsh ├── kind-api-objects.crsh ├── kube-nodes-provider.crsh ├── capv_provider.crsh ├── capa_provider.crsh ├── kcp-provider.crsh ├── kind-capi-bootstrap.crsh └── windows_capv_provider.crsh ├── .ci ├── prebuild │ └── gofmt_check.go ├── common │ └── init.go ├── deploy │ └── travis_deploy.go └── build │ └── build.go ├── NOTICE.txt ├── .github └── workflows │ ├── deploy.yaml │ └── compile-test.yaml ├── .goreleaser.yml ├── exec ├── main_test.go ├── executor.go └── executor_test.go ├── kcp ├── client.go └── walker.go ├── provider └── kube_config.go ├── ssh ├── main_test.go ├── test_support.go ├── agent_test.go └── ssh.go ├── logging ├── cli.go └── file.go ├── TODO.md ├── ROADMAP.md ├── go.mod └── CODE-OF-CONDUCT.md /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .vendor 3 | .build 4 | dist 5 | .idea 6 | .DS_Store 7 | .testing -------------------------------------------------------------------------------- /cmd/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Package cmd provides entry points for CLI commands. 5 | package cmd 6 | -------------------------------------------------------------------------------- /archiver/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Package archiver provides functionality to create tar files. 5 | package archiver 6 | -------------------------------------------------------------------------------- /k8s/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Package k8s provides functionalities to interact with the Kubernetes API server 5 | package k8s 6 | -------------------------------------------------------------------------------- /testing/kind-cluster-docker.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | extraMounts: 6 | - hostPath: /var/run/docker.sock 7 | containerPath: /var/run/docker.sock -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## CLA 4 | 5 | New contributors will need to sign a contributor license agreement before code changes can be merged. Follow the instructions given by `vmwclabot` after opening a pull request. -------------------------------------------------------------------------------- /starlark/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Package starlark provides starlark builtins to be used in the diagnostics script. 5 | package starlark 6 | -------------------------------------------------------------------------------- /starlark/starlark_exec_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | func TestExec(t *testing.T) { 11 | 12 | } 13 | -------------------------------------------------------------------------------- /cmd/cmd_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package cmd 5 | 6 | import ( 7 | "testing" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | func TestCmd(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "Cmd Suite") 16 | } 17 | -------------------------------------------------------------------------------- /util/util_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package util 5 | 6 | import ( 7 | "testing" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | func TestUtil(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "Util Suite") 16 | } 17 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "os" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/vmware-tanzu/crash-diagnostics/cmd" 11 | ) 12 | 13 | func main() { 14 | if err := cmd.Run(); err != nil { 15 | logrus.Error(err) 16 | os.Exit(1) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /examples/script-args.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | conf=crashd_config(workdir=args.workdir) 5 | kube_capture(what="logs", namespaces=["default", "cert-manager", "tkg-system"], kube_config = kube_config(path=args.kubecfg)) 6 | 7 | # bundle files stored in working dir 8 | archive(output_file=args.output, source_paths=[args.workdir]) 9 | -------------------------------------------------------------------------------- /examples/pod-logs.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | conf=crashd_config(workdir="/tmp/crashlogs") 5 | set_defaults(kube_config(path="{0}/.kube/config".format(os.home))) 6 | kube_capture(what="logs", namespaces=["default", "cert-manager", "tkg-system"]) 7 | 8 | # bundle files stored in working dir 9 | archive(output_file="/tmp/craslogs.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /.ci/prebuild/gofmt_check.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | 10 | "github.com/vladimirvivien/gexe" 11 | ) 12 | 13 | func main() { 14 | e := gexe.New() 15 | if e.Run("gofmt -s -l .") != "" { 16 | fmt.Println("Go code failed gofmt check:") 17 | e.Runout("gofmt -s -d .") 18 | os.Exit(1) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /k8s/k8s.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "k8s.io/client-go/rest" 9 | ) 10 | 11 | const BaseDirname = "kubecapture" 12 | 13 | type Container interface { 14 | Fetch(context.Context, rest.Interface) (io.ReadCloser, error) 15 | Write(io.ReadCloser, string) error 16 | } 17 | 18 | func writeError(errStr error, w io.Writer) error { 19 | _, err := fmt.Fprintln(w, errStr.Error()) 20 | return err 21 | } 22 | -------------------------------------------------------------------------------- /k8s/k8s_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "testing" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | func TestK8s(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "K8s Suite") 16 | } 17 | 18 | var searchResults []SearchResult 19 | 20 | var _ = BeforeSuite(func() { 21 | searchResults = populateSearchResults() 22 | }) 23 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Crash Recovery and Diagnostics for Kubernetes 2 | Copyright (c) 2019 VMware, Inc. All Rights Reserved. 3 | 4 | This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in compliance with the Apache 2.0 License. 5 | 6 | This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 7 | 8 | -------------------------------------------------------------------------------- /util/path.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package util 5 | 6 | import ( 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | // ExpandPath converts the file path to include the home directory when prefixed with `~`. 12 | func ExpandPath(path string) (string, error) { 13 | home, err := os.UserHomeDir() 14 | if err != nil { 15 | return "", err 16 | } 17 | 18 | if path[0] == '~' { 19 | path = filepath.Join(home, path[1:]) 20 | } 21 | return path, nil 22 | } 23 | -------------------------------------------------------------------------------- /.ci/common/init.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/vladimirvivien/gexe" 7 | ) 8 | 9 | var ( 10 | // PkgRoot project package root 11 | PkgRoot string 12 | // Version default build version 13 | Version string 14 | // GitSHA last commit sha 15 | GitSHA string 16 | ) 17 | 18 | func init() { 19 | e := gexe.New() 20 | PkgRoot = "github.com/vmware-tanzu/crash-diagnostics" 21 | Version = fmt.Sprintf("%s-unreleased", e.Run("git rev-parse --abbrev-ref HEAD")) 22 | GitSHA = e.Run("git rev-parse HEAD") 23 | } 24 | -------------------------------------------------------------------------------- /cmd/defaults.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | var ( 9 | // CrashdDir is the directory path created at crashd runtime 10 | CrashdDir = filepath.Join(os.Getenv("HOME"), ".crashd") 11 | 12 | // ArgsFile is the path of the defaults args file. 13 | ArgsFile = filepath.Join(CrashdDir, "args") 14 | ) 15 | 16 | // CreateCrashdDir creates a .crashd directory which can be used as a default workdir 17 | // for script execution. It will also house the default args file. 18 | func CreateCrashdDir() error { 19 | if _, err := os.Stat(CrashdDir); os.IsNotExist(err) { 20 | return os.Mkdir(CrashdDir, 0755) 21 | } 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /cmd/buildinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/spf13/cobra" 10 | "github.com/vmware-tanzu/crash-diagnostics/buildinfo" 11 | ) 12 | 13 | func newBuildinfoCommand() *cobra.Command { 14 | cmd := &cobra.Command{ 15 | Args: cobra.NoArgs, 16 | Use: "version", 17 | Short: "prints version", 18 | Long: "Prints version information for the program", 19 | RunE: func(cmd *cobra.Command, args []string) error { 20 | fmt.Printf("Version: %s\nGitSHA: %s\n", buildinfo.Version, buildinfo.GitSHA) 21 | return nil 22 | }, 23 | } 24 | return cmd 25 | } 26 | -------------------------------------------------------------------------------- /testing/kindcluster_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package testing 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | func TestCreateKindCluster(t *testing.T) { 11 | k := NewKindCluster("./kind-cluster-docker.yaml", "testing-test-cluster", "") 12 | if err := k.Create(); err != nil { 13 | t.Error(err) 14 | } 15 | 16 | if err := k.Create(); err != nil { 17 | t.Error(err) 18 | } 19 | 20 | if k.GetKubeCtlContext() != "kind-testing-test-cluster" { 21 | t.Errorf("Unexpected kubectl context name %s", k.GetKubeCtlContext()) 22 | } 23 | 24 | if err := k.Destroy(); err != nil { 25 | t.Error(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.ci/deploy/travis_deploy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/vladimirvivien/gexe" 8 | ) 9 | 10 | func main() { 11 | e := gexe.New() 12 | fmt.Println("Running binary release...") 13 | 14 | //ensure we're travis and configuree 15 | if e.Eval("${GITHUB_TOKEN}") == "" { 16 | fmt.Println("missing GITHUB_TOKEN env") 17 | os.Exit(1) 18 | } 19 | 20 | // release on tag push 21 | if e.Eval("${PUBLISH}") == "" { 22 | fmt.Println("PUBLISH not set, skipping binary publish") 23 | e.Runout("goreleaser --rm-dist --skip-validate --skip-publish") 24 | } else { 25 | fmt.Println("Publishing binaries with goreleaser") 26 | e.Runout("goreleaser --rm-dist") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /k8s/secret.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | 6 | apierrors "k8s.io/apimachinery/pkg/api/errors" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/client-go/kubernetes" 9 | ) 10 | 11 | func GetSecretData(ctx context.Context, client kubernetes.Interface, namespace string, name string, keys []string) (map[string][]byte, error) { 12 | data := make(map[string][]byte) 13 | 14 | secret, err := client.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) 15 | if err != nil { 16 | if apierrors.IsNotFound(err) { 17 | return data, nil 18 | } 19 | return data, err 20 | } 21 | 22 | for _, key := range keys { 23 | data[key] = secret.Data[key] 24 | } 25 | 26 | return data, nil 27 | } 28 | -------------------------------------------------------------------------------- /k8s/bastion.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/vladimirvivien/gexe" 8 | ) 9 | 10 | func FetchBastionIpAddress(clusterName, namespace, kubeConfigPath string) (string, error) { 11 | if namespace == "" { 12 | namespace = "default" 13 | } 14 | p := gexe.RunProc(fmt.Sprintf( 15 | `kubectl get awscluster -l cluster.x-k8s.io/cluster-name=%s -o jsonpath='{.status.bastion.publicIp}' --namespace %s --kubeconfig %s`, 16 | clusterName, 17 | namespace, 18 | kubeConfigPath, 19 | )) 20 | 21 | if p.Err() != nil { 22 | return "", fmt.Errorf("kubectl get awscluster failed: %s: %s", p.Err(), p.Result()) 23 | } 24 | 25 | result := strings.TrimSpace(p.Result()) 26 | return strings.ReplaceAll(result, "'", ""), nil 27 | } 28 | -------------------------------------------------------------------------------- /util/path_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package util 5 | 6 | import ( 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var _ = Describe("ExpandPath", func() { 12 | 13 | It("returns the same path when input does not contain ~", func() { 14 | input := "/foo/bar" 15 | path, err := ExpandPath(input) 16 | Expect(err).NotTo(HaveOccurred()) 17 | Expect(path).To(Equal(input)) 18 | }) 19 | 20 | It("replaces the ~ with home directory path", func() { 21 | input := "~/foo/bar" 22 | path, err := ExpandPath(input) 23 | Expect(err).NotTo(HaveOccurred()) 24 | Expect(path).NotTo(Equal(input)) 25 | Expect(path).NotTo(ContainSubstring("~")) 26 | }) 27 | }) 28 | -------------------------------------------------------------------------------- /starlark/starlark_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "testing" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var ( 14 | k8sconfig string 15 | workdir string 16 | ) 17 | 18 | func TestStarlarkSuite(t *testing.T) { 19 | RegisterFailHandler(Fail) 20 | RunSpecs(t, "Starlark Suite") 21 | } 22 | 23 | var _ = BeforeSuite(func() { 24 | // setup (if necessary) and retrieve kind's kubecfg 25 | k8sCfg, err := testSupport.SetupKindKubeConfig() 26 | Expect(err).NotTo(HaveOccurred()) 27 | k8sconfig = k8sCfg 28 | workdir = testSupport.TmpDirRoot() 29 | }) 30 | 31 | var _ = AfterSuite(func() { 32 | // clean up is done in main_test.go 33 | }) 34 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Crash Diagnostics Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.**' 7 | 8 | jobs: 9 | go-release: 10 | name: goreleaser-release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - 14 | name: Checkout 15 | uses: actions/checkout@v4 16 | - 17 | name: Set up Go 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version: 1.22.1 21 | - 22 | name: gofmt-check 23 | run: GO111MODULE=on go run .ci/prebuild/gofmt_check.go 24 | - 25 | name: Binary release 26 | uses: goreleaser/goreleaser-action@v6 27 | with: 28 | version: latest 29 | args: release --clean 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | # you may remove this if you don't use vgo 4 | - go mod tidy 5 | builds: 6 | - id: crashd 7 | binary: crashd 8 | env: 9 | - CGO_ENABLED=0 10 | goos: 11 | - darwin 12 | - linux 13 | goarch: 14 | - amd64 15 | - arm64 16 | - 386 17 | ignore: 18 | - goos: darwin 19 | goarch: 386 20 | - goos: darwin 21 | goarch: arm64 22 | ldflags: -s -w -X github.com/vmware-tanzu/crash-diagnostics/buildinfo.Version=v{{.Version}} -X github.com/vmware-tanzu/crash-diagnostics/buildinfo.GitSHA={{.FullCommit}} 23 | archives: 24 | - id: tar 25 | format: tar.gz 26 | name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm}}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}' 27 | checksum: 28 | name_template: 'checksums.txt' 29 | 30 | -------------------------------------------------------------------------------- /k8s/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/sirupsen/logrus" 11 | 12 | testcrashd "github.com/vmware-tanzu/crash-diagnostics/testing" 13 | ) 14 | 15 | var ( 16 | support *testcrashd.TestSupport 17 | ) 18 | 19 | func TestMain(m *testing.M) { 20 | test, err := testcrashd.Init() 21 | if err != nil { 22 | logrus.Fatal("failed to initialize test support:", err) 23 | } 24 | 25 | support = test 26 | 27 | if err := support.SetupKindCluster(); err != nil { 28 | logrus.Fatal(err) 29 | } 30 | _, err = support.SetupKindKubeConfig() 31 | if err != nil { 32 | logrus.Fatal(err) 33 | } 34 | 35 | result := m.Run() 36 | 37 | if err := support.TearDown(); err != nil { 38 | logrus.Fatal(err) 39 | } 40 | 41 | os.Exit(result) 42 | } 43 | -------------------------------------------------------------------------------- /examples/kube_exec.crsh: -------------------------------------------------------------------------------- 1 | work_dir = args.workdir if hasattr(args, "workdir") else fail("Error: workdir argument is required but not provided.") 2 | conf = crashd_config(workdir=work_dir) 3 | kube_config_path = args.kubeconfig 4 | set_defaults(kube_config(path=kube_config_path)) 5 | 6 | # Exec into pod and run a long-running command. The command timeout period is controlled via timeout_in_seconds 7 | #Output is appended in file under work_dir/.out 8 | kube_exec(namespace=args.namespace,pod="nginx", timeout_in_seconds=3, cmd=["sh", "-c" ,"while true; do echo 'Running'; sleep 1; done"]) 9 | 10 | # Exec into pod and run short-lived command. The output will be appended in work_dir/.out 11 | kube_exec(pod="nginx", cmd=["ls"]) 12 | 13 | # Exec into pod and run short-lived command. The output will be stored into file: work_dir/nginx_version.txt 14 | kube_exec(pod="nginx", output_file="nginx_version.txt",container="nginx", cmd=["nginx", "-v"]) -------------------------------------------------------------------------------- /examples/host-list-provider.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # This script shows how to use the host list provider. 5 | # As its name implies, this provider takes a list of hosts 6 | # and allows command functions to execute on those hosts using 7 | # SSH. 8 | # 9 | # This example requires an SSH server running on the targeted hosts. 10 | 11 | # setup and configuration 12 | ssh=ssh_config( 13 | username=os.username, 14 | private_key_path=args.ssh_pk_path, 15 | port=args.ssh_port, 16 | max_retries=50, 17 | ) 18 | 19 | provider=host_list_provider(hosts=["localhost", "127.0.0.1"], ssh_config=ssh) 20 | hosts=resources(provider=provider) 21 | 22 | # commands to run on each host 23 | uptimes = run(cmd="uptime", resources=hosts) 24 | 25 | # result for resource 0 (localhost) 26 | print(uptimes[0].result) 27 | # result for resource 1 (127.0.0.1) 28 | print(uptimes[1].result) -------------------------------------------------------------------------------- /examples/kind-api-objects.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | conf=crashd_config(workdir="/tmp/crashobjs") 5 | nspaces=[ 6 | "capi-kubeadm-bootstrap-system", 7 | "capi-kubeadm-control-plane-system", 8 | "capi-system", "capi-webhook-system", 9 | "capv-system", "capa-system", 10 | "cert-manager", "tkg-system", 11 | ] 12 | 13 | set_defaults(kube_config(path=args.kubecfg)) 14 | 15 | # capture Kubernetes API object and store in files (under working dir) 16 | kube_capture(what="objects", kinds=["services", "pods"], namespaces=nspaces) 17 | kube_capture(what="objects", kinds=["deployments", "replicasets"], namespaces=nspaces) 18 | kube_capture(what="objects", kinds=["clusters", "machines", "machinesets", "machinedeployments"], namespaces=["tkg-system"]) 19 | 20 | # bundle files stored in working dir 21 | archive(output_file="/tmp/crashobjs.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /examples/kube-nodes-provider.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # This script shows how to use the kube nodes provider. 5 | # The kube node provider uses the Kubernetes Nodes objects 6 | # to enumerate compute resources that are part of the cluster. 7 | # It uses SSH to execute commands on those on nodes. 8 | # 9 | # This example requires an SSH and a Kubernetes cluster. 10 | 11 | # setup and configuration 12 | ssh=ssh_config( 13 | username=os.username, 14 | private_key_path=args.ssh_pk_path, 15 | port=args.ssh_port, 16 | max_retries=50, 17 | ) 18 | 19 | hosts=resources( 20 | provider=kube_nodes_provider( 21 | kube_config=kube_config(path=args.kubecfg), 22 | ssh_config=ssh, 23 | ), 24 | ) 25 | 26 | # commands to run on each host 27 | uptimes = run(cmd="uptime", resources=hosts) 28 | 29 | # result for resource 0 (localhost) 30 | print(uptimes.result) -------------------------------------------------------------------------------- /.github/workflows/compile-test.yaml: -------------------------------------------------------------------------------- 1 | name: Crash Diagnostics Build 2 | on: [pull_request] 3 | jobs: 4 | go-build: 5 | name: Build-Test-Binary 6 | runs-on: ubuntu-latest 7 | steps: 8 | 9 | - name: Set up Go 10 | uses: actions/setup-go@v5 11 | with: 12 | go-version: 1.22.1 13 | id: go 14 | 15 | - name: Check out code into the Go module directory 16 | uses: actions/checkout@v4 17 | 18 | - name: test 19 | run: | 20 | sudo ufw allow 2200:2300/tcp 21 | sudo ufw enable 22 | sudo ufw status verbose 23 | go get sigs.k8s.io/kind@v0.26.0 24 | go test -timeout 600s -v -p 1 ./... 25 | 26 | - name: Run gofmt 27 | run: GO111MODULE=on go run .ci/prebuild/gofmt_check.go 28 | 29 | - name: Run linter 30 | uses: golangci/golangci-lint-action@v6.1.1 31 | with: 32 | version: v1.63.4 33 | only-new-issues: true 34 | args: --timeout 5m 35 | -------------------------------------------------------------------------------- /exec/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package exec 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/sirupsen/logrus" 11 | testcrashd "github.com/vmware-tanzu/crash-diagnostics/testing" 12 | ) 13 | 14 | var ( 15 | support *testcrashd.TestSupport 16 | ) 17 | 18 | func TestMain(m *testing.M) { 19 | test, err := testcrashd.Init() 20 | if err != nil { 21 | logrus.Fatal("failed to initialize test support:", err) 22 | } 23 | 24 | support = test 25 | 26 | if err := support.SetupSSHServer(); err != nil { 27 | logrus.Fatal(err) 28 | } 29 | 30 | if err := support.SetupKindCluster(); err != nil { 31 | logrus.Fatal(err) 32 | } 33 | _, err = support.SetupKindKubeConfig() 34 | if err != nil { 35 | logrus.Fatal(err) 36 | } 37 | 38 | result := m.Run() 39 | 40 | if err := support.TearDown(); err != nil { 41 | logrus.Fatal(err) 42 | } 43 | 44 | os.Exit(result) 45 | } 46 | -------------------------------------------------------------------------------- /starlark/prog_avail_local.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/vladimirvivien/gexe" 10 | "go.starlark.net/starlark" 11 | ) 12 | 13 | // progAvailLocalFunc is a built-in starlark function that checks if a program is available locally. 14 | // It returns the path to the command if availble or else, returns an empty string. 15 | // Starlark format: prog_avail_local(prog=) 16 | func progAvailLocalFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 17 | var progStr string 18 | if err := starlark.UnpackArgs( 19 | identifiers.progAvailLocal, args, kwargs, 20 | "prog", &progStr, 21 | ); err != nil { 22 | return starlark.None, fmt.Errorf("%s: %s", identifiers.progAvailLocal, err) 23 | } 24 | 25 | p := gexe.Prog().Avail(progStr) 26 | return starlark.String(p), nil 27 | } 28 | -------------------------------------------------------------------------------- /kcp/client.go: -------------------------------------------------------------------------------- 1 | package kcp 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "k8s.io/client-go/tools/clientcmd" 8 | 9 | kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" 10 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 11 | ) 12 | 13 | func NewFromConfig(config clientcmdapi.Config) (kcpclientset.ClusterInterface, error) { 14 | // Convert api.Config to rest.Config 15 | clientConfig := clientcmd.NewDefaultClientConfig(config, &clientcmd.ConfigOverrides{}) 16 | kcpRestConfig, err := clientConfig.ClientConfig() 17 | if err != nil { 18 | return nil, fmt.Errorf("failed to create kcp REST config: %w", err) 19 | } 20 | 21 | u, err := url.Parse(kcpRestConfig.Host) 22 | if err != nil { 23 | return nil, err 24 | } 25 | u.Path = "" 26 | kcpRestConfig.Host = u.String() 27 | 28 | kcpCoreClient, err := kcpclientset.NewForConfig(kcpRestConfig) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to create kcpCoreClient: %w", err) 31 | } 32 | 33 | return kcpCoreClient, nil 34 | } 35 | -------------------------------------------------------------------------------- /provider/kube_config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package provider 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 10 | ) 11 | 12 | // KubeConfig returns the kubeconfig that needs to be used by the provider. 13 | // The path of the management kubeconfig file gets returned if the workload cluster name is empty 14 | func KubeConfig(mgmtKubeConfigPath, workloadClusterName, workloadClusterNamespace string) (string, error) { 15 | var err error 16 | 17 | if workloadClusterNamespace == "" { 18 | workloadClusterNamespace = "default" 19 | } 20 | kubeConfigPath := mgmtKubeConfigPath 21 | if len(workloadClusterName) != 0 { 22 | kubeConfigPath, err = k8s.FetchWorkloadConfig(workloadClusterName, workloadClusterNamespace, mgmtKubeConfigPath) 23 | if err != nil { 24 | err = fmt.Errorf("could not fetch kubeconfig for workload cluster %s: %w", workloadClusterName, err) 25 | } 26 | } 27 | return kubeConfigPath, err 28 | } 29 | -------------------------------------------------------------------------------- /k8s/cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "errors" 8 | 9 | "k8s.io/client-go/tools/clientcmd" 10 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 11 | ) 12 | 13 | type Config interface { 14 | GetClusterName() (string, error) 15 | GetCurrentContext() string 16 | } 17 | 18 | type KubeConfig struct { 19 | config *clientcmdapi.Config 20 | } 21 | 22 | func (kcfg *KubeConfig) GetClusterName() (string, error) { 23 | currCtx := kcfg.GetCurrentContext() 24 | if ctx, ok := kcfg.config.Contexts[currCtx]; ok { 25 | return ctx.Cluster, nil 26 | } else { 27 | return "", errors.New("unknown context: " + currCtx) 28 | } 29 | } 30 | 31 | func (kcfg *KubeConfig) GetCurrentContext() string { 32 | return kcfg.config.CurrentContext 33 | } 34 | 35 | func LoadKubeCfg(kubeConfigPath string) (Config, error) { 36 | cfg, err := clientcmd.LoadFromFile(kubeConfigPath) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return &KubeConfig{config: cfg}, nil 41 | } 42 | -------------------------------------------------------------------------------- /starlark/os_builtins.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "os" 9 | "runtime" 10 | 11 | "go.starlark.net/starlark" 12 | "go.starlark.net/starlarkstruct" 13 | ) 14 | 15 | func setupOSStruct() *starlarkstruct.Struct { 16 | return starlarkstruct.FromStringDict(starlark.String(identifiers.os), 17 | starlark.StringDict{ 18 | "name": starlark.String(runtime.GOOS), 19 | "username": starlark.String(getUsername()), 20 | "home": starlark.String(os.Getenv("HOME")), 21 | "getenv": starlark.NewBuiltin("getenv", getEnvFunc), 22 | }, 23 | ) 24 | } 25 | 26 | func getEnvFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 27 | if args == nil || args.Len() == 0 { 28 | return starlark.None, nil 29 | } 30 | key, ok := args.Index(0).(starlark.String) 31 | if !ok { 32 | return starlark.None, errors.New("os.getenv: invalid env key") 33 | } 34 | 35 | return starlark.String(os.Getenv(string(key))), nil 36 | } 37 | -------------------------------------------------------------------------------- /starlark/run_local.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/vladimirvivien/gexe" 10 | "go.starlark.net/starlark" 11 | ) 12 | 13 | // runLocalFunc is a built-in starlark function that runs a provided command on the local machine. 14 | // It returns the result of the command as struct containing information about the executed command. 15 | // Starlark format: run_local() 16 | func runLocalFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 17 | var cmdStr string 18 | if err := starlark.UnpackArgs( 19 | identifiers.runLocal, args, kwargs, 20 | "cmd", &cmdStr, 21 | ); err != nil { 22 | return starlark.None, fmt.Errorf("%s: %s", identifiers.run, err) 23 | } 24 | 25 | p := gexe.RunProc(cmdStr) 26 | result := p.Result() 27 | if p.Err() != nil { 28 | result = fmt.Sprintf("%s error: %s: %s", identifiers.runLocal, p.Err(), p.Result()) 29 | } 30 | 31 | return starlark.String(result), nil 32 | } 33 | -------------------------------------------------------------------------------- /k8s/kube_config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "encoding/base64" 8 | "fmt" 9 | "io" 10 | "os" 11 | 12 | "github.com/vladimirvivien/gexe" 13 | ) 14 | 15 | // FetchWorkloadConfig... 16 | func FetchWorkloadConfig(clusterName, clusterNamespace, mgmtKubeConfigPath string) (string, error) { 17 | var filePath string 18 | cmdStr := fmt.Sprintf(`kubectl get secrets/%s-kubeconfig --template '{{.data.value}}' --namespace=%s --kubeconfig %s`, clusterName, clusterNamespace, mgmtKubeConfigPath) 19 | p := gexe.RunProc(cmdStr) 20 | if p.Err() != nil { 21 | return filePath, fmt.Errorf("kubectl get secrets failed: %s: %s", p.Err(), p.Result()) 22 | } 23 | 24 | f, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("%s-workload-config", clusterName)) 25 | if err != nil { 26 | return filePath, fmt.Errorf("Cannot create temporary file: %w", err) 27 | } 28 | filePath = f.Name() 29 | defer f.Close() 30 | 31 | base64Dec := base64.NewDecoder(base64.StdEncoding, p.Out()) 32 | if _, err := io.Copy(f, base64Dec); err != nil { 33 | return filePath, fmt.Errorf("error decoding workload kubeconfig: %w", err) 34 | } 35 | return filePath, nil 36 | } 37 | -------------------------------------------------------------------------------- /.ci/build/build.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/vladimirvivien/gexe" 10 | ci "github.com/vmware-tanzu/crash-diagnostics/.ci/common" 11 | ) 12 | 13 | func main() { 14 | arches := []string{"amd64"} 15 | oses := []string{"darwin", "linux"} 16 | 17 | e := gexe.New() 18 | e.SetEnv("PKG_ROOT", ci.PkgRoot) 19 | e.SetEnv("VERSION", ci.Version) 20 | e.SetEnv("GIT_SHA", ci.GitSHA) 21 | e.SetEnv("LDFLAGS", `"-X ${PKG_ROOT}/buildinfo.Version=${VERSION} -X ${PKG_ROOT}/buildinfo.GitSHA=${GIT_SHA}"`) 22 | 23 | for _, arch := range arches { 24 | for _, os := range oses { 25 | binary := fmt.Sprintf(".build/%s/%s/crash-diagnostics", arch, os) 26 | gobuild(arch, os, e.Val("LDFLAGS"), binary) 27 | } 28 | } 29 | } 30 | 31 | func gobuild(arch, os, ldflags, binary string) { 32 | b := gexe.New() 33 | b.SetVar("arch", arch) 34 | b.SetVar("os", os) 35 | b.SetVar("ldflags", ldflags) 36 | b.SetVar("binary", binary) 37 | result := b.Envs("CGO_ENABLED=0 GOOS=$os GOARCH=$arch").Run("go build -o $binary -ldflags $ldflags .") 38 | if result != "" { 39 | fmt.Printf("Build for %s/%s failed: %s\n", arch, os, result) 40 | return 41 | } 42 | fmt.Printf("Build %s/%s OK: %s\n", arch, os, binary) 43 | } 44 | -------------------------------------------------------------------------------- /ssh/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package ssh 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/sirupsen/logrus" 11 | 12 | testcrashd "github.com/vmware-tanzu/crash-diagnostics/testing" 13 | ) 14 | 15 | var ( 16 | support *testcrashd.TestSupport 17 | testSSHArgs SSHArgs 18 | testSSHArgsIPv6 SSHArgs 19 | ) 20 | 21 | func TestMain(m *testing.M) { 22 | test, err := testcrashd.Init() 23 | if err != nil { 24 | logrus.Fatal(err) 25 | } 26 | support = test 27 | 28 | if err := support.SetupSSHServer(); err != nil { 29 | logrus.Fatal(err) 30 | } 31 | 32 | testSSHArgs = SSHArgs{ 33 | User: support.CurrentUsername(), 34 | PrivateKeyPath: support.PrivateKeyPath(), 35 | Host: "127.0.0.1", 36 | Port: support.PortValue(), 37 | MaxRetries: support.MaxConnectionRetries(), 38 | } 39 | 40 | testSSHArgsIPv6 = SSHArgs{ 41 | User: support.CurrentUsername(), 42 | PrivateKeyPath: support.PrivateKeyPath(), 43 | Host: "::1", 44 | Port: support.PortValue(), 45 | MaxRetries: support.MaxConnectionRetries(), 46 | } 47 | 48 | testResult := m.Run() 49 | 50 | logrus.Debug("Shutting down test...") 51 | if err := support.TearDown(); err != nil { 52 | logrus.Fatal(err) 53 | } 54 | 55 | os.Exit(testResult) 56 | } 57 | -------------------------------------------------------------------------------- /examples/capv_provider.crsh: -------------------------------------------------------------------------------- 1 | conf = crashd_config(workdir=args.workdir) 2 | ssh_conf = ssh_config(username="capv", private_key_path=args.private_key) 3 | kube_conf = kube_config(path=args.mc_config) 4 | 5 | #list out management and workload cluster nodes 6 | wc_provider=capv_provider( 7 | workload_cluster=args.cluster_name, 8 | ssh_config=ssh_conf, 9 | mgmt_kube_config=kube_conf 10 | ) 11 | nodes = resources(provider=wc_provider) 12 | 13 | capture(cmd="sudo df -i", resources=nodes) 14 | capture(cmd="sudo crictl info", resources=nodes) 15 | capture(cmd="df -h /var/lib/containerd", resources=nodes) 16 | capture(cmd="sudo systemctl status kubelet", resources=nodes) 17 | capture(cmd="sudo systemctl status containerd", resources=nodes) 18 | capture(cmd="sudo journalctl -xeu kubelet", resources=nodes) 19 | 20 | capture(cmd="sudo cat /var/log/cloud-init-output.log", resources=nodes) 21 | capture(cmd="sudo cat /var/log/cloud-init.log", resources=nodes) 22 | 23 | #add code to collect pod info from cluster 24 | set_defaults(kube_config(capi_provider = wc_provider)) 25 | 26 | pod_ns=["default", "kube-system"] 27 | 28 | kube_capture(what="logs", namespaces=pod_ns) 29 | kube_capture(what="objects", kinds=["pods", "services"], namespaces=pod_ns) 30 | kube_capture(what="objects", kinds=["deployments", "replicasets"], groups=["apps"], namespaces=pod_ns) 31 | 32 | archive(output_file="diagnostics.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /logging/cli.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | // CLIHook creates returns a hook which will log at the specified level. 10 | type CLIHook struct { 11 | Logger *logrus.Logger 12 | level logrus.Level 13 | } 14 | 15 | func NewCLIHook(w io.Writer, level logrus.Level) *CLIHook { 16 | logger := logrus.New() 17 | logger.SetLevel(level) 18 | logger.SetOutput(w) 19 | 20 | return &CLIHook{logger, level} 21 | } 22 | 23 | func (hook *CLIHook) Fire(entry *logrus.Entry) error { 24 | if !hook.Logger.IsLevelEnabled(entry.Level) { 25 | return nil 26 | } 27 | switch entry.Level { 28 | case logrus.PanicLevel: 29 | hook.Logger.Panic(entry.Message) 30 | case logrus.FatalLevel: 31 | hook.Logger.Fatal(entry.Message) 32 | case logrus.ErrorLevel: 33 | hook.Logger.Error(entry.Message) 34 | case logrus.WarnLevel: 35 | hook.Logger.Warning(entry.Message) 36 | case logrus.InfoLevel: 37 | hook.Logger.Info(entry.Message) 38 | case logrus.DebugLevel: 39 | hook.Logger.Debug(entry.Message) 40 | case logrus.TraceLevel: 41 | hook.Logger.Trace(entry.Message) 42 | default: 43 | hook.Logger.Info(entry.Message) 44 | } 45 | return nil 46 | } 47 | 48 | func (hook *CLIHook) Levels() []logrus.Level { 49 | output := []logrus.Level{} 50 | for _, v := range logrus.AllLevels { 51 | if hook.Logger.IsLevelEnabled(v) { 52 | output = append(output, v) 53 | } 54 | } 55 | return output 56 | } 57 | -------------------------------------------------------------------------------- /util/args.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package util 5 | 6 | import ( 7 | "bufio" 8 | "fmt" 9 | "log" 10 | "os" 11 | "strings" 12 | 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // ReadArgsFile parses the args file and populates the map with the contents 17 | // of that file. The parsing follows the following rules: 18 | // * each line should contain only a single key=value pair 19 | // * lines starting with # are ignored 20 | // * empty lines are ignored 21 | // * any line not following the above patterns are ignored with a warning message 22 | func ReadArgsFile(path string, args map[string]string) error { 23 | path, err := ExpandPath(path) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | file, err := os.Open(path) 29 | if err != nil { 30 | return fmt.Errorf("args file not found: %s: %w", path, err) 31 | } 32 | defer file.Close() 33 | 34 | scanner := bufio.NewScanner(file) 35 | if err := scanner.Err(); err != nil { 36 | log.Fatal(err) 37 | return err 38 | } 39 | 40 | for scanner.Scan() { 41 | line := scanner.Text() 42 | if !strings.HasPrefix(line, "#") && len(strings.TrimSpace(line)) != 0 { 43 | if pair := strings.Split(line, "="); len(pair) == 2 { 44 | args[strings.TrimSpace(pair[0])] = strings.TrimSpace(pair[1]) 45 | } else { 46 | logrus.Warnf("unknown entry in args file: %s", line) 47 | } 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /k8s/container.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "fmt" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | func GetContainers(podItem unstructured.Unstructured) ([]Container, error) { 12 | var containers []Container 13 | coreContainers, err := _getPodContainers(podItem) 14 | if err != nil { 15 | return containers, err 16 | } 17 | 18 | for _, c := range coreContainers { 19 | containers = append(containers, NewContainerLogger(podItem.GetNamespace(), podItem.GetName(), c)) 20 | } 21 | return containers, nil 22 | } 23 | 24 | func _getPodContainers(podItem unstructured.Unstructured) ([]corev1.Container, error) { 25 | var containers []corev1.Container 26 | 27 | pod := new(corev1.Pod) 28 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(podItem.Object, &pod); err != nil { 29 | return nil, fmt.Errorf("error converting container objects: %s", err) 30 | } 31 | 32 | containers = append(containers, pod.Spec.InitContainers...) 33 | containers = append(containers, pod.Spec.Containers...) 34 | containers = append(containers, _getPodEphemeralContainers(pod)...) 35 | return containers, nil 36 | } 37 | 38 | func _getPodEphemeralContainers(pod *corev1.Pod) []corev1.Container { 39 | var containers []corev1.Container 40 | for _, ec := range pod.Spec.EphemeralContainers { 41 | containers = append(containers, corev1.Container(ec.EphemeralContainerCommon)) 42 | } 43 | return containers 44 | } 45 | -------------------------------------------------------------------------------- /examples/capa_provider.crsh: -------------------------------------------------------------------------------- 1 | conf = crashd_config(workdir=args.workdir) 2 | ssh_conf = ssh_config(username="ec2-user", private_key_path=args.private_key) 3 | kube_conf = kube_config(path=args.mc_config) 4 | 5 | #list out management and workload cluster nodes 6 | wc_provider=capa_provider( 7 | workload_cluster=args.cluster_name, 8 | namespace=args.cluster_ns, 9 | ssh_config=ssh_conf, 10 | mgmt_kube_config=kube_conf 11 | ) 12 | nodes = resources(provider=wc_provider) 13 | 14 | capture(cmd="sudo df -i", resources=nodes) 15 | capture(cmd="sudo crictl info", resources=nodes) 16 | capture(cmd="df -h /var/lib/containerd", resources=nodes) 17 | capture(cmd="sudo systemctl status kubelet", resources=nodes) 18 | capture(cmd="sudo systemctl status containerd", resources=nodes) 19 | capture(cmd="sudo journalctl -xeu kubelet", resources=nodes) 20 | 21 | capture(cmd="sudo cat /var/log/cloud-init-output.log", resources=nodes) 22 | capture(cmd="sudo cat /var/log/cloud-init.log", resources=nodes) 23 | 24 | #add code to collect pod info from cluster 25 | set_defaults(kube_config(capi_provider = wc_provider)) 26 | 27 | pod_ns=["default", "kube-system"] 28 | 29 | kube_capture(what="logs", namespaces=pod_ns) 30 | kube_capture(what="objects", kinds=["pods", "services"], namespaces=pod_ns) 31 | kube_capture(what="objects", kinds=["deployments", "replicasets"], groups=["apps"], namespaces=pod_ns) 32 | kube_capture(what="objects", output_format="yaml", kinds=["deployments", "replicasets"], groups=["apps"], namespaces=pod_ns) 33 | 34 | archive(output_file="diagnostics.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /examples/kcp-provider.crsh: -------------------------------------------------------------------------------- 1 | # object kinds to capture 2 | kinds = [ 3 | "workspaces", 4 | "configmap" 5 | ] 6 | 7 | def configure_work_dir(context): 8 | work_dir = args.workdir if hasattr(args, "workdir") else fail("Error: workdir argument is required but not provided.") 9 | context_dir = work_dir + "/" + context 10 | conf = crashd_config(workdir=context_dir) 11 | 12 | def capture_kcp_objects(): 13 | # configure a tunnel to the pods 14 | tunnel_config=kube_port_forward_config(namespace="default", service="ucp-api-testorg", target_port=6443) 15 | 16 | # fetch all the workspaces in the KCP instance as contexts 17 | kcp_provider_result = kcp_provider( 18 | kcp_admin_secret_namespace="default", 19 | kcp_admin_secret_name="ucp-core-controllers-testorg-admin-kubeconfig", 20 | kcp_cert_secret_name="ucp-core-controllers-testorg-admin-cert", 21 | tunnel_config=tunnel_config 22 | ) 23 | 24 | # capture kubernetes objects from all kcp workspaces 25 | for context in kcp_provider_result.contexts: 26 | print("Capturing kcp objects for", context) 27 | 28 | # set kubeconfig path and context in threadlocal 29 | set_defaults(kube_config(capi_provider=kcp_provider_result, cluster_context=context)) 30 | 31 | # configure work directory based on context name - context-name should use "/" 32 | configure_work_dir(context) 33 | 34 | # capture objects 35 | kube_capture(what="objects", kinds=kinds, namespaces=["default"], output_format="yaml", tunnel_config=tunnel_config) 36 | 37 | def main(): 38 | capture_kcp_objects() 39 | 40 | main() 41 | -------------------------------------------------------------------------------- /examples/kind-capi-bootstrap.crsh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Kind CAPI Bootstrap example 5 | # The following script extracts CAPI bootstrap info from a kind cluster. 6 | 7 | # declare global default config for script 8 | conf=crashd_config(workdir="/tmp/crashd-test") 9 | 10 | kind_cluster = args.cluster_name 11 | 12 | # exports kind logs to a file under workdir directory 13 | run_local("kind export logs --name {0} {1}/kind-logs".format(kind_cluster, conf.workdir)) 14 | 15 | # runs `kind get kubeconfig` to capture kubeconfig file 16 | kind_cfg = capture_local( 17 | cmd="kind get kubeconfig --name {0}".format(kind_cluster), 18 | file_name="kind.kubecfg" 19 | ) 20 | 21 | # declares default configuration for Kubernetes commands 22 | 23 | nspaces=[ 24 | "capi-kubeadm-bootstrap-system", 25 | "capi-kubeadm-control-plane-system", 26 | "capi-system", "capi-webhook-system", 27 | "capv-system", "capa-system", 28 | "cert-manager", "tkg-system", 29 | ] 30 | 31 | 32 | kconf = kube_config(path=kind_cfg) 33 | 34 | # capture Kubernetes API object and store in files (under working dir) 35 | kube_capture(what="objects", kinds=["services", "pods"], namespaces=nspaces, kube_config = kconf) 36 | kube_capture(what="objects", kinds=["deployments", "replicasets"], namespaces=nspaces, kube_config = kconf) 37 | kube_capture(what="objects", kinds=["clusters", "machines", "machinesets", "machinedeployments"], namespaces=["tkg-system"], kube_config = kconf) 38 | 39 | # bundle files stored in working dir 40 | archive(output_file="/tmp/crashout.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /starlark/hostlist_provider.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | 9 | "go.starlark.net/starlark" 10 | "go.starlark.net/starlarkstruct" 11 | ) 12 | 13 | // hostListProvider is a built-in starlark function that collects compute resources as a list of host IPs 14 | // Starlark format: host_list_provider(hosts= [, ssh_config=ssh_config()]) 15 | func hostListProvider(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 16 | var hosts *starlark.List 17 | var sshCfg *starlarkstruct.Struct 18 | 19 | if err := starlark.UnpackArgs( 20 | identifiers.crashdCfg, args, kwargs, 21 | "hosts", &hosts, 22 | "ssh_config?", &sshCfg, 23 | ); err != nil { 24 | return starlark.None, fmt.Errorf("%s: %s", identifiers.hostListProvider, err) 25 | } 26 | 27 | if hosts == nil || hosts.Len() == 0 { 28 | return starlark.None, fmt.Errorf("%s: missing argument: hosts", identifiers.hostListProvider) 29 | } 30 | 31 | if sshCfg == nil { 32 | data := thread.Local(identifiers.sshCfg) 33 | cfg, ok := data.(*starlarkstruct.Struct) 34 | if !ok { 35 | return nil, fmt.Errorf("%s: default ssh_config not found", identifiers.hostListProvider) 36 | } 37 | sshCfg = cfg 38 | } 39 | 40 | cfgStruct := starlark.StringDict{ 41 | "kind": starlark.String(identifiers.hostListProvider), 42 | "transport": starlark.String("ssh"), 43 | "hosts": hosts, 44 | identifiers.sshCfg: sshCfg, 45 | } 46 | 47 | return starlarkstruct.FromStringDict(starlark.String(identifiers.hostListProvider), cfgStruct), nil 48 | } 49 | -------------------------------------------------------------------------------- /k8s/nodes.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | 10 | coreV1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | ) 13 | 14 | func GetNodeAddresses(ctx context.Context, kubeconfigPath string, names, labels []string) ([]string, error) { 15 | client, err := New(kubeconfigPath) 16 | if err != nil { 17 | return nil, fmt.Errorf("could not initialize search client: %w", err) 18 | } 19 | 20 | nodes, err := getNodes(ctx, client, names, labels) 21 | if err != nil { 22 | return nil, fmt.Errorf("could not fetch nodes: %w", err) 23 | } 24 | 25 | var nodeIps []string 26 | for _, node := range nodes { 27 | nodeIps = append(nodeIps, getNodeInternalIP(node)) 28 | } 29 | return nodeIps, nil 30 | } 31 | 32 | func getNodes(ctx context.Context, k8sc *Client, names, labels []string) ([]*coreV1.Node, error) { 33 | nodeResults, err := k8sc.Search(ctx, SearchParams{ 34 | Groups: []string{"core"}, 35 | Kinds: []string{"nodes"}, 36 | Names: names, 37 | Labels: labels, 38 | }) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | // collate 44 | var nodes []*coreV1.Node 45 | for _, result := range nodeResults { 46 | for _, item := range result.List.Items { 47 | node := new(coreV1.Node) 48 | if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item.Object, &node); err != nil { 49 | return nil, err 50 | } 51 | nodes = append(nodes, node) 52 | } 53 | } 54 | return nodes, nil 55 | } 56 | 57 | func getNodeInternalIP(node *coreV1.Node) (ipAddr string) { 58 | for _, addr := range node.Status.Addresses { 59 | if addr.Type == "InternalIP" { 60 | ipAddr = addr.Address 61 | return 62 | } 63 | } 64 | return 65 | } 66 | -------------------------------------------------------------------------------- /exec/executor.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package exec 5 | 6 | import ( 7 | "fmt" 8 | "io" 9 | "os" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/starlark" 12 | ) 13 | 14 | type ArgMap map[string]string 15 | 16 | func Execute(name string, source io.Reader, args ArgMap, restrictedMode bool) error { 17 | star, err := newExecutor(args, restrictedMode) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | return execute(star, name, source) 23 | } 24 | 25 | func ExecuteFile(file *os.File, args ArgMap, restrictedMode bool) error { 26 | return Execute(file.Name(), file, args, restrictedMode) 27 | } 28 | 29 | type StarlarkModule struct { 30 | Name string 31 | Source io.Reader 32 | } 33 | 34 | func ExecuteWithModules(name string, source io.Reader, args ArgMap, restrictedMode bool, modules ...StarlarkModule) error { 35 | star, err := newExecutor(args, restrictedMode) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | // load modules 41 | for _, mod := range modules { 42 | if err := star.Preload(mod.Name, mod.Source); err != nil { 43 | return fmt.Errorf("module load: %w", err) 44 | } 45 | } 46 | 47 | return execute(star, name, source) 48 | } 49 | 50 | func newExecutor(args ArgMap, restrictedMode bool) (*starlark.Executor, error) { 51 | star := starlark.New(restrictedMode) 52 | 53 | if args != nil { 54 | starStruct, err := starlark.NewGoValue(args).ToStarlarkStruct("args") 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | star.AddPredeclared("args", starStruct) 60 | } 61 | 62 | return star, nil 63 | } 64 | 65 | func execute(star *starlark.Executor, name string, source io.Reader) error { 66 | if err := star.Exec(name, source); err != nil { 67 | return fmt.Errorf("exec failed: %w", err) 68 | } 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /starlark/log.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/vmware-tanzu/crash-diagnostics/logging" 12 | "go.starlark.net/starlark" 13 | ) 14 | 15 | // logFunc implements a starlark built-in func for simple message logging. 16 | // This iteration uses Go's standard log package. 17 | // Example: 18 | // 19 | // log(msg="message", [prefix="info"]) 20 | func logFunc(t *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 21 | var msg string 22 | var prefix string 23 | if err := starlark.UnpackArgs( 24 | identifiers.log, args, kwargs, 25 | "msg", &msg, 26 | "prefix?", &prefix, 27 | ); err != nil { 28 | return starlark.None, fmt.Errorf("%s: %s", identifiers.log, err) 29 | } 30 | 31 | // retrieve logger from thread 32 | loggerLocal := t.Local(identifiers.log) 33 | if loggerLocal == nil { 34 | addDefaultLogger(t) 35 | loggerLocal = t.Local(identifiers.log) 36 | } 37 | 38 | switch logger := loggerLocal.(type) { 39 | case *log.Logger: 40 | if prefix != "" { 41 | logger.Printf("%s: %s", prefix, msg) 42 | } else { 43 | logger.Print(msg) 44 | } 45 | case *logrus.Logger: 46 | if prefix != "" { 47 | logger.Printf("%s: %s", prefix, msg) 48 | } else { 49 | logger.Print(msg) 50 | } 51 | default: 52 | return starlark.None, fmt.Errorf("local logger has unknown type %T", loggerLocal) 53 | } 54 | 55 | return starlark.None, nil 56 | } 57 | 58 | func addDefaultLogger(t *starlark.Thread) { 59 | loggerLocal := t.Local(identifiers.log) 60 | if loggerLocal == nil { 61 | logger := logrus.StandardLogger() 62 | t.SetLocal(identifiers.log, logger) 63 | if fh := logging.GetFirstFileHook(logger); fh != nil { 64 | t.SetLocal(identifiers.logPath, fh.FilePath) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /starlark/kube_nodes_provider_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "go.starlark.net/starlark" 11 | "go.starlark.net/starlarkstruct" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | ) 16 | 17 | var _ = Describe("kube_nodes_provider", func() { 18 | var ( 19 | executor *Executor 20 | err error 21 | ) 22 | 23 | execSetup := func(crashdScript string) error { 24 | executor = New() 25 | return executor.Exec("test.kube.nodes.provider", strings.NewReader(crashdScript)) 26 | } 27 | 28 | It("returns a struct with the list of k8s nodes", func() { 29 | crashdScript := fmt.Sprintf(` 30 | cfg = kube_config(path="%s") 31 | provider = kube_nodes_provider(kube_config = cfg, ssh_config = ssh_config(username="uname", private_key_path="path"))`, k8sconfig) 32 | err = execSetup(crashdScript) 33 | Expect(err).NotTo(HaveOccurred()) 34 | 35 | data := executor.result["provider"] 36 | Expect(data).NotTo(BeNil()) 37 | 38 | provider, ok := data.(*starlarkstruct.Struct) 39 | Expect(ok).To(BeTrue()) 40 | 41 | val, err := provider.Attr("hosts") 42 | Expect(err).NotTo(HaveOccurred()) 43 | 44 | list := val.(*starlark.List) 45 | Expect(list.Len()).To(Equal(1)) 46 | }) 47 | 48 | It("returns a struct with ssh config", func() { 49 | crashdScript := fmt.Sprintf(` 50 | cfg = kube_config(path="%s") 51 | ssh_cfg = ssh_config(username="uname", private_key_path="path") 52 | provider = kube_nodes_provider(kube_config=cfg, ssh_config = ssh_cfg)`, k8sconfig) 53 | err = execSetup(crashdScript) 54 | Expect(err).NotTo(HaveOccurred()) 55 | 56 | data := executor.result["provider"] 57 | Expect(data).NotTo(BeNil()) 58 | 59 | provider, ok := data.(*starlarkstruct.Struct) 60 | Expect(ok).To(BeTrue()) 61 | 62 | sshCfg, err := provider.Attr(identifiers.sshCfg) 63 | Expect(err).NotTo(HaveOccurred()) 64 | Expect(sshCfg).NotTo(BeNil()) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /starlark/prog_avail_local_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | "testing" 10 | 11 | "go.starlark.net/starlark" 12 | ) 13 | 14 | func TestProgAvailLocalScript(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | scriptSnippet string 18 | exists bool 19 | }{ 20 | { 21 | name: "prog_avail_local checks for 'go' using positional args", 22 | scriptSnippet: "prog_avail_local(prog='go')", 23 | exists: true, 24 | }, 25 | { 26 | name: "prog_avail_local checks for 'go' using keyword args", 27 | scriptSnippet: "prog_avail_local('go')", 28 | exists: true, 29 | }, 30 | { 31 | name: "prog_avail_local checks for 'nonexistant' using positional args", 32 | scriptSnippet: "prog_avail_local(prog='nonexistant')", 33 | exists: false, 34 | }, 35 | { 36 | name: "prog_avail_local checks for 'nonexistant' using keyword args", 37 | scriptSnippet: "prog_avail_local('nonexistant')", 38 | exists: false, 39 | }, 40 | } 41 | 42 | for _, test := range tests { 43 | t.Run(test.name, func(t *testing.T) { 44 | script := fmt.Sprintf(`path=%v`, test.scriptSnippet) 45 | exe := New() 46 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | resultVal := exe.result["path"] 51 | if resultVal == nil { 52 | t.Fatal("prog_avail_local() should be assigned to a variable for test") 53 | } 54 | 55 | result, ok := resultVal.(starlark.String) 56 | if !ok { 57 | t.Fatal("prog_avail_local() should return a string") 58 | } 59 | 60 | if (len(string(result)) == 0) == test.exists { 61 | if test.exists { 62 | t.Fatalf("expecting prog to exists but 'prog_avail_local' didnt find it") 63 | } else { 64 | t.Fatalf("expecting prog to not exists but 'prog_avail_local' found it") 65 | 66 | } 67 | } 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /k8s/container_logger.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/sirupsen/logrus" 11 | corev1 "k8s.io/api/core/v1" 12 | "k8s.io/client-go/kubernetes/scheme" 13 | "k8s.io/client-go/rest" 14 | ) 15 | 16 | type ContainerLogsImpl struct { 17 | namespace string 18 | podName string 19 | container corev1.Container 20 | } 21 | 22 | func NewContainerLogger(namespace, podName string, container corev1.Container) ContainerLogsImpl { 23 | return ContainerLogsImpl{ 24 | namespace: namespace, 25 | podName: podName, 26 | container: container, 27 | } 28 | } 29 | 30 | func (c ContainerLogsImpl) Fetch(ctx context.Context, restApi rest.Interface) (io.ReadCloser, error) { 31 | opts := &corev1.PodLogOptions{Container: c.container.Name} 32 | req := restApi.Get().Namespace(c.namespace).Name(c.podName).Resource("pods").SubResource("log").VersionedParams(opts, scheme.ParameterCodec) 33 | stream, err := req.Stream(ctx) 34 | if err != nil { 35 | err = fmt.Errorf("failed to create container log stream for container with name %s: %w", c.container.Name, err) 36 | } 37 | return stream, err 38 | } 39 | 40 | func (c ContainerLogsImpl) Write(reader io.ReadCloser, rootDir string) error { 41 | containerLogDir := filepath.Join(rootDir, c.container.Name) 42 | if err := os.MkdirAll(containerLogDir, 0744); err != nil && !os.IsExist(err) { 43 | return fmt.Errorf("error creating container log dir: %s", err) 44 | } 45 | 46 | path := filepath.Join(containerLogDir, fmt.Sprintf("%s.log", c.container.Name)) 47 | logrus.Debugf("Writing pod container log %s", path) 48 | 49 | file, err := os.Create(path) 50 | if err != nil { 51 | return err 52 | } 53 | defer file.Close() 54 | 55 | defer reader.Close() 56 | if _, err := io.Copy(file, reader); err != nil { 57 | cpErr := fmt.Errorf("failed to copy container log:\n%s", err) 58 | if wErr := writeError(cpErr, file); wErr != nil { 59 | return fmt.Errorf("failed to write previous err [%s] to file: %s", err, wErr) 60 | } 61 | return err 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /kcp/walker.go: -------------------------------------------------------------------------------- 1 | package kcp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" 8 | "github.com/kcp-dev/logicalcluster/v3" 9 | 10 | pluginhelpers "github.com/kcp-dev/kcp/cli/pkg/helpers" 11 | apierrors "k8s.io/apimachinery/pkg/api/errors" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | ) 14 | 15 | type WorkspaceWalker struct { 16 | KCPClusterClient kcpclientset.ClusterInterface 17 | Root logicalcluster.Path 18 | } 19 | 20 | func (walker *WorkspaceWalker) FetchAllWorkspaces(ctx context.Context) ([]logicalcluster.Path, error) { 21 | contexts := make([]logicalcluster.Path, 0) 22 | contexts = append(contexts, walker.Root) 23 | 24 | children, err := walker.fetchChildren(ctx, walker.Root) 25 | if err != nil { 26 | return []logicalcluster.Path{}, err 27 | } 28 | 29 | contexts = append(contexts, children...) 30 | 31 | return contexts, nil 32 | } 33 | 34 | func (walker *WorkspaceWalker) fetchChildren(ctx context.Context, parent logicalcluster.Path) ([]logicalcluster.Path, error) { 35 | children := make([]logicalcluster.Path, 0) 36 | 37 | results, err := walker.KCPClusterClient.TenancyV1alpha1().Workspaces().Cluster(parent).List(ctx, metav1.ListOptions{}) 38 | if err != nil { 39 | if apierrors.IsNotFound(err) { 40 | return []logicalcluster.Path{}, nil 41 | } 42 | return []logicalcluster.Path{}, err 43 | } 44 | 45 | for _, workspace := range results.Items { 46 | _, _, err := pluginhelpers.ParseClusterURL(workspace.Spec.URL) 47 | if err != nil { 48 | return []logicalcluster.Path{}, fmt.Errorf("current config context URL %q does not point to workspace", workspace.Spec.URL) 49 | } 50 | 51 | fullName := parent.String() + ":" + workspace.Name 52 | children = append(children, logicalcluster.NewPath(fullName)) 53 | grandchildren, err := walker.fetchChildren(ctx, logicalcluster.NewPath(fullName)) 54 | if err != nil { 55 | return []logicalcluster.Path{}, err 56 | } 57 | 58 | children = append(children, grandchildren...) 59 | } 60 | 61 | return children, nil 62 | } 63 | -------------------------------------------------------------------------------- /starlark/run_local_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "strings" 8 | "testing" 9 | 10 | "go.starlark.net/starlark" 11 | ) 12 | 13 | func TestRunLocalFunc(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | args func(t *testing.T) starlark.Tuple 17 | eval func(t *testing.T, args starlark.Tuple) 18 | }{ 19 | { 20 | name: "simple command", 21 | args: func(t *testing.T) starlark.Tuple { return starlark.Tuple{starlark.String("echo 'Hello World!'")} }, 22 | eval: func(t *testing.T, args starlark.Tuple) { 23 | val, err := runLocalFunc(newTestThreadLocal(t), nil, args, nil) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | result := "" 28 | if r, ok := val.(starlark.String); ok { 29 | result = string(r) 30 | } 31 | if result != "Hello World!" { 32 | t.Errorf("unexpected result: %s", result) 33 | } 34 | }, 35 | }, 36 | } 37 | 38 | for _, test := range tests { 39 | t.Run(test.name, func(t *testing.T) { 40 | test.eval(t, test.args(t)) 41 | }) 42 | } 43 | } 44 | 45 | func TestRunLocalScript(t *testing.T) { 46 | tests := []struct { 47 | name string 48 | script string 49 | eval func(t *testing.T, script string) 50 | }{ 51 | { 52 | name: "run local", 53 | script: ` 54 | result = run_local("""echo 'Hello World!'""") 55 | `, 56 | eval: func(t *testing.T, script string) { 57 | exe := New() 58 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 59 | t.Fatal(err) 60 | } 61 | 62 | resultVal := exe.result["result"] 63 | if resultVal == nil { 64 | t.Fatal("run_local() should be assigned to a variable for test") 65 | } 66 | result, ok := resultVal.(starlark.String) 67 | if !ok { 68 | t.Fatal("run_local() should return a string") 69 | } 70 | 71 | if string(result) != "Hello World!" { 72 | t.Fatalf("uneexpected result %s", result) 73 | } 74 | 75 | }, 76 | }, 77 | } 78 | 79 | for _, test := range tests { 80 | t.Run(test.name, func(t *testing.T) { 81 | test.eval(t, test.script) 82 | }) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /util/args_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package util 5 | 6 | import ( 7 | "os" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/ginkgo/extensions/table" 11 | . "github.com/onsi/gomega" 12 | "github.com/onsi/gomega/gbytes" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | var _ = Describe("ReadArgsFile", func() { 17 | var args map[string]string 18 | 19 | BeforeEach(func() { 20 | args = map[string]string{} 21 | }) 22 | 23 | It("returns an error when an invalid file name is passed", func() { 24 | err := ReadArgsFile("/foo/blah", args) 25 | Expect(err).To(HaveOccurred()) 26 | }) 27 | 28 | Context("with valid file", func() { 29 | DescribeTable("length of args map", func(input string, size int, warnMsgPresent bool) { 30 | f := writeContentToFile(input) 31 | defer f.Close() 32 | 33 | warnBuffer := gbytes.NewBuffer() 34 | logrus.SetOutput(warnBuffer) 35 | 36 | err := ReadArgsFile(f.Name(), args) 37 | Expect(err).NotTo(HaveOccurred()) 38 | Expect(args).To(HaveLen(size)) 39 | 40 | if warnMsgPresent { 41 | Expect(warnBuffer).To(gbytes.Say("unknown entry in args file")) 42 | } 43 | }, 44 | Entry("valid with no spaces", ` 45 | key=value 46 | foo=bar 47 | `, 2, false), 48 | Entry("valid with spaces", ` 49 | # key represents earth is round 50 | key = value 51 | foo= bar 52 | bloop =blah 53 | `, 3, false), 54 | Entry("valid with empty values", ` 55 | key = 56 | foo= bar 57 | bloop= 58 | `, 3, false), 59 | Entry("invalid", ` 60 | key value 61 | foo 62 | bar 63 | `, 0, true)) 64 | }) 65 | 66 | It("accepts comments in the args file", func() { 67 | f := writeContentToFile(`# key represents A 68 | key = value 69 | # foo represents B 70 | foo= bar`) 71 | defer f.Close() 72 | 73 | err := ReadArgsFile(f.Name(), args) 74 | Expect(err).NotTo(HaveOccurred()) 75 | Expect(args).To(HaveLen(2)) 76 | }) 77 | 78 | }) 79 | 80 | var writeContentToFile = func(content string) *os.File { 81 | f, err := os.CreateTemp(os.TempDir(), "read_file_args") 82 | Expect(err).NotTo(HaveOccurred()) 83 | 84 | err = os.WriteFile(f.Name(), []byte(content), 0644) 85 | Expect(err).NotTo(HaveOccurred()) 86 | 87 | return f 88 | } 89 | -------------------------------------------------------------------------------- /cmd/run_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "time" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/ginkgo/extensions/table" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | var _ = Describe("Run", func() { 17 | 18 | Context("For args-file and args", func() { 19 | 20 | var argsBackupFile string 21 | 22 | JustBeforeEach(func() { 23 | if _, err := os.Stat(ArgsFile); err == nil { 24 | argsBackupFile = fmt.Sprintf("%s.BKP.%s", ArgsFile, time.Now().String()) 25 | Expect(os.Rename(ArgsFile, argsBackupFile)).NotTo(HaveOccurred()) 26 | } 27 | }) 28 | 29 | JustAfterEach(func() { 30 | if argsBackupFile != "" { 31 | Expect(os.Rename(argsBackupFile, ArgsFile)).NotTo(HaveOccurred()) 32 | } 33 | }) 34 | 35 | DescribeTable("processScriptArguments", func(argsFileContent string, args map[string]string, size int) { 36 | f, err := os.CreateTemp(os.TempDir(), "") 37 | Expect(err).NotTo(HaveOccurred()) 38 | 39 | err = os.WriteFile(f.Name(), []byte(argsFileContent), 0644) 40 | Expect(err).NotTo(HaveOccurred()) 41 | 42 | defer f.Close() 43 | 44 | flags := &runFlags{ 45 | args: args, 46 | argsFile: f.Name(), 47 | } 48 | scriptArgs, err := processScriptArguments(flags) 49 | Expect(err).NotTo(HaveOccurred()) 50 | Expect(scriptArgs).To(HaveLen(size)) 51 | }, 52 | Entry("no overlapping keys", "key=value", map[string]string{"a": "b"}, 2), 53 | Entry("overlapping keys", "key=value", map[string]string{"key": "b"}, 1), 54 | Entry("file with no keys", "", map[string]string{"key": "b"}, 1), 55 | Entry("with file and without args", "key=value", map[string]string{}, 1), 56 | ) 57 | 58 | Context("With no default args file", func() { 59 | DescribeTable("does not throw an error", func(args map[string]string, size int) { 60 | scriptArgs, err := processScriptArguments(&runFlags{ 61 | args: args, 62 | argsFile: ArgsFile, 63 | }) 64 | Expect(err).NotTo(HaveOccurred()) 65 | Expect(scriptArgs).To(HaveLen(size)) 66 | }, 67 | Entry("with args", map[string]string{"a": "b"}, 1), 68 | Entry("without args", map[string]string{}, 0), 69 | ) 70 | }) 71 | }) 72 | }) 73 | -------------------------------------------------------------------------------- /starlark/set_defaults.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "go.starlark.net/starlark" 11 | "go.starlark.net/starlarkstruct" 12 | ) 13 | 14 | const UnknownDefaultErrStr = "unknown value to be set as default" 15 | 16 | // SetDefaultsFunc is the built-in fn that saves the arguments to the local Starlark thread. 17 | // Starlark format: set_defaults([ssh_config()][, kube_config()][, resources()]) 18 | func SetDefaultsFunc(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, _ []starlark.Tuple) (starlark.Value, error) { 19 | var val starlark.Value 20 | 21 | if args.Len() == 0 { 22 | return starlark.None, errors.New("atleast one of kube_config, ssh_config or resources is required") 23 | } 24 | 25 | iter := args.Iterate() 26 | defer iter.Done() 27 | for iter.Next(&val) { 28 | switch val.Type() { 29 | case "struct": 30 | constStr, err := GetConstructor(val) 31 | if err != nil { 32 | return starlark.None, fmt.Errorf("%s: %w", UnknownDefaultErrStr, err) 33 | } 34 | if constStr == identifiers.kubeCfg { 35 | thread.SetLocal(identifiers.kubeCfg, val) 36 | } else if constStr == identifiers.sshCfg { 37 | thread.SetLocal(identifiers.sshCfg, val) 38 | } else { 39 | return starlark.None, errors.New(UnknownDefaultErrStr) 40 | } 41 | case "list": 42 | list := val.(*starlark.List) 43 | if list.Len() > 0 { 44 | resourceVal := list.Index(0) 45 | constStr, err := GetConstructor(resourceVal) 46 | if err != nil || constStr != identifiers.hostResource { 47 | return starlark.None, fmt.Errorf("%s: %w", UnknownDefaultErrStr, err) 48 | } 49 | thread.SetLocal(identifiers.resources, list) 50 | } 51 | default: 52 | return starlark.None, errors.New(UnknownDefaultErrStr) 53 | } 54 | } 55 | 56 | return starlark.None, nil 57 | } 58 | 59 | func GetConstructor(val starlark.Value) (string, error) { 60 | s, ok := val.(*starlarkstruct.Struct) 61 | if !ok { 62 | return "", errors.New("cannot convert value to struct") 63 | } 64 | constructor, ok := s.Constructor().(starlark.String) 65 | if !ok { 66 | return "", errors.New("cannot convert constructor value to string") 67 | } 68 | return constructor.GoString(), nil 69 | } 70 | -------------------------------------------------------------------------------- /starlark/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "testing" 10 | 11 | "github.com/sirupsen/logrus" 12 | "go.starlark.net/starlark" 13 | "go.starlark.net/starlarkstruct" 14 | 15 | testcrashd "github.com/vmware-tanzu/crash-diagnostics/testing" 16 | ) 17 | 18 | var ( 19 | testSupport *testcrashd.TestSupport 20 | ) 21 | 22 | func TestMain(m *testing.M) { 23 | test, err := testcrashd.Init() 24 | if err != nil { 25 | logrus.Fatal(err) 26 | } 27 | testSupport = test 28 | 29 | // precaution 30 | if testSupport == nil { 31 | logrus.Fatal("failed to setup test support") 32 | } 33 | 34 | if err := testSupport.SetupSSHServer(); err != nil { 35 | logrus.Fatal(err) 36 | } 37 | 38 | if err := testSupport.SetupKindCluster(); err != nil { 39 | logrus.Fatal(err) 40 | } 41 | 42 | if _, err := testSupport.SetupKindKubeConfig(); err != nil { 43 | logrus.Fatal(err) 44 | } 45 | 46 | result := m.Run() 47 | 48 | if err := testSupport.TearDown(); err != nil { 49 | logrus.Fatal(err) 50 | } 51 | 52 | os.Exit(result) 53 | } 54 | 55 | func makeTestSSHConfig(pkPath, port, username string) *starlarkstruct.Struct { 56 | return starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{ 57 | identifiers.username: starlark.String(username), 58 | identifiers.port: starlark.String(port), 59 | identifiers.privateKeyPath: starlark.String(pkPath), 60 | identifiers.maxRetries: starlark.String(fmt.Sprintf("%d", testSupport.MaxConnectionRetries())), 61 | }) 62 | } 63 | 64 | func makeTestSSHHostResource(addr string, sshCfg *starlarkstruct.Struct) *starlarkstruct.Struct { 65 | return starlarkstruct.FromStringDict( 66 | starlarkstruct.Default, 67 | starlark.StringDict{ 68 | "kind": starlark.String(identifiers.hostResource), 69 | "provider": starlark.String(identifiers.hostListProvider), 70 | "host": starlark.String(addr), 71 | "transport": starlark.String("ssh"), 72 | "ssh_config": sshCfg, 73 | }, 74 | ) 75 | } 76 | 77 | func newTestThreadLocal(t *testing.T) *starlark.Thread { 78 | thread := &starlark.Thread{Name: "test-crashd"} 79 | if err := setupLocalDefaults(thread); err != nil { 80 | t.Fatalf("failed to setup new thread local: %s", err) 81 | } 82 | return thread 83 | } 84 | -------------------------------------------------------------------------------- /starlark/capture_local.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/vladimirvivien/gexe" 13 | "go.starlark.net/starlark" 14 | ) 15 | 16 | // captureLocalFunc is a built-in starlark function that runs a provided command on the local machine. 17 | // The output of the command is stored in a file at a specified location under the workdir directory. 18 | // Starlark format: capture_local(cmd= [,workdir=path][,file_name=name][,desc=description][,append=append]) 19 | func captureLocalFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | var cmdStr, workdir, fileName, desc string 21 | var append bool 22 | if err := starlark.UnpackArgs( 23 | identifiers.captureLocal, args, kwargs, 24 | "cmd", &cmdStr, 25 | "workdir?", &workdir, 26 | "file_name?", &fileName, 27 | "desc?", &desc, 28 | "append?", &append, 29 | ); err != nil { 30 | return starlark.None, fmt.Errorf("%s: %s", identifiers.captureLocal, err) 31 | } 32 | 33 | if len(workdir) == 0 { 34 | dir, err := getWorkdirFromThread(thread) 35 | if err != nil { 36 | return starlark.None, err 37 | } 38 | workdir = dir 39 | } 40 | if len(fileName) == 0 { 41 | fileName = fmt.Sprintf("%s.txt", sanitizeStr(cmdStr)) 42 | } 43 | 44 | filePath := filepath.Join(workdir, fileName) 45 | if err := os.MkdirAll(workdir, 0744); err != nil && !os.IsExist(err) { 46 | msg := fmt.Sprintf("%s error: %s", identifiers.captureLocal, err) 47 | return starlark.String(msg), nil 48 | } 49 | 50 | p := gexe.RunProc(cmdStr) 51 | // upon error, write error in file, return filepath 52 | if p.Err() != nil { 53 | msg := fmt.Sprintf("%s error: %s: %s", identifiers.captureLocal, p.Err(), p.Result()) 54 | if err := captureOutput(strings.NewReader(msg), filePath, desc, append); err != nil { 55 | msg := fmt.Sprintf("%s error: %s", identifiers.captureLocal, err) 56 | return starlark.String(msg), nil 57 | } 58 | return starlark.String(filePath), nil 59 | } 60 | 61 | if err := captureOutput(p.Out(), filePath, desc, append); err != nil { 62 | msg := fmt.Sprintf("%s error: %s", identifiers.captureLocal, err) 63 | return starlark.String(msg), nil 64 | } 65 | 66 | return starlark.String(filePath), nil 67 | } 68 | -------------------------------------------------------------------------------- /starlark/resources_kube_nodes_provider_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "go.starlark.net/starlark" 11 | "go.starlark.net/starlarkstruct" 12 | 13 | . "github.com/onsi/ginkgo/extensions/table" 14 | . "github.com/onsi/gomega" 15 | ) 16 | 17 | var _ = DescribeTable("resources with kube_nodes_provider()", func(scriptFunc func() string) { 18 | 19 | executor := New() 20 | crashdScript := scriptFunc() 21 | err := executor.Exec("test.resources.kube.nodes.provider", strings.NewReader(crashdScript)) 22 | Expect(err).NotTo(HaveOccurred()) 23 | 24 | data := executor.result["res"] 25 | Expect(data).NotTo(BeNil()) 26 | 27 | resources, ok := data.(*starlark.List) 28 | Expect(ok).To(BeTrue()) 29 | Expect(resources.Len()).To(Equal(1)) 30 | 31 | resStruct, ok := resources.Index(0).(*starlarkstruct.Struct) 32 | Expect(ok).To(BeTrue()) 33 | 34 | val, err := resStruct.Attr("kind") 35 | Expect(err).NotTo(HaveOccurred()) 36 | Expect(trimQuotes(val.String())).To(Equal(identifiers.hostResource)) 37 | 38 | transport, err := resStruct.Attr("transport") 39 | Expect(err).NotTo(HaveOccurred()) 40 | Expect(trimQuotes(transport.String())).To(Equal("ssh")) 41 | 42 | sshCfg, err := resStruct.Attr(identifiers.sshCfg) 43 | Expect(err).NotTo(HaveOccurred()) 44 | Expect(sshCfg).NotTo(BeNil()) 45 | 46 | host, err := resStruct.Attr("host") 47 | Expect(err).NotTo(HaveOccurred()) 48 | // Regex to match IP address of the host 49 | Expect(trimQuotes(host.String())).To(MatchRegexp("^([1-9]?[0-9]{2}\\.)([0-9]{1,3}\\.){2}[0-9]{1,3}$")) 50 | }, 51 | Entry("default ssh config and passed kube_config", func() string { 52 | return fmt.Sprintf(` 53 | set_defaults(ssh_config(username="uname", private_key_path="path")) 54 | res = resources(provider = kube_nodes_provider(kube_config = kube_config(path="%s")))`, k8sconfig) 55 | }), 56 | Entry("default kube config and passed ssh_config", func() string { 57 | return fmt.Sprintf(` 58 | set_defaults(kube_config(path="%s")) 59 | res = resources(provider=kube_nodes_provider(ssh_config = ssh_config(username="uname", private_key_path="path")))`, k8sconfig) 60 | }), 61 | Entry("default kube_config and ssh_config", func() string { 62 | return fmt.Sprintf(` 63 | set_defaults(kube_config(path="%s"), ssh_config(username="uname", private_key_path="path")) 64 | res = resources(provider=kube_nodes_provider())`, k8sconfig) 65 | }), 66 | ) 67 | -------------------------------------------------------------------------------- /cmd/crashd.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package cmd 5 | 6 | import ( 7 | "io" 8 | "os" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | "github.com/vmware-tanzu/crash-diagnostics/buildinfo" 13 | "github.com/vmware-tanzu/crash-diagnostics/logging" 14 | ) 15 | 16 | const ( 17 | defaultLogLevel = logrus.InfoLevel 18 | CliName = "crashd" 19 | ) 20 | 21 | // globalFlags flags for the command 22 | type globalFlags struct { 23 | debug bool 24 | logFile string 25 | } 26 | 27 | // crashDiagnosticsCommand creates a main cli command 28 | func crashDiagnosticsCommand() *cobra.Command { 29 | flags := &globalFlags{debug: false, logFile: "auto"} 30 | cmd := &cobra.Command{ 31 | Args: cobra.NoArgs, 32 | Use: CliName, 33 | Short: "runs the crashd program", 34 | Long: "Runs the crashd program to execute script that interacts with Kubernetes clusters", 35 | PersistentPreRunE: func(cmd *cobra.Command, args []string) error { 36 | return preRun(flags) 37 | }, 38 | SilenceUsage: true, 39 | Version: buildinfo.Version, 40 | } 41 | 42 | cmd.PersistentFlags().BoolVar( 43 | &flags.debug, 44 | "debug", 45 | flags.debug, 46 | "sets log level to debug", 47 | ) 48 | 49 | cmd.PersistentFlags().StringVar( 50 | &flags.logFile, 51 | "log-file", 52 | flags.logFile, 53 | "Filepath to log to. Defaults to 'auto' which will generate a unique log file. If empty, will disable logging to a file.", 54 | ) 55 | 56 | cmd.AddCommand(newRunCommand()) 57 | cmd.AddCommand(newBuildinfoCommand()) 58 | return cmd 59 | } 60 | 61 | func preRun(flags *globalFlags) error { 62 | if err := CreateCrashdDir(); err != nil { 63 | return err 64 | } 65 | 66 | if len(flags.logFile) > 0 { 67 | // Log everything to file, regardless of settings for CLI. 68 | filehook, err := logging.NewFileHook(flags.logFile) 69 | if err != nil { 70 | logrus.Warning("Failed to log to file, logging to stdout (default)") 71 | } else { 72 | logrus.AddHook(filehook) 73 | } 74 | } 75 | 76 | level := defaultLogLevel 77 | if flags.debug { 78 | level = logrus.DebugLevel 79 | } 80 | logrus.AddHook(logging.NewCLIHook(os.Stdout, level)) 81 | 82 | // Set to trace so all hooks fire. We will handle levels differently for CLI/file. 83 | logrus.SetOutput(io.Discard) 84 | logrus.SetLevel(logrus.TraceLevel) 85 | 86 | return nil 87 | } 88 | 89 | // Run starts the command 90 | func Run() error { 91 | return crashDiagnosticsCommand().Execute() 92 | } 93 | -------------------------------------------------------------------------------- /k8s/object_writer.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "github.com/sirupsen/logrus" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | "k8s.io/cli-runtime/pkg/printers" 12 | ) 13 | 14 | type ObjectWriter struct { 15 | writeDir string 16 | printer printers.ResourcePrinter 17 | singleFile bool 18 | } 19 | 20 | func (w *ObjectWriter) Write(result SearchResult) (string, error) { 21 | // namespaced on group and version to avoid overwrites 22 | grp := func() string { 23 | if result.GroupVersionResource.Group == "" { 24 | return "core" 25 | } 26 | return result.GroupVersionResource.Group 27 | }() 28 | 29 | w.writeDir = filepath.Join(w.writeDir, fmt.Sprintf("%s_%s", grp, result.GroupVersionResource.Version)) 30 | 31 | // add resource namespace if needed 32 | if result.Namespaced { 33 | w.writeDir = filepath.Join(w.writeDir, result.Namespace) 34 | } 35 | 36 | now := time.Now().Format("2006-01-02T15-04-05Z.0000") 37 | var extension string 38 | if _, ok := w.printer.(*printers.JSONPrinter); ok { 39 | extension = "json" 40 | } else { 41 | extension = "yaml" 42 | } 43 | 44 | if w.singleFile { 45 | if err := os.MkdirAll(w.writeDir, 0744); err != nil && !os.IsExist(err) { 46 | return "", fmt.Errorf("failed to create search result dir: %s", err) 47 | } 48 | path := filepath.Join(w.writeDir, fmt.Sprintf("%s-%s.%s", result.ResourceName, now, extension)) 49 | return w.writeDir, w.writeFile(result.List, path) 50 | } else { 51 | w.writeDir = filepath.Join(w.writeDir, result.ResourceName) 52 | if err := os.MkdirAll(w.writeDir, 0744); err != nil && !os.IsExist(err) { 53 | return "", fmt.Errorf("failed to create search result dir: %s", err) 54 | } 55 | 56 | for i := range result.List.Items { 57 | u := &result.List.Items[i] 58 | path := filepath.Join(w.writeDir, fmt.Sprintf("%s-%s.%s", u.GetName(), now, extension)) 59 | if err := w.writeFile(u, path); err != nil { 60 | return "", err 61 | } 62 | } 63 | } 64 | 65 | return w.writeDir, nil 66 | } 67 | 68 | func (w *ObjectWriter) writeFile(o runtime.Object, path string) error { 69 | file, err := os.Create(path) 70 | if err != nil { 71 | return err 72 | } 73 | defer file.Close() 74 | 75 | logrus.Debugf("objectWriter: saving %s search results to: %s", o.GetObjectKind(), path) 76 | 77 | if err := w.printer.PrintObj(o, file); err != nil { 78 | if wErr := writeError(err, file); wErr != nil { 79 | return fmt.Errorf("objectWriter: failed to write previous err [%s] to file: %s", err, wErr) 80 | } 81 | return err 82 | } 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 2 | # v.0.1.0-alpha.0 3 | This tag/version reflects migration to github 4 | * [x] Reset tag/version to alpha.0 5 | * [x] Add ROADMAP.md 6 | * [x] Start CHANGELOG notes 7 | * [x] Apply tag v0.1.0-alpha.0 8 | * [x] Update release notes in GitHub 9 | 10 | # v0.1.0-alpha.4 11 | * [x] Add automated build (Makefile etc) 12 | * [x] Git describe version reporting (i.e. `crash-diagnostics --version`) 13 | * [x] Add project badges to README 14 | * [x] Apply tag v0.1.0-alpha.4 15 | 16 | * Translate identified TODOs to issues. 17 | 18 | # v.0.1.0-alpha.5 19 | * [x] Uniform directive parameter format (i.e. DIRECTIVE param:value) 20 | 21 | # v.0.1.0-alpha.6 22 | * [x] Introduce support for quoted strings in directives 23 | 24 | # v0.1.0-alpha.7 25 | * [x] Revamp EVN variable expansion (i.e. use os.Expand) 26 | * [x] Suppport for shell variable expansion format (i.e. ${Var}) 27 | * [x] Deprecate support for Go style templating ( i.e. {{.Var}} ) 28 | * [x] Apply tag v0.1.0-alpha.7 29 | 30 | # v0.1.0-alpha.8 31 | * [x] New directive `RUN` (i.e. RUN shell:"/bin/bash" cmd:"" ) 32 | * [x] Grammatical/word corrections in crash-diagnostics --help output 33 | 34 | # v0.1.0-alpha.9 35 | * [x] Embedded quotes bug fix for `RUN` and `CAPTURE` 36 | 37 | # v0.1.0 Release 38 | * [x] Doc udpdate 39 | * [x] Todo and Roadmap updates 40 | 41 | # v0.1.1 42 | * [x] Fix parameter expansion clash between tool parser and shell 43 | * [x] Introduce ability to escape variable expansion 44 | * [x] Update docs 45 | * [x] Update changelog doc 46 | 47 | # v0.2.0-alpha.0 48 | * [x] New directive `KUBEGET` 49 | * [x] Update doc 50 | 51 | 52 | # v0.2.1-alpha.0 53 | * [x] Introduce support for command echo parameter 54 | * [x] Documentation update 55 | 56 | # v0.2.1-alpha.1 57 | * [x] Remove support for local execution model 58 | * [x] The default executor will use SSH/SCP even when targeting local machine 59 | * [x] Update test for new executor backend 60 | * [x] Update CI/CD to automate end-to-end tests using SSH server 61 | * [x] Documentation update 62 | 63 | # v0.2.1-alpha.1 64 | * [ ] Update FROM to accept node related params 65 | * [ ] Update test code for FROM command and execution 66 | * [ ] Integrate kind for end-to-end tests 67 | * [ ] Update docs 68 | 69 | # v0.2.2-alpha.0 70 | * [ ] Initial CloudAPI support 71 | 72 | # Other Tasks 73 | * [ ] Documentation update (tutorials and how tos) 74 | * [ ] Recipes (i.e. Diagnostics.file files for different well known debg) 75 | * [ ] Cloud API recipes (i.e. recipes to debug CAPV) 76 | 77 | # v0.3.0 78 | * Redesign the script/configuration language for Crash Diagnostics 79 | * Refactor internal and implement support for [Starlark](https://github.com/bazelbuild/starlark) language -------------------------------------------------------------------------------- /starlark/hostlist_provider_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "strings" 8 | "testing" 9 | 10 | "go.starlark.net/starlark" 11 | "go.starlark.net/starlarkstruct" 12 | ) 13 | 14 | func TestHostListProvider(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | script string 18 | eval func(t *testing.T, script string) 19 | }{ 20 | { 21 | name: "single host", 22 | script: `provider = host_list_provider(hosts=["foo.host"], ssh_config = ssh_config(username="uname", private_key_path="path"))`, 23 | eval: func(t *testing.T, script string) { 24 | exe := New() 25 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 26 | t.Fatal(err) 27 | } 28 | data := exe.result["provider"] 29 | if data == nil { 30 | t.Fatalf("%s function not returning value", identifiers.hostListProvider) 31 | } 32 | provider, ok := data.(*starlarkstruct.Struct) 33 | if !ok { 34 | t.Fatalf("expecting *starlark.Struct, got %T", data) 35 | } 36 | val, err := provider.Attr("hosts") 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | list := val.(*starlark.List) 41 | if list.Len() != 1 { 42 | t.Fatalf("expecting %d items for argument 'hosts', got %d", 2, list.Len()) 43 | } 44 | 45 | sshcfg, err := provider.Attr(identifiers.sshCfg) 46 | if err != nil { 47 | t.Error(err) 48 | } 49 | if sshcfg == nil { 50 | t.Errorf("%s missing ssh_config", identifiers.hostListProvider) 51 | } 52 | }, 53 | }, 54 | { 55 | name: "multiple hosts", 56 | script: `provider = host_list_provider(hosts=["foo.host.1", "foo.host.2"], ssh_config = ssh_config(username="uname", private_key_path="path"))`, 57 | eval: func(t *testing.T, script string) { 58 | exe := New() 59 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 60 | t.Fatal(err) 61 | } 62 | data := exe.result["provider"] 63 | if data == nil { 64 | t.Fatalf("%s function not returning value", identifiers.hostListProvider) 65 | } 66 | provider, ok := data.(*starlarkstruct.Struct) 67 | if !ok { 68 | t.Fatalf("expecting *starlark.Struct, got %T", data) 69 | } 70 | 71 | val, err := provider.Attr("hosts") 72 | if err != nil { 73 | t.Fatal(err) 74 | } 75 | list := val.(*starlark.List) 76 | if list.Len() != 2 { 77 | t.Fatalf("expecting %d items for argument 'hosts', got %d", 2, list.Len()) 78 | } 79 | }, 80 | }, 81 | } 82 | 83 | for _, test := range tests { 84 | t.Run(test.name, func(t *testing.T) { 85 | test.eval(t, test.script) 86 | }) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /starlark/archive.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/vmware-tanzu/crash-diagnostics/logging" 11 | "go.starlark.net/starlark" 12 | 13 | "github.com/vmware-tanzu/crash-diagnostics/archiver" 14 | ) 15 | 16 | // archiveFunc is a built-in starlark function that bundles specified directories into 17 | // an arhive format (i.e. tar.gz) 18 | // Starlark format: archive(output_file= ,source_paths=list, includeLogs?=[True|False], includeScript?=[True|False]) 19 | func archiveFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | var outputFile string 21 | var paths *starlark.List 22 | 23 | // Default to true so that it helps users in debugging. 24 | includeLogs := true 25 | includeScript := true 26 | 27 | if err := starlark.UnpackArgs( 28 | identifiers.archive, args, kwargs, 29 | "output_file?", &outputFile, 30 | "source_paths", &paths, 31 | "includeLogs?", &includeLogs, 32 | "includeScript?", &includeScript, 33 | ); err != nil { 34 | return starlark.None, fmt.Errorf("%s: %s", identifiers.archive, err) 35 | } 36 | 37 | if len(outputFile) == 0 { 38 | outputFile = "archive.tar.gz" 39 | } 40 | 41 | // Always include the script executed and the logs. 42 | if script := thread.Local(identifiers.scriptName); includeScript && script != nil && len(script.(string)) > 0 { 43 | if err := paths.Append(starlark.String(script.(string))); err != nil { 44 | logrus.Warnf("Unexpected error when adding script to archive paths: %v", err) 45 | } 46 | } 47 | if logPath := thread.Local(identifiers.logPath); includeLogs && logPath != nil && len(logPath.(string)) > 0 { 48 | if err := paths.Append(starlark.String(logPath.(string))); err != nil { 49 | logrus.Warnf("Unexpected error when adding log path to archive paths: %v", err) 50 | } 51 | if err := logging.CloseFileHooks(nil); err != nil { 52 | logrus.Warnf("Unexpected error when closing file hooks: %v", err) 53 | } 54 | } 55 | 56 | if paths != nil && paths.Len() == 0 { 57 | return starlark.None, fmt.Errorf("%s: one or more paths required", identifiers.archive) 58 | } 59 | 60 | if err := archiver.Tar(outputFile, getPathElements(paths)...); err != nil { 61 | return starlark.None, fmt.Errorf("%s failed: %s", identifiers.archive, err) 62 | } 63 | 64 | return starlark.String(outputFile), nil 65 | } 66 | 67 | func getPathElements(paths *starlark.List) []string { 68 | pathElems := []string{} 69 | for i := 0; i < paths.Len(); i++ { 70 | if val, ok := paths.Index(i).(starlark.String); ok { 71 | pathElems = append(pathElems, string(val)) 72 | } 73 | } 74 | return pathElems 75 | } 76 | -------------------------------------------------------------------------------- /starlark/archive_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "go.starlark.net/starlark" 12 | ) 13 | 14 | func TestArchiveFunc(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | args func(t *testing.T) []starlark.Tuple 18 | eval func(t *testing.T, kwargs []starlark.Tuple) 19 | }{ 20 | { 21 | name: "archive single file", 22 | args: func(t *testing.T) []starlark.Tuple { 23 | return []starlark.Tuple{ 24 | {starlark.String("output_file"), starlark.String("/tmp/out.tar.gz")}, 25 | {starlark.String("source_paths"), starlark.NewList([]starlark.Value{starlark.String(defaults.workdir)})}, 26 | } 27 | }, 28 | eval: func(t *testing.T, kwargs []starlark.Tuple) { 29 | val, err := archiveFunc(newTestThreadLocal(t), nil, nil, kwargs) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | expected := "/tmp/out.tar.gz" 34 | defer func() { 35 | os.RemoveAll(expected) 36 | os.RemoveAll(defaults.workdir) 37 | }() 38 | 39 | result := "" 40 | if r, ok := val.(starlark.String); ok { 41 | result = string(r) 42 | } 43 | if result != expected { 44 | t.Errorf("unexpected result: %s", result) 45 | } 46 | }, 47 | }, 48 | } 49 | for _, test := range tests { 50 | t.Run(test.name, func(t *testing.T) { 51 | test.eval(t, test.args(t)) 52 | }) 53 | } 54 | } 55 | 56 | func TestArchiveScript(t *testing.T) { 57 | tests := []struct { 58 | name string 59 | script string 60 | eval func(t *testing.T, script string) 61 | }{ 62 | { 63 | name: "archive defaults", 64 | script: `result = archive(output_file="/tmp/archive.tar.gz", source_paths=["/tmp/crashd"])`, 65 | eval: func(t *testing.T, script string) { 66 | exe := New() 67 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | expected := "/tmp/archive.tar.gz" 72 | var result string 73 | resultVal := exe.result["result"] 74 | if resultVal == nil { 75 | t.Fatal("archive() should be assigned to a variable for test") 76 | } 77 | res, ok := resultVal.(starlark.String) 78 | if !ok { 79 | t.Fatal("archive() should return a string") 80 | } 81 | result = string(res) 82 | defer func() { 83 | os.RemoveAll(result) 84 | os.RemoveAll(defaults.workdir) 85 | }() 86 | 87 | if result != expected { 88 | t.Errorf("unexpected result: %s", result) 89 | } 90 | }, 91 | }, 92 | } 93 | 94 | for _, test := range tests { 95 | t.Run(test.name, func(t *testing.T) { 96 | test.eval(t, test.script) 97 | }) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /archiver/tarrer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package archiver 5 | 6 | import ( 7 | "archive/tar" 8 | "compress/gzip" 9 | "io" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // Tar compresses the file sources specified by paths into a single 18 | // tarball specified by tarName. 19 | func Tar(tarName string, paths ...string) (err error) { 20 | logrus.Debugf("Archiving %v in %s", paths, tarName) 21 | tarFile, err := os.Create(tarName) 22 | if err != nil { 23 | return err 24 | } 25 | defer func() { 26 | err = tarFile.Close() 27 | }() 28 | 29 | absTar, err := filepath.Abs(tarName) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | // enable compression if file ends in .gz 35 | tw := tar.NewWriter(tarFile) 36 | if strings.HasSuffix(tarName, ".gz") || strings.HasSuffix(tarName, ".gzip") { 37 | gz := gzip.NewWriter(tarFile) 38 | defer gz.Close() 39 | tw = tar.NewWriter(gz) 40 | } 41 | defer tw.Close() 42 | 43 | // walk each path and add encountered file to tar 44 | for _, path := range paths { 45 | // validate path 46 | path = filepath.Clean(path) 47 | absPath, err := filepath.Abs(path) 48 | if err != nil { 49 | logrus.Error(err) 50 | continue 51 | } 52 | if absPath == absTar { 53 | logrus.Errorf("Tar file %s cannot be the source, skipping path", tarName) 54 | continue 55 | } 56 | if absPath == filepath.Dir(absTar) { 57 | logrus.Errorf("Tar file %s cannot be in source %s, skipping path", tarName, absPath) 58 | continue 59 | } 60 | 61 | // build tar 62 | err = filepath.Walk(path, func(file string, finfo os.FileInfo, err error) error { 63 | if err != nil { 64 | return err 65 | } 66 | 67 | relFilePath := file 68 | if filepath.IsAbs(path) { 69 | relFilePath, err = filepath.Rel("/", file) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | 75 | hdr, err := tar.FileInfoHeader(finfo, finfo.Name()) 76 | if err != nil { 77 | return err 78 | } 79 | // ensure header has relative file path 80 | hdr.Name = relFilePath 81 | if err := tw.WriteHeader(hdr); err != nil { 82 | return err 83 | } 84 | if finfo.Mode().IsDir() { 85 | return nil 86 | } 87 | 88 | // add file to tar 89 | srcFile, err := os.Open(file) 90 | if err != nil { 91 | return err 92 | } 93 | defer srcFile.Close() 94 | _, err = io.Copy(tw, srcFile) 95 | if err != nil { 96 | return err 97 | } 98 | logrus.Debugf("Archived %s", file) 99 | return nil 100 | }) 101 | if err != nil { 102 | logrus.Errorf("failed to add %s to archive %s: %v", path, tarName, err) 103 | } 104 | } 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /starlark/kube_get.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 12 | "go.starlark.net/starlark" 13 | "go.starlark.net/starlarkstruct" 14 | ) 15 | 16 | // KubeGetFn is a starlark built-in for the fetching kubernetes objects 17 | func KubeGetFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 18 | objects := starlark.NewList([]starlark.Value{}) 19 | var groups, categories, kinds, namespaces, versions, names, labels, containers *starlark.List 20 | var kubeConfig *starlarkstruct.Struct 21 | 22 | if err := starlark.UnpackArgs( 23 | identifiers.kubeGet, args, kwargs, 24 | "groups?", &groups, 25 | "categories?", &categories, 26 | "kinds?", &kinds, 27 | "namespaces?", &namespaces, 28 | "versions?", &versions, 29 | "names?", &names, 30 | "labels?", &labels, 31 | "containers?", &containers, 32 | "kube_config?", &kubeConfig, 33 | ); err != nil { 34 | return starlark.None, fmt.Errorf("failed to read args: %w", err) 35 | } 36 | 37 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 38 | if !ok || ctx == nil { 39 | return starlark.None, errors.New("script context not found") 40 | } 41 | 42 | if kubeConfig == nil { 43 | kubeConfig = thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 44 | } 45 | path, err := getKubeConfigPathFromStruct(kubeConfig) 46 | if err != nil { 47 | return starlark.None, fmt.Errorf("failed to get kubeconfig: %w", err) 48 | } 49 | clusterCtxName := getKubeConfigContextNameFromStruct(kubeConfig) 50 | 51 | client, err := k8s.New(path, clusterCtxName) 52 | if err != nil { 53 | return starlark.None, fmt.Errorf("could not initialize search client: %w", err) 54 | } 55 | 56 | searchParams := k8s.SearchParams{ 57 | Groups: toSlice(groups), 58 | Categories: toSlice(categories), 59 | Kinds: toSlice(kinds), 60 | Namespaces: toSlice(namespaces), 61 | Versions: toSlice(versions), 62 | Names: toSlice(names), 63 | Labels: toSlice(labels), 64 | Containers: toSlice(containers), 65 | } 66 | searchResults, searchErr := client.Search(ctx, searchParams) 67 | if searchErr == nil { 68 | for _, searchResult := range searchResults { 69 | srValue := searchResult.ToStarlarkValue() 70 | if err := objects.Append(srValue); err != nil { 71 | searchErr = fmt.Errorf("could not collect kube_get() results: %w", err) 72 | break 73 | } 74 | } 75 | } 76 | 77 | return starlarkstruct.FromStringDict( 78 | starlark.String(identifiers.kubeGet), 79 | starlark.StringDict{ 80 | "objs": objects, 81 | "error": func() starlark.String { 82 | if searchErr != nil { 83 | return starlark.String(searchErr.Error()) 84 | } 85 | return "" 86 | }(), 87 | }), nil 88 | } 89 | -------------------------------------------------------------------------------- /cmd/run.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "strings" 10 | 11 | "github.com/spf13/cobra" 12 | "github.com/vmware-tanzu/crash-diagnostics/exec" 13 | "github.com/vmware-tanzu/crash-diagnostics/util" 14 | ) 15 | 16 | type runFlags struct { 17 | args map[string]string 18 | argsFile string 19 | restrictedMode bool 20 | } 21 | 22 | func defaultRunFlags() *runFlags { 23 | return &runFlags{ 24 | args: make(map[string]string), 25 | argsFile: ArgsFile, 26 | restrictedMode: false, 27 | } 28 | } 29 | 30 | // newRunCommand creates a command to run the Diagnostics script a file 31 | func newRunCommand() *cobra.Command { 32 | flags := defaultRunFlags() 33 | 34 | cmd := &cobra.Command{ 35 | Args: cobra.ExactArgs(1), 36 | Use: "run ", 37 | Short: "runs a script file", 38 | Long: "Parses and executes the specified script file", 39 | RunE: func(cmd *cobra.Command, args []string) error { 40 | return run(flags, args[0]) 41 | }, 42 | } 43 | cmd.Flags().StringToStringVar(&flags.args, "args", flags.args, "comma-separated key=value pairs passed to the script (i.e. --args 'key0=val0,key1=val1')") 44 | cmd.Flags().StringVar(&flags.argsFile, "args-file", flags.argsFile, "path to a file containing key=value argument pairs that are passed to the script file") 45 | cmd.Flags().BoolVar(&flags.restrictedMode, "restrictedMode", flags.restrictedMode, "run the script in a restricted mode that prevents usage of certain grammar functions") 46 | return cmd 47 | } 48 | 49 | func run(flags *runFlags, path string) error { 50 | file, err := os.Open(path) 51 | if err != nil { 52 | return fmt.Errorf("failed to open script file: %s: %w", path, err) 53 | } 54 | defer file.Close() 55 | 56 | scriptArgs, err := processScriptArguments(flags) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | if err := exec.ExecuteFile(file, scriptArgs, flags.restrictedMode); err != nil { 62 | return fmt.Errorf("execution failed for %s: %w", file.Name(), err) 63 | } 64 | 65 | return nil 66 | } 67 | 68 | // prepares a map of key-value strings to be passed to the execution script 69 | // It builds the map from the args-file as well as the args flag passed to 70 | // the run command. 71 | func processScriptArguments(flags *runFlags) (map[string]string, error) { 72 | scriptArgs := map[string]string{} 73 | 74 | // get args from script args file 75 | err := util.ReadArgsFile(flags.argsFile, scriptArgs) 76 | if err != nil && flags.argsFile != ArgsFile { 77 | return nil, fmt.Errorf("failed to parse scriptArgs file %s: %w", flags.argsFile, err) 78 | } 79 | 80 | // any value specified by the args flag overrides 81 | // value with same key in the args-file 82 | for k, v := range flags.args { 83 | scriptArgs[strings.TrimSpace(k)] = strings.TrimSpace(v) 84 | } 85 | 86 | return scriptArgs, nil 87 | } 88 | -------------------------------------------------------------------------------- /starlark/ssh_config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "strings" 8 | "testing" 9 | 10 | "go.starlark.net/starlarkstruct" 11 | ) 12 | 13 | func TestSSHConfigNew(t *testing.T) { 14 | e := New() 15 | if e.thread == nil { 16 | t.Error("thread is nil") 17 | } 18 | } 19 | 20 | func TestSSHConfigFunc(t *testing.T) { 21 | tests := []struct { 22 | name string 23 | script string 24 | eval func(t *testing.T, script string) 25 | }{ 26 | { 27 | name: "ssh_config saved in thread", 28 | script: `set_defaults(ssh_config(username="uname", private_key_path="path"))`, 29 | eval: func(t *testing.T, script string) { 30 | exe := New() 31 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 32 | t.Fatal(err) 33 | } 34 | data := exe.thread.Local(identifiers.sshCfg) 35 | if data == nil { 36 | t.Fatal("ssh_config not saved in thread local") 37 | } 38 | cfg, ok := data.(*starlarkstruct.Struct) 39 | if !ok { 40 | t.Fatalf("unexpected type for thread local key ssh_config: %T", data) 41 | } 42 | val, err := cfg.Attr("username") 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | if trimQuotes(val.String()) != "uname" { 47 | t.Fatalf("unexpected value for key 'foo': %s", val.String()) 48 | } 49 | }, 50 | }, 51 | 52 | { 53 | name: "ssh_config returned value", 54 | script: `cfg = ssh_config(username="uname", private_key_path="path")`, 55 | eval: func(t *testing.T, script string) { 56 | exe := New() 57 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 58 | t.Fatal(err) 59 | } 60 | data := exe.result["cfg"] 61 | if data == nil { 62 | t.Fatal("ssh_config function not returning value") 63 | } 64 | cfg, ok := data.(*starlarkstruct.Struct) 65 | if !ok { 66 | t.Fatalf("unexpected type for thread local key ssh_config: %T", data) 67 | } 68 | val, err := cfg.Attr("private_key_path") 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | if trimQuotes(val.String()) != "path" { 73 | t.Fatalf("unexpected value for key %s in ssh_config", val.String()) 74 | } 75 | }, 76 | }, 77 | 78 | { 79 | name: "crash_config default", 80 | script: `one = 1`, 81 | eval: func(t *testing.T, script string) { 82 | exe := New() 83 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 84 | t.Fatal(err) 85 | } 86 | data := exe.thread.Local(identifiers.sshCfg) 87 | if data == nil { 88 | t.Fatal("default ssh_config is not present in thread local") 89 | } 90 | }, 91 | }, 92 | } 93 | 94 | for _, test := range tests { 95 | t.Run(test.name, func(t *testing.T) { 96 | test.eval(t, test.script) 97 | }) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # Crash Diagnostics Roadmap 2 | This project has been in development through several releases. The release cadance is designed to allow the implemented features to mature overtime and lessen technical debts. Each release series will consist of alpha and beta releases (when necessary) before each major release to allow time for the code to be properly exercized by the community. 3 | 4 | This roadmap has a short and medium term views of the type of design and functionalities that the tool should support prior to a `1.0` release. 5 | 6 | ## v0.1.x-Releases 7 | These releases are meant to introduce new features and introduce fundamental designs that will allow the tool to feature-scale. This will change often and may break backward compactivity until a GA version is reached. Starting with `v0.1.0-alpha.0`, the main themes for `v0.1.x` release series are: 8 | 9 | * Feedback - continued requirement gathering from early adopters. 10 | * Documentation - solidify the documentation early for easy usage. 11 | * Standardization - ensure that script directives are consistent and predictable for improved usability. 12 | * Feature growth - continue to improve on current features and add new ones. 13 | 14 | ## v0.2.x-Releases 15 | The 0.2.x releases will continue to provide stability of features introduced in previous release series (v0.1.0). 16 | There will be two main themes in this release: 17 | * Introduction of new `KUBEGET` directive to query objects from the API server 18 | * Start a collection of Diagnostics files for troubleshooting recipes 19 | * Redesign the execution backend into a pluggable system allowing different execution runtime (i.e. SSH, HTTP, gRPC, cloud provider, etc) 20 | 21 | ### Features 22 | The following additional features are also planned for this series. 23 | * Go API - ensure a clear API surface for code reuse and embedding. 24 | * Pluggable executors - make executors (the code that executes the translated script directives) work using pluggable API (i.e. interface) 25 | 26 | 27 | ## v0.3.x-Releases 28 | This series of release will see the redsign of the internals of Crash Diagnostics to move away from a custom configuration and adopt the [Starlark](https://github.com/bazelbuild/starlark) language (a dialect of Python): 29 | * Refactor the internal implementation to use Starlark 30 | * Introduce/implement several Starlark functions to replace the directives from previous file format. 31 | * Develop ability to extract data/logs from Cluster-API managed clusters 32 | 33 | See the Google Doc design document [here](https://docs.google.com/document/d/1pqYOdTf6ZIT_GSis-AVzlOTm3kyyg-32-seIfULaYEs/edit?usp=sharing). 34 | 35 | 36 | ## v0.4.x-Releases 37 | This series of releases will explore optimization features: 38 | * Parsing and execution optimization (i.e. parallel execution) 39 | * A Uniform retry strategies (smart enough to requeue actions when failed) 40 | 41 | ## v0.5.x-Releases 42 | Exploring other interesting ideas: 43 | * Automated diagnostics (would be nice) 44 | * And more... -------------------------------------------------------------------------------- /exec/executor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package exec 5 | 6 | import ( 7 | "os" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestExampleScripts(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | scriptPath string 16 | args ArgMap 17 | }{ 18 | { 19 | name: "api objects", 20 | scriptPath: "../examples/kind-api-objects.crsh", 21 | args: ArgMap{"kubecfg": support.KindKubeConfigFile()}, 22 | }, 23 | { 24 | name: "pod logs", 25 | scriptPath: "../examples/pod-logs.crsh", 26 | args: ArgMap{"kubecfg": support.KindKubeConfigFile()}, 27 | }, 28 | { 29 | name: "script with args", 30 | scriptPath: "../examples/script-args.crsh", 31 | args: ArgMap{ 32 | "workdir": "/tmp/crashargs", 33 | "kubecfg": support.KindKubeConfigFile(), 34 | "output": "/tmp/craslogs.tar.gz", 35 | }, 36 | }, 37 | { 38 | name: "host-list provider", 39 | scriptPath: "../examples/host-list-provider.crsh", 40 | args: ArgMap{ 41 | "kubecfg": support.KindKubeConfigFile(), 42 | "ssh_pk_path": support.PrivateKeyPath(), 43 | "ssh_port": support.PortValue(), 44 | }, 45 | }, 46 | { 47 | name: "kind-capi-bootstrap", 48 | scriptPath: "../examples/kind-capi-bootstrap.crsh", 49 | args: ArgMap{"cluster_name": support.ResourceName()}, 50 | }, 51 | } 52 | 53 | for _, test := range tests { 54 | t.Run(test.name, func(t *testing.T) { 55 | file, err := os.Open(test.scriptPath) 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | defer file.Close() 60 | if err := ExecuteFile(file, test.args, false); err != nil { 61 | t.Fatal(err) 62 | } 63 | }) 64 | } 65 | } 66 | 67 | func TestExecute(t *testing.T) { 68 | tests := []struct { 69 | name string 70 | script string 71 | exec func(t *testing.T, script string) 72 | }{ 73 | { 74 | name: "execute single script", 75 | script: `result = run_local("echo 'Hello World!'")`, 76 | exec: func(t *testing.T, script string) { 77 | if err := Execute("run_local", strings.NewReader(script), ArgMap{}, false); err != nil { 78 | t.Fatal(err) 79 | } 80 | }, 81 | }, 82 | { 83 | name: "execute with modules", 84 | script: `result = multiply(2, 3)`, 85 | exec: func(t *testing.T, script string) { 86 | mod := ` 87 | def multiply(x, y): 88 | log (msg="{} * {} = {}".format(x,y,x*y)) 89 | ` 90 | if err := ExecuteWithModules( 91 | "multiply", 92 | strings.NewReader(script), 93 | ArgMap{}, 94 | false, 95 | StarlarkModule{Name: "lib", Source: strings.NewReader(mod)}); err != nil { 96 | t.Fatal(err) 97 | } 98 | }, 99 | }, 100 | } 101 | 102 | for _, test := range tests { 103 | t.Run(test.name, func(t *testing.T) { 104 | test.exec(t, test.script) 105 | }) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /starlark/kube_nodes_provider.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 12 | 13 | "go.starlark.net/starlark" 14 | "go.starlark.net/starlarkstruct" 15 | ) 16 | 17 | // KubeNodesProviderFn is a built-in starlark function that collects compute resources from a k8s cluster 18 | // Starlark format: kube_nodes_provider([kube_config=kube_config(), ssh_config=ssh_config(), names=["foo", "bar], labels=["bar", "baz"]]) 19 | func KubeNodesProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | 21 | var names, labels *starlark.List 22 | var kubeConfig, sshConfig *starlarkstruct.Struct 23 | 24 | if err := starlark.UnpackArgs( 25 | identifiers.kubeNodesProvider, args, kwargs, 26 | "names?", &names, 27 | "labels?", &labels, 28 | "kube_config?", &kubeConfig, 29 | "ssh_config?", &sshConfig, 30 | ); err != nil { 31 | return starlark.None, fmt.Errorf("failed to read args: %w", err) 32 | } 33 | 34 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 35 | if !ok || ctx == nil { 36 | return starlark.None, errors.New("script context not found") 37 | } 38 | 39 | if kubeConfig == nil { 40 | kubeConfig = thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 41 | } 42 | path, err := getKubeConfigPathFromStruct(kubeConfig) 43 | if err != nil { 44 | return starlark.None, fmt.Errorf("failed to kubeconfig: %w", err) 45 | } 46 | 47 | if sshConfig == nil { 48 | sshConfig = thread.Local(identifiers.sshCfg).(*starlarkstruct.Struct) 49 | } 50 | 51 | return newKubeNodesProvider(ctx, path, sshConfig, toSlice(names), toSlice(labels)) 52 | } 53 | 54 | // newKubeNodesProvider returns a struct with k8s cluster node provider info 55 | func newKubeNodesProvider(ctx context.Context, kubeconfig string, sshConfig *starlarkstruct.Struct, names, labels []string) (*starlarkstruct.Struct, error) { 56 | 57 | searchParams := k8s.SearchParams{ 58 | Names: names, 59 | Labels: labels, 60 | } 61 | nodeAddresses, err := k8s.GetNodeAddresses(ctx, kubeconfig, searchParams.Names, searchParams.Labels) 62 | if err != nil { 63 | return nil, fmt.Errorf("could not fetch node addresses: %w", err) 64 | } 65 | 66 | // dictionary for node provider struct 67 | kubeNodesProviderDict := starlark.StringDict{ 68 | "kind": starlark.String(identifiers.kubeNodesProvider), 69 | "transport": starlark.String("ssh"), 70 | identifiers.sshCfg: sshConfig, 71 | } 72 | 73 | // add node info to dictionary 74 | var nodeIps []starlark.Value 75 | for _, node := range nodeAddresses { 76 | nodeIps = append(nodeIps, starlark.String(node)) 77 | } 78 | kubeNodesProviderDict["hosts"] = starlark.NewList(nodeIps) 79 | 80 | return starlarkstruct.FromStringDict(starlark.String(identifiers.kubeNodesProvider), kubeNodesProviderDict), nil 81 | } 82 | -------------------------------------------------------------------------------- /starlark/kube_port_forward.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "math/rand" 10 | "net" 11 | "time" 12 | 13 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 14 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | "k8s.io/apimachinery/pkg/labels" 16 | 17 | "go.starlark.net/starlark" 18 | "go.starlark.net/starlarkstruct" 19 | ) 20 | 21 | // KubePortForwardrFn is a built-in starlark function that collects compute resources from a k8s cluster 22 | // Starlark format: kube_port_forward_config(service="bar", target_port=664, [namespace="foo"]) 23 | func KubePortForwardrFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 24 | 25 | var namespace, service string 26 | var targetPort int 27 | 28 | if err := starlark.UnpackArgs( 29 | identifiers.kubePortForwardConfig, args, kwargs, 30 | "namespace?", &namespace, 31 | "service", &service, 32 | "target_port", &targetPort, 33 | ); err != nil { 34 | return starlark.None, fmt.Errorf("failed to read args: %w", err) 35 | } 36 | 37 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 38 | if !ok || ctx == nil { 39 | return starlark.None, fmt.Errorf("script context not found") 40 | } 41 | 42 | kubeConfig := thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 43 | kubeConfigPath, err := getKubeConfigPathFromStruct(kubeConfig) 44 | if err != nil { 45 | return starlark.None, fmt.Errorf("failed to kubeconfig: %w", err) 46 | } 47 | 48 | client, err := k8s.New(kubeConfigPath) 49 | if err != nil { 50 | return nil, fmt.Errorf("could not initialize search client: %w", err) 51 | } 52 | svc, err := client.Typed.CoreV1().Services(namespace).Get(ctx, service, v1.GetOptions{}) 53 | if err != nil { 54 | return nil, fmt.Errorf("could not get service: %w", err) 55 | } 56 | 57 | selector := labels.SelectorFromSet(svc.Spec.Selector) 58 | 59 | pods, err := client.Typed.CoreV1().Pods(svc.Namespace).List(ctx, v1.ListOptions{LabelSelector: selector.String()}) 60 | if err != nil || len(pods.Items) == 0 { 61 | return nil, fmt.Errorf("could not list pods: %w", err) 62 | } 63 | 64 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 65 | var randomPort int 66 | var found bool 67 | 68 | for !found { 69 | // get a random port between 49152-65535 70 | randomPort = r.Intn(16383) + 49152 71 | address := fmt.Sprintf(":%d", randomPort) 72 | listener, err := net.Listen("tcp", address) 73 | if err != nil { 74 | continue 75 | } 76 | defer listener.Close() 77 | found = true 78 | } 79 | 80 | tunnelConfigDict := starlark.StringDict{ 81 | "namespace": starlark.String(namespace), 82 | "pod_name": starlark.String(pods.Items[0].Name), 83 | "target_port": starlark.MakeInt(targetPort), 84 | "local_port": starlark.MakeInt(randomPort), 85 | } 86 | 87 | return starlarkstruct.FromStringDict(starlark.String(identifiers.kubePortForwardConfig), tunnelConfigDict), nil 88 | } 89 | -------------------------------------------------------------------------------- /starlark/kube_exec.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "path/filepath" 11 | "time" 12 | 13 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 14 | "go.starlark.net/starlark" 15 | "go.starlark.net/starlarkstruct" 16 | ) 17 | 18 | // KubeExecFn is a starlark built-in for executing command in target K8s pods 19 | func KubeExecFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | var namespace, pod, container, workdir, outputfile string 21 | var timeout int 22 | var command *starlark.List 23 | var kubeConfig *starlarkstruct.Struct 24 | 25 | if err := starlark.UnpackArgs( 26 | identifiers.kubeExec, args, kwargs, 27 | "namespace?", &namespace, 28 | "pod", &pod, 29 | "container?", &container, 30 | "cmd", &command, 31 | "workdir?", &workdir, 32 | "output_file?", &outputfile, 33 | "kube_config?", &kubeConfig, 34 | "timeout_in_seconds?", &timeout, 35 | ); err != nil { 36 | return starlark.None, fmt.Errorf("failed to read args: %w", err) 37 | } 38 | 39 | if namespace == "" { 40 | namespace = "default" 41 | } 42 | if timeout == 0 { 43 | //Default timeout if not specified is 2 Minutes 44 | timeout = 120 45 | } 46 | 47 | if len(workdir) == 0 { 48 | //Defaults to crashd_config.workdir or /tmp/crashd 49 | if dir, err := getWorkdirFromThread(thread); err == nil { 50 | workdir = dir 51 | } 52 | } 53 | 54 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 55 | if !ok || ctx == nil { 56 | return starlark.None, errors.New("script context not found") 57 | } 58 | 59 | if kubeConfig == nil { 60 | kubeConfig = thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 61 | } 62 | path, err := getKubeConfigPathFromStruct(kubeConfig) 63 | if err != nil { 64 | return starlark.None, fmt.Errorf("failed to get kubeconfig: %w", err) 65 | } 66 | clusterCtxName := getKubeConfigContextNameFromStruct(kubeConfig) 67 | 68 | execOpts := k8s.ExecOptions{ 69 | Namespace: namespace, 70 | Podname: pod, 71 | ContainerName: container, 72 | Command: toSlice(command), 73 | Timeout: time.Duration(timeout) * time.Second, 74 | } 75 | executor, err := k8s.NewExecutor(path, clusterCtxName, execOpts) 76 | if err != nil { 77 | return starlark.None, fmt.Errorf("could not initialize search client: %w", err) 78 | } 79 | 80 | outputFilePath := filepath.Join(trimQuotes(workdir), outputfile) 81 | if outputfile == "" { 82 | outputFilePath = filepath.Join(trimQuotes(workdir), pod+".out") 83 | } 84 | err = executor.ExecCommand(ctx, outputFilePath, execOpts) 85 | 86 | return starlarkstruct.FromStringDict( 87 | starlark.String(identifiers.kubeCapture), 88 | starlark.StringDict{ 89 | "file": starlark.String(outputFilePath), 90 | "error": func() starlark.String { 91 | if err != nil { 92 | return starlark.String(err.Error()) 93 | } 94 | return "" 95 | }(), 96 | }), nil 97 | } 98 | -------------------------------------------------------------------------------- /starlark/set_defaults_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "strings" 8 | 9 | "go.starlark.net/starlark" 10 | "go.starlark.net/starlarkstruct" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/ginkgo/extensions/table" 14 | . "github.com/onsi/gomega" 15 | ) 16 | 17 | var _ = Describe("set_defaults", func() { 18 | 19 | DescribeTable("sets the inputs as default", func(crashdScript string) { 20 | e := New() 21 | err := e.Exec("test.set_defaults", strings.NewReader(crashdScript)) 22 | Expect(err).NotTo(HaveOccurred()) 23 | 24 | kubeConfig := e.thread.Local(identifiers.kubeCfg) 25 | Expect(kubeConfig).NotTo(BeNil()) 26 | Expect(kubeConfig).To(BeAssignableToTypeOf(&starlarkstruct.Struct{})) 27 | 28 | sshConfig := e.thread.Local(identifiers.sshCfg) 29 | Expect(sshConfig).NotTo(BeNil()) 30 | Expect(sshConfig).To(BeAssignableToTypeOf(&starlarkstruct.Struct{})) 31 | 32 | resources := e.thread.Local(identifiers.resources) 33 | Expect(resources).NotTo(BeNil()) 34 | Expect(resources).To(BeAssignableToTypeOf(&starlark.List{})) 35 | }, 36 | Entry("single inputs", ` 37 | kube_cfg = kube_config(path="/foo/bar") 38 | set_defaults(kube_cfg) 39 | 40 | ssh_cfg = ssh_config(username="baz") 41 | set_defaults(ssh_cfg) 42 | 43 | res = resources(hosts=["127.0.0.1","localhost"]) 44 | set_defaults(res) 45 | `), 46 | Entry("single inputs with inline declarations", ` 47 | set_defaults(kube_config(path="/foo/bar")) 48 | set_defaults(ssh_config(username="baz")) 49 | set_defaults(resources(hosts=["127.0.0.1","localhost"])) 50 | `), 51 | Entry("multiple inputs with inline declarations", ` 52 | set_defaults(kube_config(path="/foo/bar"), ssh_config(username="baz")) 53 | set_defaults(resources(hosts=["127.0.0.1","localhost"])) 54 | `), 55 | Entry("multiple inputs with inline declarations", ` 56 | set_defaults(ssh_config(username="baz")) 57 | set_defaults(kube_config(path="/foo/bar"), resources(hosts=["127.0.0.1","localhost"])) 58 | `), 59 | ) 60 | 61 | Context("When a default ssh_config is not declared", func() { 62 | 63 | It("fails to evaluate resources as a set_defaults option", func() { 64 | e := New() 65 | err := e.Exec("test.set_defaults", strings.NewReader(` 66 | ssh_cfg = ssh_config(username="baz") 67 | set_defaults(resources = resources(hosts=["127.0.0.1","localhost"]), ssh_config = ssh_cfg) 68 | `)) 69 | Expect(err).To(HaveOccurred()) 70 | }) 71 | }) 72 | 73 | DescribeTable("throws an error", func(crashdScript string) { 74 | e := New() 75 | err := e.Exec("test.set_defaults", strings.NewReader(crashdScript)) 76 | Expect(err).To(HaveOccurred()) 77 | 78 | kubeConfig := e.thread.Local(identifiers.kubeCfg) 79 | Expect(kubeConfig).NotTo(BeNil()) 80 | 81 | sshConfig := e.thread.Local(identifiers.sshCfg) 82 | Expect(sshConfig).NotTo(BeNil()) 83 | }, Entry("no input", ` 84 | kube_cfg = kube_config(path="/foo/bar") 85 | ssh_cfg = ssh_config(username="baz") 86 | set_defaults() 87 | `), 88 | Entry("incorrect input", ` 89 | set_defaults("/foo") 90 | `), 91 | Entry("keyword inputs", ` 92 | set_defaults(kube_config = kube_config(path="/foo/bar")) 93 | `), 94 | ) 95 | }) 96 | -------------------------------------------------------------------------------- /k8s/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | ) 10 | 11 | func TestClientNew(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | test func(*testing.T) 15 | }{ 16 | { 17 | name: "client with no cluster context", 18 | test: func(t *testing.T) { 19 | client, err := New(support.KindKubeConfigFile(), support.KindClusterContextName()) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | results, err := client.Search(context.TODO(), SearchParams{Kinds: []string{"pods"}}) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | count := 0 28 | for _, result := range results { 29 | count = len(result.List.Items) + count 30 | } 31 | t.Logf("found %d objects", count) 32 | }, 33 | }, 34 | { 35 | name: "client with cluster context", 36 | test: func(t *testing.T) { 37 | client, err := New(support.KindKubeConfigFile(), support.KindClusterContextName()) 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | results, err := client.Search(context.TODO(), SearchParams{Kinds: []string{"pods"}}) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | count := 0 46 | for _, result := range results { 47 | count = len(result.List.Items) + count 48 | } 49 | t.Logf("found %d objects", count) 50 | }, 51 | }, 52 | } 53 | 54 | for _, test := range tests { 55 | t.Run(test.name, func(t *testing.T) { 56 | test.test(t) 57 | }) 58 | } 59 | } 60 | 61 | func TestClient_Search(t *testing.T) { 62 | tests := []struct { 63 | name string 64 | params SearchParams 65 | shouldFail bool 66 | eval func(t *testing.T, results []SearchResult) 67 | }{ 68 | { 69 | name: "empty params", 70 | params: SearchParams{}, 71 | shouldFail: true, 72 | }, 73 | { 74 | name: "groups only", 75 | params: SearchParams{Groups: []string{"apps"}}, 76 | eval: func(t *testing.T, results []SearchResult) { 77 | if len(results) == 0 { 78 | t.Errorf("no objects found") 79 | } 80 | count := 0 81 | for _, result := range results { 82 | count = len(result.List.Items) + count 83 | } 84 | t.Logf("found %d objects", count) 85 | }, 86 | }, 87 | { 88 | name: "kinds (resources) only", 89 | params: SearchParams{Kinds: []string{"pods"}}, 90 | eval: func(t *testing.T, results []SearchResult) { 91 | if len(results) == 0 { 92 | t.Errorf("no objects found") 93 | } 94 | count := 0 95 | for _, result := range results { 96 | count = len(result.List.Items) + count 97 | } 98 | t.Logf("found %d objects", count) 99 | }, 100 | }, 101 | } 102 | 103 | for _, test := range tests { 104 | t.Run(test.name, func(t *testing.T) { 105 | client, err := New(support.KindKubeConfigFile()) 106 | if err != nil { 107 | t.Fatal(err) 108 | } 109 | results, err := client.Search(context.TODO(), test.params) 110 | if err != nil && !test.shouldFail { 111 | t.Fatal(err) 112 | } 113 | if test.eval != nil { 114 | test.eval(t, results) 115 | } 116 | }) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /k8s/search_result.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "go.starlark.net/starlark" 8 | "go.starlark.net/starlarkstruct" 9 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 10 | "k8s.io/apimachinery/pkg/runtime/schema" 11 | ) 12 | 13 | // SearchResult is the object representation of the kubernetes objects 14 | // returned by querying the API server 15 | type SearchResult struct { 16 | ListKind string 17 | ResourceName string 18 | ResourceKind string 19 | GroupVersionResource schema.GroupVersionResource 20 | List *unstructured.UnstructuredList 21 | Namespaced bool 22 | Namespace string 23 | } 24 | 25 | // ToStarlarkValue converts the SearchResult object to a starlark dictionary 26 | func (sr SearchResult) ToStarlarkValue() *starlarkstruct.Struct { 27 | var values []starlark.Value 28 | listDict := starlark.StringDict{} 29 | 30 | if sr.List != nil { 31 | for _, item := range sr.List.Items { 32 | values = append(values, convertToStruct(item)) 33 | } 34 | listDict = starlark.StringDict{ 35 | "Object": convertToStarlarkPrimitive(sr.List.Object), 36 | "Items": starlark.NewList(values), 37 | } 38 | } 39 | listStruct := starlarkstruct.FromStringDict(starlarkstruct.Default, listDict) 40 | 41 | grValDict := starlark.StringDict{ 42 | "Group": starlark.String(sr.GroupVersionResource.Group), 43 | "Version": starlark.String(sr.GroupVersionResource.Version), 44 | "Resource": starlark.String(sr.GroupVersionResource.Resource), 45 | } 46 | 47 | dict := starlark.StringDict{ 48 | "ListKind": starlark.String(sr.ListKind), 49 | "ResourceName": starlark.String(sr.ResourceName), 50 | "ResourceKind": starlark.String(sr.ResourceKind), 51 | "Namespaced": starlark.Bool(sr.Namespaced), 52 | "Namespace": starlark.String(sr.Namespace), 53 | "GroupVersionResource": starlarkstruct.FromStringDict(starlarkstruct.Default, grValDict), 54 | "List": listStruct, 55 | } 56 | 57 | return starlarkstruct.FromStringDict(starlark.String("search_result"), dict) 58 | } 59 | 60 | // convertToStruct returns a starlark struct constructed from the contents of the input. 61 | func convertToStruct(obj unstructured.Unstructured) starlark.Value { 62 | return convertToStarlarkPrimitive(obj.Object) 63 | } 64 | 65 | // convertToStarlarkPrimitive returns a starlark value based on the Golang type passed 66 | // as input to the function. 67 | func convertToStarlarkPrimitive(input interface{}) starlark.Value { 68 | var value starlark.Value 69 | switch val := input.(type) { 70 | case string: 71 | value = starlark.String(val) 72 | case int, int32, int64: 73 | value = starlark.MakeInt64(val.(int64)) 74 | case bool: 75 | value = starlark.Bool(val) 76 | case []interface{}: 77 | var structs []starlark.Value 78 | for _, i := range val { 79 | structs = append(structs, convertToStarlarkPrimitive(i)) 80 | } 81 | value = starlark.NewList(structs) 82 | case map[string]interface{}: 83 | dict := starlark.StringDict{} 84 | for k, v := range val { 85 | dict[k] = convertToStarlarkPrimitive(v) 86 | } 87 | value = starlarkstruct.FromStringDict(starlarkstruct.Default, dict) 88 | default: 89 | value = starlark.None 90 | } 91 | return value 92 | } 93 | -------------------------------------------------------------------------------- /starlark/crashd_config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/vmware-tanzu/crash-diagnostics/ssh" 12 | "github.com/vmware-tanzu/crash-diagnostics/util" 13 | "go.starlark.net/starlark" 14 | "go.starlark.net/starlarkstruct" 15 | ) 16 | 17 | // addDefaultCrashdConf initializes a Starlark Dict with default 18 | // crashd_config configuration data 19 | func addDefaultCrashdConf(thread *starlark.Thread) error { 20 | args := []starlark.Tuple{ 21 | {starlark.String("gid"), starlark.String(getGid())}, 22 | {starlark.String("uid"), starlark.String(getUid())}, 23 | {starlark.String("workdir"), starlark.String(defaults.workdir)}, 24 | } 25 | 26 | _, err := crashdConfigFn(thread, nil, nil, args) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | return nil 32 | } 33 | 34 | // crashdConfigFn is built-in starlark function that saves and returns the kwargs as a struct value. 35 | // Starlark format: crashd_config(workdir=path, default_shell=shellpath, requires=["command0",...,"commandN"]) 36 | func crashdConfigFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 37 | var workdir, gid, uid, defaultShell string 38 | var useSSHAgent bool 39 | requires := starlark.NewList([]starlark.Value{}) 40 | 41 | if err := starlark.UnpackArgs( 42 | identifiers.crashdCfg, args, kwargs, 43 | "workdir?", &workdir, 44 | "gid?", &gid, 45 | "uid?", &uid, 46 | "default_shell?", &defaultShell, 47 | "requires?", &requires, 48 | "use_ssh_agent?", &useSSHAgent, 49 | ); err != nil { 50 | return starlark.None, fmt.Errorf("%s: %s", identifiers.crashdCfg, err) 51 | } 52 | 53 | // validate 54 | if len(workdir) == 0 { 55 | workdir = defaults.workdir 56 | } 57 | 58 | if len(gid) == 0 { 59 | gid = getGid() 60 | } 61 | 62 | if len(uid) == 0 { 63 | uid = getUid() 64 | } 65 | 66 | if err := makeCrashdWorkdir(workdir); err != nil { 67 | return starlark.None, fmt.Errorf("%s: %s", identifiers.crashdCfg, err) 68 | } 69 | 70 | if useSSHAgent { 71 | agent, err := ssh.StartAgent() 72 | if err != nil { 73 | return starlark.None, fmt.Errorf("failed to start ssh agent: %w", err) 74 | } 75 | 76 | // sets the ssh_agent variable in the current Starlark thread 77 | thread.SetLocal(identifiers.sshAgent, agent) 78 | } 79 | 80 | workdir, err := util.ExpandPath(workdir) 81 | if err != nil { 82 | return starlark.None, err 83 | } 84 | 85 | cfgStruct := starlarkstruct.FromStringDict(starlark.String(identifiers.crashdCfg), starlark.StringDict{ 86 | "workdir": starlark.String(workdir), 87 | "gid": starlark.String(gid), 88 | "uid": starlark.String(uid), 89 | "default_shell": starlark.String(defaultShell), 90 | "requires": requires, 91 | }) 92 | 93 | // save values to be used as default 94 | thread.SetLocal(identifiers.crashdCfg, cfgStruct) 95 | 96 | return cfgStruct, nil 97 | } 98 | 99 | func makeCrashdWorkdir(path string) error { 100 | if _, err := os.Stat(path); err != nil && !os.IsNotExist(err) { 101 | return err 102 | } 103 | logrus.Debugf("creating working directory %s", path) 104 | if err := os.MkdirAll(path, 0744); err != nil && !os.IsExist(err) { 105 | return err 106 | } 107 | 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /ssh/test_support.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package ssh 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | func makeLocalTestDir(t *testing.T, dir string) { 16 | t.Logf("creating local test dir: %s", dir) 17 | if err := os.MkdirAll(dir, 0744); err != nil && !os.IsExist(err) { 18 | t.Fatalf("makeLocalTestDir: failed to create dir: %s", err) 19 | } 20 | t.Logf("dir created: %s", dir) 21 | } 22 | 23 | func MakeLocalTestFile(t *testing.T, filePath, content string) { 24 | srcDir := filepath.Dir(filePath) 25 | if len(srcDir) > 0 && srcDir != "." { 26 | makeLocalTestDir(t, srcDir) 27 | } 28 | 29 | t.Logf("creating local test file: %s", filePath) 30 | file, err := os.Create(filePath) 31 | if err != nil { 32 | t.Fatalf("MakeLocalTestFile: failed to create file: %s", err) 33 | } 34 | defer file.Close() 35 | buf := bytes.NewBufferString(content) 36 | if _, err := buf.WriteTo(file); err != nil { 37 | t.Fatal(err) 38 | } 39 | t.Logf("local test file created: %s", file.Name()) 40 | } 41 | 42 | func RemoveLocalTestFile(t *testing.T, fileName string) { 43 | t.Logf("removing local test path: %s", fileName) 44 | if err := os.RemoveAll(fileName); err != nil && !os.IsNotExist(err) { 45 | t.Fatalf("RemoveLocalTestFile: failed: %s", err) 46 | } 47 | } 48 | 49 | func makeRemoteTestSSHDir(t *testing.T, args SSHArgs, dir string) { 50 | t.Logf("creating remote test dir over SSH: %s", dir) 51 | _, err := Run(args, nil, fmt.Sprintf(`mkdir -p %s`, dir)) 52 | if err != nil { 53 | t.Fatalf("makeRemoteTestSSHDir: failed: %s", err) 54 | } 55 | // validate 56 | result, err := Run(args, nil, fmt.Sprintf(`stat %s`, dir)) 57 | if err != nil { 58 | t.Fatalf("makeRemoteTestSSHDir %s", err) 59 | } 60 | t.Logf("dir created: %s", result) 61 | } 62 | 63 | func MakeRemoteTestSSHFile(t *testing.T, args SSHArgs, filePath, content string) { 64 | MakeRemoteTestSSHDir(t, args, filePath) 65 | 66 | t.Logf("creating test file over SSH: %s", filePath) 67 | _, err := Run(args, nil, fmt.Sprintf(`echo '%s' > %s`, content, filePath)) 68 | if err != nil { 69 | t.Fatalf("MakeRemoteTestSSHFile: failed: %s", err) 70 | } 71 | 72 | result, _ := Run(args, nil, fmt.Sprintf(`ls %s`, filePath)) 73 | t.Logf("file created: %s", result) 74 | } 75 | 76 | func MakeRemoteTestSSHDir(t *testing.T, args SSHArgs, filePath string) { 77 | dir := filepath.Dir(filePath) 78 | if len(dir) > 0 && dir != "." { 79 | makeRemoteTestSSHDir(t, args, dir) 80 | } 81 | } 82 | 83 | func AssertRemoteTestSSHFile(t *testing.T, args SSHArgs, filePath string) { 84 | t.Logf("stat remote SSH test file: %s", filePath) 85 | _, err := Run(args, nil, fmt.Sprintf(`stat %s`, filePath)) 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | } 90 | 91 | func RemoveRemoteTestSSHFile(t *testing.T, args SSHArgs, fileName string) { 92 | t.Logf("removing test file over SSH: %s", fileName) 93 | _, err := Run(args, nil, fmt.Sprintf(`rm -rf %s`, fileName)) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | } 98 | 99 | func getTestFileContent(t *testing.T, fileName string) string { 100 | file, err := os.Open(fileName) 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | buf := new(bytes.Buffer) 105 | if _, err := buf.ReadFrom(file); err != nil { 106 | t.Fatal(err) 107 | } 108 | return strings.TrimSpace(buf.String()) 109 | } 110 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vmware-tanzu/crash-diagnostics 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/kcp-dev/kcp/cli v0.27.1 9 | github.com/kcp-dev/kcp/sdk v0.27.1 10 | github.com/onsi/ginkgo v1.16.5 11 | github.com/onsi/gomega v1.36.2 12 | github.com/sirupsen/logrus v1.9.3 13 | github.com/spf13/cobra v1.8.1 14 | github.com/vladimirvivien/gexe v0.4.0 15 | go.starlark.net v0.0.0-20241226192728-8dfa5b98479f 16 | k8s.io/api v0.32.1 17 | k8s.io/apimachinery v0.32.1 18 | k8s.io/cli-runtime v0.32.1 19 | k8s.io/client-go v0.32.1 20 | k8s.io/utils v0.0.0-20241210054802-24370beab758 21 | ) 22 | 23 | require ( 24 | github.com/pkg/errors v0.9.1 // indirect 25 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 26 | ) 27 | 28 | require ( 29 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 30 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 31 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 32 | github.com/fsnotify/fsnotify v1.7.0 // indirect 33 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 34 | github.com/go-logr/logr v1.4.2 // indirect 35 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 36 | github.com/go-openapi/jsonreference v0.20.2 // indirect 37 | github.com/go-openapi/swag v0.23.0 // indirect 38 | github.com/gogo/protobuf v1.3.2 // indirect 39 | github.com/golang/protobuf v1.5.4 // indirect 40 | github.com/google/gnostic-models v0.6.8 // indirect 41 | github.com/google/go-cmp v0.6.0 // indirect 42 | github.com/google/gofuzz v1.2.0 // indirect 43 | github.com/google/uuid v1.6.0 // indirect 44 | github.com/gorilla/websocket v1.5.0 // indirect 45 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 46 | github.com/josharian/intern v1.0.0 // indirect 47 | github.com/json-iterator/go v1.1.12 // indirect 48 | github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20250223115924-431177b024f3 // indirect 49 | github.com/kcp-dev/logicalcluster/v3 v3.0.5 50 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 51 | github.com/mailru/easyjson v0.7.7 // indirect 52 | github.com/moby/spdystream v0.5.0 // indirect 53 | github.com/moby/term v0.5.0 // indirect 54 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 55 | github.com/modern-go/reflect2 v1.0.2 // indirect 56 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 57 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 58 | github.com/nxadm/tail v1.4.8 // indirect 59 | github.com/rogpeppe/go-internal v1.13.1 // indirect 60 | github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect 61 | github.com/stretchr/testify v1.10.0 // indirect 62 | github.com/x448/float16 v0.8.4 // indirect 63 | golang.org/x/net v0.36.0 // indirect 64 | golang.org/x/oauth2 v0.23.0 // indirect 65 | golang.org/x/sys v0.30.0 // indirect 66 | golang.org/x/term v0.29.0 // indirect 67 | golang.org/x/text v0.22.0 // indirect 68 | golang.org/x/time v0.7.0 // indirect 69 | google.golang.org/protobuf v1.36.1 // indirect 70 | gopkg.in/inf.v0 v0.9.1 // indirect 71 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 72 | gopkg.in/yaml.v3 v3.0.1 // indirect 73 | k8s.io/apiextensions-apiserver v0.31.6 // indirect 74 | k8s.io/klog/v2 v2.130.1 // indirect 75 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect 76 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 77 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect 78 | sigs.k8s.io/yaml v1.4.0 // indirect 79 | ) 80 | -------------------------------------------------------------------------------- /starlark/capv_provider.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 12 | "github.com/vmware-tanzu/crash-diagnostics/provider" 13 | "go.starlark.net/starlark" 14 | "go.starlark.net/starlarkstruct" 15 | ) 16 | 17 | // CapvProviderFn is a built-in starlark function that collects compute resources from a k8s cluster 18 | // Starlark format: capv_provider(kube_config=kube_config(), ssh_config=ssh_config()[workload_cluster=, namespace=, nodes=["foo", "bar], labels=["bar", "baz"]]) 19 | func CapvProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | 21 | var ( 22 | workloadCluster, namespace string 23 | names, labels *starlark.List 24 | sshConfig, mgmtKubeConfig *starlarkstruct.Struct 25 | ) 26 | 27 | err := starlark.UnpackArgs("capv_provider", args, kwargs, 28 | "ssh_config", &sshConfig, 29 | "mgmt_kube_config", &mgmtKubeConfig, 30 | "workload_cluster?", &workloadCluster, 31 | "namespace?", &namespace, 32 | "labels?", &labels, 33 | "nodes?", &names) 34 | if err != nil { 35 | return starlark.None, fmt.Errorf("failed to unpack input arguments: %w", err) 36 | } 37 | 38 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 39 | if !ok || ctx == nil { 40 | return starlark.None, errors.New("script context not found") 41 | } 42 | 43 | if sshConfig == nil || mgmtKubeConfig == nil { 44 | return starlark.None, errors.New("capv_provider requires the name of the management cluster, the ssh configuration and the management cluster kubeconfig") 45 | } 46 | 47 | if mgmtKubeConfig == nil { 48 | mgmtKubeConfig = thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 49 | } 50 | mgmtKubeConfigPath, err := getKubeConfigPathFromStruct(mgmtKubeConfig) 51 | if err != nil { 52 | return starlark.None, fmt.Errorf("failed to extract management kubeconfig: %w", err) 53 | } 54 | 55 | providerConfigPath, err := provider.KubeConfig(mgmtKubeConfigPath, workloadCluster, namespace) 56 | if err != nil { 57 | return starlark.None, err 58 | } 59 | 60 | nodeAddresses, err := k8s.GetNodeAddresses(ctx, providerConfigPath, toSlice(names), toSlice(labels)) 61 | if err != nil { 62 | return starlark.None, fmt.Errorf("could not fetch host addresses: %w", err) 63 | } 64 | 65 | // dictionary for capv provider struct 66 | capvProviderDict := starlark.StringDict{ 67 | "kind": starlark.String(identifiers.capvProvider), 68 | "transport": starlark.String("ssh"), 69 | "kube_config": starlark.String(providerConfigPath), 70 | } 71 | 72 | // add node info to dictionary 73 | var nodeIps []starlark.Value 74 | for _, node := range nodeAddresses { 75 | nodeIps = append(nodeIps, starlark.String(node)) 76 | } 77 | capvProviderDict["hosts"] = starlark.NewList(nodeIps) 78 | 79 | // add ssh info to dictionary 80 | if _, ok := capvProviderDict[identifiers.sshCfg]; !ok { 81 | capvProviderDict[identifiers.sshCfg] = sshConfig 82 | } 83 | 84 | return starlarkstruct.FromStringDict(starlark.String(identifiers.capvProvider), capvProviderDict), nil 85 | } 86 | 87 | // TODO: Needs to be moved to a single package 88 | func toSlice(list *starlark.List) []string { 89 | var elems []string 90 | if list != nil { 91 | for i := 0; i < list.Len(); i++ { 92 | if val, ok := list.Index(i).(starlark.String); ok { 93 | elems = append(elems, string(val)) 94 | } 95 | } 96 | } 97 | return elems 98 | } 99 | -------------------------------------------------------------------------------- /starlark/resources.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "go.starlark.net/starlark" 11 | "go.starlark.net/starlarkstruct" 12 | ) 13 | 14 | // resourcesFunc is a built-in starlark function that prepares returns compute list of resources. 15 | // Starlark format: resources(provider=) 16 | func resourcesFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 17 | var hosts *starlark.List 18 | var provider *starlarkstruct.Struct 19 | if err := starlark.UnpackArgs( 20 | identifiers.crashdCfg, args, kwargs, 21 | "hosts?", &hosts, 22 | "provider?", &provider, 23 | ); err != nil { 24 | return starlark.None, fmt.Errorf("%s: %s", identifiers.hostListProvider, err) 25 | } 26 | 27 | if hosts == nil && provider == nil { 28 | return starlark.None, fmt.Errorf("%s: hosts or provider argument required", identifiers.resources) 29 | } 30 | 31 | if hosts != nil && provider != nil { 32 | return starlark.None, fmt.Errorf("%s: specify hosts or provider argument", identifiers.resources) 33 | } 34 | 35 | if hosts != nil { 36 | prov, err := hostListProvider(thread, nil, nil, []starlark.Tuple{{starlark.String("hosts"), hosts}}) 37 | if err != nil { 38 | return starlark.None, err 39 | } 40 | provider = prov.(*starlarkstruct.Struct) 41 | } 42 | 43 | // enumerate resources from provider 44 | resources, err := enum(provider) 45 | if err != nil { 46 | return starlark.None, err 47 | } 48 | 49 | return resources, nil 50 | } 51 | 52 | // enum returns a list of structs containing the fully enumerated compute resource 53 | // info needed to execute commands. 54 | func enum(provider *starlarkstruct.Struct) (*starlark.List, error) { 55 | if provider == nil { 56 | return nil, errors.New("missing provider") 57 | } 58 | 59 | var resources []starlark.Value 60 | 61 | kindVal, err := provider.Attr("kind") 62 | if err != nil { 63 | return nil, errors.New("provider missing field kind") 64 | } 65 | 66 | kind := trimQuotes(kindVal.String()) 67 | 68 | switch kind { 69 | case identifiers.hostListProvider, identifiers.kubeNodesProvider, identifiers.capvProvider, identifiers.capaProvider: 70 | hosts, err := provider.Attr("hosts") 71 | if err != nil { 72 | return nil, fmt.Errorf("hosts not found in %s", identifiers.hostListProvider) 73 | } 74 | 75 | hostList, ok := hosts.(*starlark.List) 76 | if !ok { 77 | return nil, fmt.Errorf("%s: unexpected type for hosts: %T", identifiers.hostListProvider, hosts) 78 | } 79 | 80 | transport, err := provider.Attr("transport") 81 | if err != nil { 82 | return nil, fmt.Errorf("transport not found in %s", identifiers.hostListProvider) 83 | } 84 | 85 | sshCfg, err := provider.Attr(identifiers.sshCfg) 86 | if err != nil { 87 | return nil, fmt.Errorf("ssh_config not found in %s", identifiers.hostListProvider) 88 | } 89 | 90 | for i := 0; i < hostList.Len(); i++ { 91 | dict := starlark.StringDict{ 92 | "kind": starlark.String(identifiers.hostResource), 93 | "provider": starlark.String(identifiers.hostListProvider), 94 | "host": hostList.Index(i), 95 | "transport": transport, 96 | "ssh_config": sshCfg, 97 | } 98 | resources = append(resources, starlarkstruct.FromStringDict(starlark.String(identifiers.hostResource), dict)) 99 | } 100 | } 101 | 102 | return starlark.NewList(resources), nil 103 | } 104 | -------------------------------------------------------------------------------- /k8s/executor.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package k8s 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "os" 11 | "time" 12 | 13 | corev1 "k8s.io/api/core/v1" 14 | "k8s.io/client-go/kubernetes/scheme" 15 | "k8s.io/client-go/rest" 16 | "k8s.io/client-go/tools/clientcmd" 17 | "k8s.io/client-go/tools/remotecommand" 18 | ) 19 | 20 | // Executor is a struct that facilitates the execution of commands in Kubernetes pods. 21 | // It uses the SPDYExecutor to stream command 22 | type Executor struct { 23 | Executor remotecommand.Executor 24 | } 25 | 26 | type ExecOptions struct { 27 | Namespace string 28 | Command []string 29 | Podname string 30 | ContainerName string 31 | Config *Config 32 | Timeout time.Duration 33 | } 34 | 35 | func NewExecutor(kubeconfig string, clusterCtxName string, opts ExecOptions) (*Executor, error) { 36 | restCfg, err := restConfig(kubeconfig, clusterCtxName) 37 | if err != nil { 38 | return nil, err 39 | } 40 | setCoreDefaultConfig(restCfg) 41 | restc, err := rest.RESTClientFor(restCfg) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | request := restc.Post(). 47 | Namespace(opts.Namespace). 48 | Resource("pods"). 49 | Name(opts.Podname). 50 | SubResource("exec"). 51 | VersionedParams(&corev1.PodExecOptions{ 52 | Container: opts.ContainerName, 53 | Command: opts.Command, 54 | Stdout: true, 55 | Stderr: true, 56 | TTY: false, 57 | }, scheme.ParameterCodec) 58 | executor, err := remotecommand.NewSPDYExecutor(restCfg, "POST", request.URL()) 59 | if err != nil { 60 | return nil, err 61 | 62 | } 63 | return &Executor{Executor: executor}, nil 64 | } 65 | 66 | // makeRESTConfig creates a new *rest.Config with a k8s context name if one is provided. 67 | func restConfig(fileName, contextName string) (*rest.Config, error) { 68 | if fileName == "" { 69 | return nil, errors.New("kubeconfig file path required") 70 | } 71 | 72 | if contextName != "" { 73 | // create the config object from k8s config path and context 74 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 75 | &clientcmd.ClientConfigLoadingRules{ExplicitPath: fileName}, 76 | &clientcmd.ConfigOverrides{ 77 | CurrentContext: contextName, 78 | }).ClientConfig() 79 | } 80 | 81 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 82 | &clientcmd.ClientConfigLoadingRules{ExplicitPath: fileName}, 83 | &clientcmd.ConfigOverrides{}, 84 | ).ClientConfig() 85 | } 86 | 87 | // ExecCommand executes a command inside a specified Kubernetes pod using the SPDYExecutor. 88 | func (k8sc *Executor) ExecCommand(ctx context.Context, outputFilePath string, execOptions ExecOptions) error { 89 | ctx, cancel := context.WithTimeout(ctx, execOptions.Timeout) 90 | defer cancel() 91 | 92 | file, err := os.OpenFile(outputFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 93 | if err != nil { 94 | return fmt.Errorf("error creating output file: %v", err) 95 | } 96 | defer file.Close() 97 | 98 | // Execute the command and stream the stdout and stderr to the file. Some commands are using stderr. 99 | err = k8sc.Executor.StreamWithContext(ctx, remotecommand.StreamOptions{ 100 | Stdout: file, 101 | Stderr: file, 102 | }) 103 | if err != nil { 104 | if err == context.DeadlineExceeded { 105 | return fmt.Errorf("command execution timed out. command:%s", execOptions.Command) 106 | } 107 | return err 108 | } 109 | 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /starlark/ssh_config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/vmware-tanzu/crash-diagnostics/ssh" 12 | "go.starlark.net/starlark" 13 | "go.starlark.net/starlarkstruct" 14 | ) 15 | 16 | // addDefaultSshConf initializes a Starlark Dict with default 17 | // ssh_config configuration data 18 | func addDefaultSSHConf(thread *starlark.Thread) error { 19 | args := makeDefaultSSHConfig() 20 | conf, err := SshConfigFn(thread, nil, nil, args) 21 | if err != nil { 22 | return err 23 | } 24 | thread.SetLocal(identifiers.sshCfg, conf) 25 | return nil 26 | } 27 | 28 | // SshConfigFn is the backing built-in fn that saves and returns its argument as struct value. 29 | // Starlark format: ssh_config(username=name[, port][, private_key_path][,max_retries][,conn_timeout][,jump_user][,jump_host]) 30 | func SshConfigFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 31 | var uname, port, pkPath, jUser, jHost string 32 | var maxRetries, connTimeout int 33 | 34 | if err := starlark.UnpackArgs( 35 | identifiers.crashdCfg, args, kwargs, 36 | "username", &uname, 37 | "port?", &port, 38 | "private_key_path?", &pkPath, 39 | "jump_user?", &jUser, 40 | "jump_host?", &jHost, 41 | "max_retries?", &maxRetries, 42 | "conn_timeout?", &connTimeout, 43 | ); err != nil { 44 | return starlark.None, fmt.Errorf("%s: %s", identifiers.hostListProvider, err) 45 | } 46 | 47 | // validation 48 | if len(uname) == 0 { 49 | return starlark.None, fmt.Errorf("%s: username required", identifiers.sshCfg) 50 | } 51 | if len(port) == 0 { 52 | port = defaults.sshPort 53 | } 54 | if maxRetries == 0 { 55 | maxRetries = defaults.connRetries 56 | } 57 | if connTimeout == 0 { 58 | connTimeout = defaults.connTimeout 59 | } 60 | if len(pkPath) == 0 { 61 | pkPath = defaults.pkPath 62 | } 63 | 64 | if agentVal := thread.Local(identifiers.sshAgent); agentVal != nil { 65 | agent, ok := agentVal.(ssh.Agent) 66 | if !ok { 67 | return starlark.None, errors.New("unable to fetch ssh-agent") 68 | } 69 | logrus.Debugf("adding key %s to ssh-agent", pkPath) 70 | if err := agent.AddKey(pkPath); err != nil { 71 | return starlark.None, fmt.Errorf("unable to add key %s: %w", pkPath, err) 72 | } 73 | } 74 | 75 | sshConfigDict := starlark.StringDict{ 76 | "username": starlark.String(uname), 77 | "port": starlark.String(port), 78 | "private_key_path": starlark.String(pkPath), 79 | "max_retries": starlark.MakeInt(maxRetries), 80 | "conn_timeout": starlark.MakeInt(connTimeout), 81 | } 82 | if len(jUser) != 0 { 83 | sshConfigDict["jump_user"] = starlark.String(jUser) 84 | } 85 | if len(jHost) != 0 { 86 | sshConfigDict["jump_host"] = starlark.String(jHost) 87 | } 88 | structVal := starlarkstruct.FromStringDict(starlark.String(identifiers.sshCfg), sshConfigDict) 89 | 90 | return structVal, nil 91 | } 92 | 93 | func makeDefaultSSHConfig() []starlark.Tuple { 94 | return []starlark.Tuple{ 95 | {starlark.String("username"), starlark.String(getUsername())}, 96 | {starlark.String("port"), starlark.String("22")}, 97 | {starlark.String("private_key_path"), starlark.String(defaults.pkPath)}, 98 | {starlark.String("max_retries"), starlark.MakeInt(defaults.connRetries)}, 99 | {starlark.String("conn_timeout"), starlark.MakeInt(defaults.connTimeout)}, 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in the Crash Recovery and Diagnostics for Kubernetes project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at oss-coc@vmware.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | 78 | -------------------------------------------------------------------------------- /k8s/search_params.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/sirupsen/logrus" 9 | "go.starlark.net/starlark" 10 | "go.starlark.net/starlarkstruct" 11 | ) 12 | 13 | type SearchParams struct { 14 | Groups []string 15 | Categories []string 16 | Kinds []string 17 | Namespaces []string 18 | Versions []string 19 | Names []string 20 | Labels []string 21 | Containers []string 22 | } 23 | 24 | func (sp SearchParams) ContainsGroup(group string) bool { 25 | return contains(sp.Groups, group) 26 | } 27 | 28 | func (sp SearchParams) ContainsVersion(version string) bool { 29 | return contains(sp.Versions, version) 30 | } 31 | 32 | func (sp SearchParams) ContainsKind(kind string) bool { 33 | return contains(sp.Kinds, kind) 34 | } 35 | 36 | func (sp SearchParams) ContainsContainer(container string) bool { 37 | return contains(sp.Containers, container) 38 | } 39 | 40 | func (sp SearchParams) ContainsName(name string) bool { 41 | return contains(sp.Names, name) 42 | } 43 | 44 | // contains performs a case-insensitive search for the item in the input array 45 | func contains(arr []string, item string) bool { 46 | if len(arr) == 0 { 47 | return false 48 | } 49 | for _, str := range arr { 50 | if strings.EqualFold(str, item) { 51 | return true 52 | } 53 | } 54 | return false 55 | } 56 | 57 | // TODO: Change this to accept a string dictionary instead 58 | func NewSearchParams(p *starlarkstruct.Struct) SearchParams { 59 | var ( 60 | kinds []string 61 | groups []string 62 | names []string 63 | namespaces []string 64 | versions []string 65 | labels []string 66 | containers []string 67 | ) 68 | 69 | groups = parseStructAttr(p, "groups") 70 | kinds = parseStructAttr(p, "kinds") 71 | names = parseStructAttr(p, "names") 72 | namespaces = parseStructAttr(p, "namespaces") 73 | if len(namespaces) == 0 { 74 | namespaces = append(namespaces, "default") 75 | } 76 | versions = parseStructAttr(p, "versions") 77 | labels = parseStructAttr(p, "labels") 78 | containers = parseStructAttr(p, "containers") 79 | 80 | return SearchParams{ 81 | Kinds: kinds, 82 | Groups: groups, 83 | Names: names, 84 | Namespaces: namespaces, 85 | Versions: versions, 86 | Labels: labels, 87 | Containers: containers, 88 | } 89 | } 90 | 91 | func parseStructAttr(p *starlarkstruct.Struct, attrName string) []string { 92 | values := make([]string, 0) 93 | 94 | attrVal, err := p.Attr(attrName) 95 | if err == nil { 96 | values, err = parse(attrVal) 97 | if err != nil { 98 | logrus.Errorf("error while parsing attr %s: %v", attrName, err) 99 | } 100 | } 101 | return values 102 | } 103 | 104 | func parse(inputValue starlark.Value) ([]string, error) { 105 | var values []string 106 | var err error 107 | 108 | switch inputValue.Type() { 109 | case "string": 110 | val, ok := inputValue.(starlark.String) 111 | if !ok { 112 | err = fmt.Errorf("cannot process starlark value %s", inputValue.String()) 113 | break 114 | } 115 | values = append(values, val.GoString()) 116 | case "list": 117 | val, ok := inputValue.(*starlark.List) 118 | if !ok { 119 | err = fmt.Errorf("cannot process starlark value %s", inputValue.String()) 120 | break 121 | } 122 | iter := val.Iterate() 123 | defer iter.Done() 124 | var x starlark.Value 125 | for iter.Next(&x) { 126 | str, _ := x.(starlark.String) 127 | values = append(values, str.GoString()) 128 | } 129 | default: 130 | err = errors.New("unknown input type for parse()") 131 | } 132 | 133 | return values, err 134 | } 135 | -------------------------------------------------------------------------------- /starlark/log_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "bytes" 8 | "log" 9 | "strings" 10 | "testing" 11 | 12 | "go.starlark.net/starlark" 13 | ) 14 | 15 | func TestLogFunc(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | kwargs []starlark.Tuple 19 | test func(*testing.T, []starlark.Tuple) 20 | }{ 21 | { 22 | name: "logging with no prefix", 23 | kwargs: []starlark.Tuple{ 24 | []starlark.Value{starlark.String("msg"), starlark.String("Logging from starlark")}, 25 | }, 26 | test: func(t *testing.T, kwargs []starlark.Tuple) { 27 | var buf bytes.Buffer 28 | logger := log.New(&buf, "", log.Lshortfile) 29 | thread := newTestThreadLocal(t) 30 | thread.SetLocal(identifiers.log, logger) 31 | 32 | if _, err := logFunc(thread, nil, nil, kwargs); err != nil { 33 | t.Fatal(err) 34 | } 35 | if !strings.Contains(buf.String(), "Logging from starlark") { 36 | t.Error("logger has unexpected log msg: ", buf.String()) 37 | } 38 | }, 39 | }, 40 | { 41 | name: "logging with prefix", 42 | kwargs: []starlark.Tuple{ 43 | []starlark.Value{starlark.String("msg"), starlark.String("Logging from starlark with prefix")}, 44 | []starlark.Value{starlark.String("prefix"), starlark.String("INFO")}, 45 | }, 46 | test: func(t *testing.T, kwargs []starlark.Tuple) { 47 | var buf bytes.Buffer 48 | logger := log.New(&buf, "", log.Lshortfile) 49 | thread := newTestThreadLocal(t) 50 | thread.SetLocal(identifiers.log, logger) 51 | 52 | if _, err := logFunc(thread, nil, nil, kwargs); err != nil { 53 | t.Fatal(err) 54 | } 55 | if !strings.Contains(buf.String(), "INFO: Logging from starlark with prefix") { 56 | t.Error("logger has unexpected log prefix and msg: ", buf.String()) 57 | } 58 | }, 59 | }, 60 | } 61 | 62 | for _, test := range tests { 63 | t.Run(test.name, func(t *testing.T) { 64 | test.test(t, test.kwargs) 65 | }) 66 | } 67 | } 68 | 69 | func TestLogFuncScript(t *testing.T) { 70 | tests := []struct { 71 | name string 72 | script string 73 | test func(*testing.T, string) 74 | }{ 75 | { 76 | name: "logging with no prefix", 77 | script: `log(msg="Logging from starlark")`, 78 | test: func(t *testing.T, script string) { 79 | exe := New() 80 | var buf bytes.Buffer 81 | logger := log.New(&buf, "", log.Lshortfile) 82 | exe.thread.SetLocal(identifiers.log, logger) 83 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 84 | t.Fatal(err) 85 | } 86 | if !strings.Contains(buf.String(), "Logging from starlark") { 87 | t.Error("logger has unexpected log msg: ", buf.String()) 88 | } 89 | }, 90 | }, 91 | { 92 | name: "logging with prefix", 93 | script: `log(prefix="INFO", msg="Logging from starlark with prefix")`, 94 | test: func(t *testing.T, script string) { 95 | exe := New() 96 | var buf bytes.Buffer 97 | logger := log.New(&buf, "", log.Lshortfile) 98 | exe.thread.SetLocal(identifiers.log, logger) 99 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 100 | t.Fatal(err) 101 | } 102 | if !strings.Contains(buf.String(), "INFO: Logging from starlark with prefix") { 103 | t.Error("logger has unexpected log prefix and msg: ", buf.String()) 104 | } 105 | }, 106 | }, 107 | } 108 | 109 | for _, test := range tests { 110 | t.Run(test.name, func(t *testing.T) { 111 | test.test(t, test.script) 112 | }) 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /testing/sshserver.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package testing 5 | 6 | import ( 7 | "fmt" 8 | "path/filepath" 9 | "strings" 10 | 11 | "errors" 12 | "github.com/sirupsen/logrus" 13 | "github.com/vladimirvivien/gexe" 14 | ) 15 | 16 | type SSHServer struct { 17 | name string 18 | port string 19 | mountDir string 20 | username string 21 | e *gexe.Echo 22 | } 23 | 24 | func NewSSHServer(serverName, username, port, sshMountDir string) (*SSHServer, error) { 25 | return &SSHServer{ 26 | name: serverName, 27 | port: port, 28 | mountDir: sshMountDir, 29 | username: username, 30 | e: gexe.New(), 31 | }, nil 32 | } 33 | 34 | // StartSSHServer starts starts sshd process using image linuxserver/openssh-server.DockerRunSSH 35 | // The server opnes up port 2222 with the following command 36 | /* 37 | 38 | docker create \ 39 | --name=test-sshd \ 40 | -e PUBLIC_KEY_FILE=$HOME/.ssh/id_rsa.pub \ 41 | -e USER_NAME=$USER \ 42 | -e SUDO_ACCESS=true \ 43 | -p 2222:2222 \ 44 | -v ./testing/server-name:/config 45 | linuxserver/openssh-server 46 | 47 | */ 48 | func (s *SSHServer) Start() error { 49 | if len(s.e.Prog().Avail("docker")) == 0 { 50 | return errors.New("docker command not found") 51 | } 52 | 53 | if strings.Contains(s.e.Run("docker ps"), s.name) { 54 | logrus.Info("Skipping SSHServer.Start, container already running:", s.name) 55 | return nil 56 | } 57 | 58 | s.e.SetVar("CONTAINER_NAME", s.name) 59 | s.e.SetVar("SSH_PORT", fmt.Sprintf("%s:2222", s.port)) 60 | s.e.SetVar("SSH_DOCKER_IMAGE", "linuxserver/openssh-server") 61 | s.e.SetVar("USERNAME", s.username) 62 | s.e.SetVar("KEY_VOLUME_MOUNT", s.mountDir) 63 | s.e.SetVar("DOCKER_MODS", "linuxserver/mods:openssh-server-openssh-client") 64 | 65 | cmd := s.e.Eval("docker run --rm --detach --name=$CONTAINER_NAME -p $SSH_PORT -e PUBLIC_KEY_FILE=/config/id_rsa.pub -e USER_NAME=$USERNAME -e DOCKER_MODS=$DOCKER_MODS -e SUDO_ACCESS=true -v $KEY_VOLUME_MOUNT:/config $SSH_DOCKER_IMAGE") 66 | logrus.Infof("Starting SSH server: %s", cmd) 67 | proc := s.e.RunProc(cmd) 68 | result := proc.Result() 69 | if proc.Err() != nil { 70 | msg := fmt.Sprintf("%s: %s", proc.Err(), result) 71 | return errors.New(msg) 72 | } 73 | logrus.Infof("SSH server container started: name=%s, port=%s (docker id - %s)", s.name, s.port, result) 74 | 75 | return nil 76 | } 77 | 78 | func (s *SSHServer) Stop() error { 79 | if len(s.e.Prog().Avail("docker")) == 0 { 80 | return errors.New("docker command not found") 81 | } 82 | 83 | s.e.SetVar("CONTAINER_NAME", s.name) 84 | 85 | if !strings.Contains(s.e.Run("docker ps"), s.name) { 86 | logrus.Info("Skipping SSHServerStop, container not running:", s.name) 87 | return nil 88 | } 89 | 90 | proc := s.e.RunProc("docker stop $CONTAINER_NAME") 91 | result := proc.Result() 92 | if proc.Err() != nil { 93 | msg := fmt.Sprintf("failed to stop container: %s: %s", proc.Err(), result) 94 | return errors.New(msg) 95 | } 96 | 97 | // attempt to remove container if still lingering 98 | if strings.Contains(s.e.Run("docker ps -a"), s.name) { 99 | logrus.Info("Forcing container removal:", s.name) 100 | proc := s.e.RunProc("docker rm --force $CONTAINER_NAME") 101 | result := proc.Result() 102 | if proc.Err() != nil { 103 | msg := fmt.Sprintf("failed to remove container: %s: %s", proc.Err(), result) 104 | return errors.New(msg) 105 | } 106 | logrus.Info("SSH server container removed: ", result) 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func (s *SSHServer) MountedDir() string { 113 | return s.mountDir 114 | } 115 | 116 | func (s *SSHServer) PrivateKey() string { 117 | return filepath.Join(s.mountDir, "id_rsa") 118 | } 119 | -------------------------------------------------------------------------------- /k8s/result_writer.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | 11 | "github.com/sirupsen/logrus" 12 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 13 | "k8s.io/cli-runtime/pkg/printers" 14 | "k8s.io/client-go/rest" 15 | ) 16 | 17 | type ResultWriter struct { 18 | workdir string 19 | writeLogs bool 20 | restApi rest.Interface 21 | printer printers.ResourcePrinter 22 | singleFile bool 23 | } 24 | 25 | func NewResultWriter(workdir, what, outputFormat, outputMode string, restApi rest.Interface) (*ResultWriter, error) { 26 | var err error 27 | workdir = filepath.Join(workdir, BaseDirname) 28 | if err := os.MkdirAll(workdir, 0744); err != nil && !os.IsExist(err) { 29 | return nil, err 30 | } 31 | 32 | writeLogs := what == "logs" || what == "all" 33 | var printer printers.ResourcePrinter 34 | if outputFormat == "" || outputFormat == "json" { 35 | printer = &printers.JSONPrinter{} 36 | } else if outputFormat == "yaml" { 37 | printer = &printers.YAMLPrinter{} 38 | } else { 39 | return nil, fmt.Errorf("unsupported output format: %s", outputFormat) 40 | } 41 | 42 | if outputMode != "" && outputMode != "single_file" && outputMode != "multiple_files" { 43 | return nil, fmt.Errorf("unsupported output mode: %s", outputMode) 44 | } 45 | singleFile := outputMode == "single_file" || outputMode == "" 46 | return &ResultWriter{ 47 | workdir: workdir, 48 | printer: printer, 49 | singleFile: singleFile, 50 | writeLogs: writeLogs, 51 | restApi: restApi, 52 | }, err 53 | } 54 | 55 | func (w *ResultWriter) GetResultDir() string { 56 | return w.workdir 57 | } 58 | 59 | func (w *ResultWriter) Write(ctx context.Context, searchResults []SearchResult) error { 60 | if len(searchResults) == 0 { 61 | return errors.New("cannot write empty (or nil) search result") 62 | } 63 | 64 | // each result represents a list of searched item 65 | // write each list in a namespaced location in working dir 66 | var wg sync.WaitGroup 67 | concurrencyLimit := 10 68 | semaphore := make(chan int, concurrencyLimit) 69 | 70 | for _, result := range searchResults { 71 | objWriter := ObjectWriter{ 72 | writeDir: w.workdir, 73 | printer: w.printer, 74 | singleFile: w.singleFile, 75 | } 76 | writeDir, err := objWriter.Write(result) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | if w.writeLogs && result.ListKind == "PodList" { 82 | if len(result.List.Items) == 0 { 83 | continue 84 | } 85 | for _, podItem := range result.List.Items { 86 | logDir := filepath.Join(writeDir, podItem.GetName()) 87 | if err := os.MkdirAll(logDir, 0744); err != nil && !os.IsExist(err) { 88 | return fmt.Errorf("failed to create pod log dir: %s", err) 89 | } 90 | 91 | containers, err := GetContainers(podItem) 92 | if err != nil { 93 | logrus.Errorf("Failed to get containers for pod %s: %s", podItem.GetName(), err) 94 | continue 95 | } 96 | for _, containerLogger := range containers { 97 | semaphore <- 1 // Acquire a slot 98 | wg.Add(1) 99 | go func(pod unstructured.Unstructured, logger Container) { 100 | defer wg.Done() 101 | defer func() { <-semaphore }() // Release the slot 102 | reader, e := logger.Fetch(ctx, w.restApi) 103 | if e != nil { 104 | logrus.Errorf("Failed to fetch container logs for pod %s: %s", pod.GetName(), e) 105 | return 106 | } 107 | e = logger.Write(reader, logDir) 108 | if e != nil { 109 | logrus.Errorf("Failed to write container logs for pod %s: %s", pod.GetName(), e) 110 | return 111 | } 112 | }(podItem, containerLogger) 113 | } 114 | } 115 | } 116 | } 117 | wg.Wait() 118 | 119 | return nil 120 | } 121 | -------------------------------------------------------------------------------- /logging/file.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const ( 13 | AutoLogFile = "auto" 14 | logDateFormat = "2006-01-02T15-04-05" 15 | ) 16 | 17 | // FileHook to send logs to the trace file regardless of CLI level. 18 | type FileHook struct { 19 | // Logger is a reference to the internal Logger that this utilizes. 20 | Logger *logrus.Logger 21 | 22 | // File is a reference to the file being written to. 23 | File *os.File 24 | 25 | // FilePath is the full path used when creating the file. 26 | FilePath string 27 | 28 | // Closed reports whether the hook has been closed. Once closed 29 | // it can't be written to again. 30 | Closed bool 31 | } 32 | 33 | func NewFileHook(path string) (*FileHook, error) { 34 | if path == AutoLogFile { 35 | path = filepath.Join(os.Getenv("HOME"), ".crashd", getLogNameFromTime(time.Now())) 36 | } 37 | file, err := os.Create(path) 38 | logger := logrus.New() 39 | logger.SetLevel(logrus.TraceLevel) 40 | logger.SetOutput(file) 41 | 42 | logrus.Infof("Detailed logs being written to: %v", path) 43 | 44 | return &FileHook{Logger: logger, File: file, FilePath: path}, err 45 | } 46 | 47 | func (hook *FileHook) Fire(entry *logrus.Entry) error { 48 | if hook.Closed { 49 | return nil 50 | } 51 | switch entry.Level { 52 | case logrus.PanicLevel: 53 | hook.Logger.Panic(entry.Message) 54 | case logrus.FatalLevel: 55 | hook.Logger.Fatal(entry.Message) 56 | case logrus.ErrorLevel: 57 | hook.Logger.Error(entry.Message) 58 | case logrus.WarnLevel: 59 | hook.Logger.Warning(entry.Message) 60 | case logrus.InfoLevel: 61 | hook.Logger.Info(entry.Message) 62 | case logrus.DebugLevel: 63 | hook.Logger.Debug(entry.Message) 64 | case logrus.TraceLevel: 65 | hook.Logger.Trace(entry.Message) 66 | default: 67 | hook.Logger.Info(entry.Message) 68 | } 69 | return nil 70 | } 71 | 72 | func (hook *FileHook) Levels() []logrus.Level { 73 | return logrus.AllLevels 74 | } 75 | 76 | // CloseFileHooks will close each file being used for each FileHook attached to the logger. 77 | // If the logger passed is nil, will reference the logrus.StandardLogger(). 78 | func CloseFileHooks(l *logrus.Logger) error { 79 | // All the hooks we utilize are just tied to the standard logger. 80 | logrus.Debugln("Closing log file; future log calls will not be persisted.") 81 | if l == nil { 82 | l = logrus.StandardLogger() 83 | } 84 | 85 | for _, fh := range GetFileHooks(l) { 86 | fh.File.Close() 87 | fh.Closed = true 88 | } 89 | return nil 90 | } 91 | 92 | // GetFirstFileHook is a convenience method to take an object and returns the first 93 | // FileHook attached to it. Accepts an interface{} since the logger objects may be put 94 | // into context or thread objects. The obj should be a *logrus.Logger object. 95 | func GetFirstFileHook(obj interface{}) *FileHook { 96 | fhs := GetFileHooks(obj) 97 | if len(fhs) > 0 { 98 | return fhs[0] 99 | } 100 | return nil 101 | } 102 | 103 | // GetFileHooks is a convenience method to take an object and returns the 104 | // FileHooks attached to it. Accepts an interface{} since the logger objects may be put 105 | // into context or thread objects. The obj should be a *logrus.Logger object. 106 | func GetFileHooks(obj interface{}) []*FileHook { 107 | l, ok := obj.(*logrus.Logger) 108 | if !ok { 109 | return nil 110 | } 111 | result := []*FileHook{} 112 | for _, hooks := range l.Hooks { 113 | for _, hook := range hooks { 114 | switch fh := hook.(type) { 115 | case *FileHook: 116 | result = append(result, fh) 117 | } 118 | } 119 | } 120 | return result 121 | } 122 | 123 | func getLogNameFromTime(t time.Time) string { 124 | return fmt.Sprintf("crashd_%v.log", t.Format(logDateFormat)) 125 | } 126 | -------------------------------------------------------------------------------- /starlark/kube_exec_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "bufio" 8 | "fmt" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | "go.starlark.net/starlark" 14 | "go.starlark.net/starlarkstruct" 15 | ) 16 | 17 | func TestKubeExecScript(t *testing.T) { 18 | workdir := testSupport.TmpDirRoot() 19 | k8sconfig := testSupport.KindKubeConfigFile() 20 | clusterName := testSupport.KindClusterContextName() 21 | err := testSupport.StartNginxPod() 22 | if err != nil { 23 | t.Error("Unexpected error while starting nginx pod", err) 24 | return 25 | } 26 | 27 | execute := func(t *testing.T, script string) *starlarkstruct.Struct { 28 | executor := New() 29 | if err := executor.Exec("test.kube.exec", strings.NewReader(script)); err != nil { 30 | t.Fatalf("failed to exec: %s", err) 31 | } 32 | if !executor.result.Has("kube_exec_output") { 33 | t.Fatalf("script result must be assigned to a value") 34 | } 35 | 36 | data, ok := executor.result["kube_exec_output"].(*starlarkstruct.Struct) 37 | if !ok { 38 | t.Fatal("script result is not a struct") 39 | } 40 | return data 41 | } 42 | 43 | tests := []struct { 44 | name string 45 | script string 46 | eval func(t *testing.T, script string) 47 | }{ 48 | { 49 | name: "exec into pod and run long-running operation", 50 | script: fmt.Sprintf(` 51 | crashd_config(workdir="%s") 52 | set_defaults(kube_config(path="%s", cluster_context="%s")) 53 | kube_exec_output=kube_exec(pod="nginx", timeout_in_seconds=3,cmd=["sh", "-c" ,"while true; do echo 'Running'; sleep 1; done"]) 54 | `, workdir, k8sconfig, clusterName), 55 | eval: func(t *testing.T, script string) { 56 | data := execute(t, script) 57 | 58 | errVal, err := data.Attr("error") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | resultErr := errVal.(starlark.String).GoString() 64 | if resultErr == "" || !strings.HasPrefix(resultErr, "command execution timed out.") { 65 | t.Fatalf("Unexpected error result: %s", resultErr) 66 | } 67 | 68 | ouputFilePath, err := data.Attr("file") 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | 73 | file, err := os.Open(trimQuotes(ouputFilePath.String())) 74 | if err != nil { 75 | t.Fatalf("result file does not exist: %s", err) 76 | } 77 | defer file.Close() 78 | 79 | var actual int 80 | scanner := bufio.NewScanner(file) 81 | for scanner.Scan() { 82 | line := scanner.Text() 83 | if line == "Running" { 84 | actual++ 85 | } 86 | } 87 | expected := 3 88 | if expected != actual { 89 | t.Fatalf("Unexpected file content. expected line numbers: %d but was %d", expected, actual) 90 | } 91 | 92 | }, 93 | }, 94 | { 95 | name: "exec into pod and run short-lived command.Output to specified file", 96 | script: fmt.Sprintf(` 97 | crashd_config(workdir="%s") 98 | set_defaults(kube_config(path="%s", cluster_context="%s")) 99 | kube_exec_output=kube_exec(pod="nginx", output_file="nginx.version",container="nginx", cmd=["nginx", "-v"]) 100 | `, workdir, k8sconfig, clusterName), 101 | eval: func(t *testing.T, script string) { 102 | data := execute(t, script) 103 | 104 | errVal, err := data.Attr("error") 105 | if err != nil { 106 | t.Fatal(err) 107 | } 108 | 109 | resultErr := errVal.(starlark.String).GoString() 110 | if resultErr != "" { 111 | t.Fatalf("expected ouput error to be empty but was %s", resultErr) 112 | } 113 | 114 | ouputFilePath, err := data.Attr("file") 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | 119 | fileContents, err := os.ReadFile(trimQuotes(ouputFilePath.String())) 120 | if err != nil { 121 | t.Fatalf("Error reading output file: %v", err) 122 | } 123 | strings.Contains(string(fileContents), "nginx version:") 124 | 125 | }, 126 | }, 127 | } 128 | 129 | for _, test := range tests { 130 | t.Run(test.name, func(t *testing.T) { 131 | test.eval(t, test.script) 132 | }) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /k8s/search_params_test.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "go.starlark.net/starlark" 8 | "go.starlark.net/starlarkstruct" 9 | ) 10 | 11 | var _ = Describe("SearchParams", func() { 12 | 13 | var searchParams SearchParams 14 | 15 | Context("Building a new instance from a Starlark struct", func() { 16 | 17 | var ( 18 | input *starlarkstruct.Struct 19 | args starlark.StringDict 20 | ) 21 | 22 | It("returns a new instance of the SearchParams type", func() { 23 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{}) 24 | searchParams = NewSearchParams(input) 25 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 26 | }) 27 | 28 | Context("With kinds", func() { 29 | 30 | Context("In the input struct", func() { 31 | 32 | It("returns a new instance with kinds struct member populated", func() { 33 | args = starlark.StringDict{ 34 | "kinds": starlark.String("deployments"), 35 | } 36 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) 37 | searchParams = NewSearchParams(input) 38 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 39 | Expect(searchParams.Kinds).To(HaveLen(1)) 40 | Expect(searchParams.Kinds).To(ConsistOf("deployments")) 41 | }) 42 | 43 | It("returns a new instance with kinds struct member populated", func() { 44 | args = starlark.StringDict{ 45 | "kinds": starlark.NewList([]starlark.Value{starlark.String("deployments"), starlark.String("replicasets")}), 46 | } 47 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) 48 | searchParams = NewSearchParams(input) 49 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 50 | Expect(searchParams.Kinds).To(HaveLen(2)) 51 | Expect(searchParams.Kinds).To(ConsistOf("deployments", "replicasets")) 52 | }) 53 | }) 54 | 55 | Context("not in the input struct", func() { 56 | 57 | It("returns a new instance with default value of kinds struct member populated", func() { 58 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{}) 59 | searchParams = NewSearchParams(input) 60 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 61 | Expect(searchParams.Kinds).To(HaveLen(0)) 62 | }) 63 | }) 64 | }) 65 | 66 | Context("With namespaces", func() { 67 | 68 | Context("In the input struct", func() { 69 | 70 | It("returns a new instance with namespaces struct member populated", func() { 71 | args = starlark.StringDict{ 72 | "namespaces": starlark.String("foo"), 73 | } 74 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) 75 | searchParams = NewSearchParams(input) 76 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 77 | Expect(searchParams.Namespaces).To(HaveLen(1)) 78 | Expect(searchParams.Namespaces).To(ConsistOf("foo")) 79 | }) 80 | 81 | It("returns a new instance with namespaces struct member populated", func() { 82 | args = starlark.StringDict{ 83 | "namespaces": starlark.NewList([]starlark.Value{starlark.String("foo"), starlark.String("bar")}), 84 | } 85 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, args) 86 | searchParams = NewSearchParams(input) 87 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 88 | Expect(searchParams.Namespaces).To(HaveLen(2)) 89 | Expect(searchParams.Namespaces).To(ConsistOf("foo", "bar")) 90 | }) 91 | }) 92 | 93 | Context("not in the input struct", func() { 94 | 95 | It("returns a new instance with default value of namespaces struct member populated", func() { 96 | input = starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{}) 97 | searchParams = NewSearchParams(input) 98 | Expect(searchParams).To(BeAssignableToTypeOf(SearchParams{})) 99 | Expect(searchParams.Namespaces).To(HaveLen(1)) 100 | Expect(searchParams.Namespaces).To(ConsistOf("default")) 101 | }) 102 | }) 103 | }) 104 | }) 105 | }) 106 | -------------------------------------------------------------------------------- /starlark/kube_config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "strings" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | "go.starlark.net/starlark" 12 | "go.starlark.net/starlarkstruct" 13 | ) 14 | 15 | var _ = Describe("kube_config", func() { 16 | 17 | var ( 18 | crashdScript string 19 | executor *Executor 20 | err error 21 | ) 22 | 23 | execSetup := func() { 24 | executor = New() 25 | err = executor.Exec("test.kube.config", strings.NewReader(crashdScript)) 26 | Expect(err).To(BeNil()) 27 | } 28 | 29 | It("throws an error when empty kube_config is used", func() { 30 | err = New().Exec("test.kube.config", strings.NewReader(`kube_config()`)) 31 | Expect(err).To(HaveOccurred()) 32 | }) 33 | 34 | Context("With path", func() { 35 | Context("With kube_config set in the script", func() { 36 | 37 | BeforeEach(func() { 38 | crashdScript = `cfg = kube_config(path="/foo/bar/kube/config")` 39 | execSetup() 40 | }) 41 | 42 | It("sets the path to the kubeconfig file", func() { 43 | kubeConfigData := executor.result["cfg"] 44 | Expect(kubeConfigData).To(BeAssignableToTypeOf(&starlarkstruct.Struct{})) 45 | 46 | cfg, _ := kubeConfigData.(*starlarkstruct.Struct) 47 | 48 | val, err := cfg.Attr("path") 49 | Expect(err).To(BeNil()) 50 | Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) 51 | }) 52 | }) 53 | 54 | Context("With kube_config returned as a value", func() { 55 | 56 | BeforeEach(func() { 57 | crashdScript = `cfg = kube_config(path="/foo/bar/kube/config")` 58 | execSetup() 59 | }) 60 | 61 | It("returns the kube config as a result", func() { 62 | Expect(executor.result.Has("cfg")).NotTo(BeNil()) 63 | 64 | kubeConfigData := executor.result["cfg"] 65 | Expect(kubeConfigData).NotTo(BeNil()) 66 | 67 | cfg, _ := kubeConfigData.(*starlarkstruct.Struct) 68 | 69 | val, err := cfg.Attr("path") 70 | Expect(err).To(BeNil()) 71 | Expect(trimQuotes(val.String())).To(Equal("/foo/bar/kube/config")) 72 | }) 73 | 74 | It("does not set the kube_config in the starlark thread", func() { 75 | kubeConfigData := executor.thread.Local(identifiers.kubeCfg) 76 | Expect(kubeConfigData).NotTo(BeNil()) 77 | }) 78 | }) 79 | }) 80 | 81 | Context("For default kube_config setup", func() { 82 | 83 | BeforeEach(func() { 84 | crashdScript = `foo = "bar"` 85 | execSetup() 86 | }) 87 | 88 | It("does not set the default kube_config in the starlark thread", func() { 89 | kubeConfigData := executor.thread.Local(identifiers.kubeCfg) 90 | Expect(kubeConfigData).NotTo(BeNil()) 91 | }) 92 | }) 93 | }) 94 | 95 | var _ = Describe("KubeConfigFn", func() { 96 | 97 | Context("With capi_provider", func() { 98 | 99 | It("populates the path from the capi provider", func() { 100 | val, err := KubeConfigFn(&starlark.Thread{Name: "test.kube.config.fn"}, nil, nil, 101 | []starlark.Tuple{ 102 | []starlark.Value{ 103 | starlark.String("capi_provider"), 104 | starlarkstruct.FromStringDict(starlark.String(identifiers.capvProvider), starlark.StringDict{ 105 | "kube_config": starlark.String("/foo/bar"), 106 | }), 107 | }, 108 | }) 109 | Expect(err).NotTo(HaveOccurred()) 110 | 111 | cfg, _ := val.(*starlarkstruct.Struct) 112 | 113 | path, err := cfg.Attr("path") 114 | Expect(err).To(BeNil()) 115 | Expect(trimQuotes(path.String())).To(Equal("/foo/bar")) 116 | }) 117 | 118 | It("throws an error when an unknown provider is passed", func() { 119 | _, err := KubeConfigFn(&starlark.Thread{Name: "test.kube.config.fn"}, nil, nil, 120 | []starlark.Tuple{ 121 | []starlark.Value{ 122 | starlark.String("capi_provider"), 123 | starlarkstruct.FromStringDict(starlark.String("meh"), starlark.StringDict{ 124 | "kube_config": starlark.String("/foo/bar"), 125 | }), 126 | }, 127 | }) 128 | Expect(err).To(HaveOccurred()) 129 | Expect(err).To(MatchError("unknown capi provider")) 130 | }) 131 | }) 132 | }) 133 | -------------------------------------------------------------------------------- /starlark/crashd_config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/ssh" 12 | "go.starlark.net/starlarkstruct" 13 | ) 14 | 15 | func testCrashdConfigNew(t *testing.T) { 16 | e := New() 17 | if e.thread == nil { 18 | t.Error("thread is nil") 19 | } 20 | } 21 | 22 | func testCrashdConfigFunc(t *testing.T) { 23 | tests := []struct { 24 | name string 25 | script string 26 | eval func(t *testing.T, script string) 27 | }{ 28 | { 29 | name: "crash_config saved in thread", 30 | script: `crashd_config(workdir="fooval", default_shell="barval")`, 31 | eval: func(t *testing.T, script string) { 32 | defer os.RemoveAll("fooval") 33 | exe := New() 34 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 35 | t.Fatal(err) 36 | } 37 | data := exe.thread.Local(identifiers.crashdCfg) 38 | if data == nil { 39 | t.Fatal("crashd_config not saved in thread local") 40 | } 41 | cfg, ok := data.(*starlarkstruct.Struct) 42 | if !ok { 43 | t.Fatalf("unexpected type for thread local key configs.crashd: %T", data) 44 | } 45 | if len(cfg.AttrNames()) != 5 { 46 | t.Fatalf("unexpected item count in configs.crashd: %d", len(cfg.AttrNames())) 47 | } 48 | 49 | }, 50 | }, 51 | 52 | { 53 | name: "crash_config returned value", 54 | script: `cfg = crashd_config(uid="fooval", gid="barval")`, 55 | eval: func(t *testing.T, script string) { 56 | exe := New() 57 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 58 | t.Fatal(err) 59 | } 60 | data := exe.result["cfg"] 61 | if data == nil { 62 | t.Fatal("crashd_config function not returning value") 63 | } 64 | }, 65 | }, 66 | 67 | { 68 | name: "crash_config default", 69 | script: `one = 1`, 70 | eval: func(t *testing.T, script string) { 71 | exe := New() 72 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 73 | t.Fatal(err) 74 | } 75 | data := exe.thread.Local(identifiers.crashdCfg) 76 | if data == nil { 77 | t.Fatal("default crashd_config not saved in thread local") 78 | } 79 | 80 | cfg, ok := data.(*starlarkstruct.Struct) 81 | if !ok { 82 | t.Fatalf("unexpected type for thread local key crashd_config: %T", data) 83 | } 84 | if len(cfg.AttrNames()) != 5 { 85 | t.Fatalf("unexpected item count in configs.crashd: %d", len(cfg.AttrNames())) 86 | } 87 | val, err := cfg.Attr("uid") 88 | if err != nil { 89 | t.Fatalf("key 'foo' not found in configs.crashd: %s", err) 90 | } 91 | if trimQuotes(val.String()) != getUid() { 92 | t.Fatalf("unexpected value for key %s in configs.crashd", val.String()) 93 | } 94 | }, 95 | }, 96 | 97 | { 98 | name: "crash_config with use-ssh-agent", 99 | script: `crashd_config(workdir="fooval", default_shell="barval", use_ssh_agent=True)`, 100 | eval: func(t *testing.T, script string) { 101 | defer os.RemoveAll("fooval") 102 | exe := New() 103 | if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { 104 | t.Fatal(err) 105 | } 106 | data := exe.thread.Local(identifiers.sshAgent) 107 | if data == nil { 108 | t.Fatal("use_ssh_agent identifier not saved in thread local") 109 | } 110 | agent, ok := data.(ssh.Agent) 111 | if !ok || agent == nil { 112 | t.Fatal("ssh agent should have been started") 113 | } 114 | }, 115 | }, 116 | } 117 | 118 | for _, test := range tests { 119 | t.Run(test.name, func(t *testing.T) { 120 | test.eval(t, test.script) 121 | }) 122 | } 123 | } 124 | 125 | func TestCrashdCfgAll(t *testing.T) { 126 | tests := []struct { 127 | name string 128 | test func(*testing.T) 129 | }{ 130 | {name: "testCrashdConfigNew", test: testCrashdConfigNew}, 131 | {name: "testCrashdConfigFunc", test: testCrashdConfigFunc}, 132 | } 133 | 134 | for _, test := range tests { 135 | t.Run(test.name, func(t *testing.T) { 136 | defer os.RemoveAll(defaults.workdir) 137 | test.test(t) 138 | }) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /ssh/agent_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package ssh 5 | 6 | import ( 7 | "bufio" 8 | "regexp" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/vladimirvivien/gexe" 13 | ) 14 | 15 | func TestParseAndValidateAgentInfo(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | info string 19 | shouldErr bool 20 | }{ 21 | { 22 | name: "valid info", 23 | shouldErr: false, 24 | info: `SSH_AUTH_SOCK=/foo/bar.1234; export SSH_AUTH_SOCK; 25 | SSH_AGENT_PID=4567; export SSH_AGENT_PID; 26 | echo Agent pid 4567;`, 27 | }, 28 | { 29 | name: "invalid info", 30 | shouldErr: true, 31 | info: `FOO=/foo/bar.1234; export BAR; 32 | BLAH=4567; export BLOOP; 33 | echo lorem ipsum 4567;`, 34 | }, 35 | { 36 | name: "invalid info", 37 | shouldErr: true, 38 | info: `SSH_AUTH_SOCK=/foo/bar.1234; export SSH_AUTH_SOCK; 39 | BLAH=4567; export BLOOP; 40 | echo lorem ipsum 4567;`, 41 | }, 42 | { 43 | name: "invalid info", 44 | shouldErr: true, 45 | info: `FOO=/foo/bar.1234; export BAR; 46 | SSH_AGENT_PID=4567; export SSH_AGENT_PID; 47 | echo lorem ipsum 4567;`, 48 | }, 49 | { 50 | name: "invalid info", 51 | shouldErr: true, 52 | info: `lorem ipsum 1; 53 | lorem ipsum 2.`, 54 | }, 55 | { 56 | name: "invalid info", 57 | shouldErr: true, 58 | info: "", 59 | }, 60 | } 61 | 62 | for _, test := range tests { 63 | t.Run(test.name, func(t *testing.T) { 64 | agentInfo, err := parseAgentInfo(strings.NewReader(test.info)) 65 | if err != nil { 66 | t.Fail() 67 | } 68 | err = validateAgentInfo(agentInfo) 69 | if err != nil && !test.shouldErr { 70 | // unexpected failures 71 | t.Fail() 72 | } else if !test.shouldErr { 73 | if _, ok := agentInfo[AgentPidIdentifier]; !ok { 74 | t.Fail() 75 | } 76 | if _, ok := agentInfo[AuthSockIdentifier]; !ok { 77 | t.Fail() 78 | } 79 | } else { 80 | // asserting error scenarios 81 | if err == nil { 82 | t.Fail() 83 | } 84 | } 85 | }) 86 | } 87 | } 88 | 89 | func TestStartAgent(t *testing.T) { 90 | a, err := StartAgent() 91 | if err != nil || a == nil { 92 | t.Fatalf("error should be nil and agent should not be nil: %v", err) 93 | } 94 | out := gexe.Run("ps -ax") 95 | if !strings.Contains(out, "ssh-agent") { 96 | t.Fatal("no ssh-agent process found") 97 | } 98 | 99 | failed := true 100 | scanner := bufio.NewScanner(strings.NewReader(out)) 101 | for scanner.Scan() { 102 | line := scanner.Text() 103 | if strings.Contains(line, "ssh-agent") { 104 | pid := strings.Split(strings.TrimSpace(line), " ")[0] 105 | // set failed to false if correct ssh-agent process is found 106 | agentStruct, _ := a.(*agent) 107 | if pid == agentStruct.Pid { 108 | failed = false 109 | } 110 | } 111 | } 112 | if failed { 113 | t.Fatal("could not find agent with correct Pid") 114 | } 115 | 116 | t.Cleanup(func() { 117 | _ = a.Stop() 118 | }) 119 | } 120 | 121 | func TestAgent(t *testing.T) { 122 | a, err := StartAgent() 123 | if err != nil { 124 | t.Fatalf("failed to start agent: %v", err) 125 | } 126 | 127 | tests := []struct { 128 | name string 129 | assert func(*testing.T, Agent) 130 | }{ 131 | { 132 | name: "GetEnvVariables", 133 | assert: func(t *testing.T, agent Agent) { 134 | vars := agent.GetEnvVariables() 135 | if len(vars) != 2 { 136 | t.Fatalf("not enough variables") 137 | } 138 | 139 | matchPID, err := regexp.MatchString(`SSH_AGENT_PID=[0-9]+`, vars[0]) 140 | if err != nil || !matchPID { 141 | t.Fatalf("SSH_AGENT_PID format does not match") 142 | } 143 | matchSock, err := regexp.MatchString(`SSH_AUTH_SOCK=\S*`, vars[1]) 144 | if err != nil || !matchSock { 145 | t.Fatalf("SSH_AUTH_SOCK format does not match") 146 | } 147 | }, 148 | }, 149 | { 150 | name: "Stop", 151 | assert: func(t *testing.T, agent Agent) { 152 | if err := agent.Stop(); err != nil { 153 | t.Errorf("failed to stop agent: %s", err) 154 | } 155 | }, 156 | }, 157 | } 158 | 159 | for _, test := range tests { 160 | t.Run(test.name, func(t *testing.T) { 161 | test.assert(t, a) 162 | }) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /starlark/capa_provider.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | 11 | "github.com/vmware-tanzu/crash-diagnostics/k8s" 12 | "github.com/vmware-tanzu/crash-diagnostics/provider" 13 | "go.starlark.net/starlark" 14 | "go.starlark.net/starlarkstruct" 15 | ) 16 | 17 | // CapaProviderFn is a built-in starlark function that collects compute resources from a k8s cluster 18 | // Starlark format: capa_provider(kube_config=kube_config(), ssh_config=ssh_config()[workload_cluster=, namespace=, nodes=["foo", "bar], labels=["bar", "baz"]]) 19 | func CapaProviderFn(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 20 | 21 | var ( 22 | workloadCluster, namespace string 23 | names, labels *starlark.List 24 | sshConfig, mgmtKubeConfig *starlarkstruct.Struct 25 | ) 26 | 27 | err := starlark.UnpackArgs("capa_provider", args, kwargs, 28 | "ssh_config", &sshConfig, 29 | "mgmt_kube_config", &mgmtKubeConfig, 30 | "workload_cluster?", &workloadCluster, 31 | "namespace?", &namespace, 32 | "labels?", &labels, 33 | "nodes?", &names) 34 | if err != nil { 35 | return starlark.None, fmt.Errorf("failed to unpack input arguments: %w", err) 36 | } 37 | 38 | ctx, ok := thread.Local(identifiers.scriptCtx).(context.Context) 39 | if !ok || ctx == nil { 40 | return starlark.None, errors.New("script context not found") 41 | } 42 | 43 | if sshConfig == nil || mgmtKubeConfig == nil { 44 | return starlark.None, errors.New("capa_provider requires the name of the management cluster, the ssh configuration and the management cluster kubeconfig") 45 | } 46 | 47 | if mgmtKubeConfig == nil { 48 | mgmtKubeConfig = thread.Local(identifiers.kubeCfg).(*starlarkstruct.Struct) 49 | } 50 | mgmtKubeConfigPath, err := getKubeConfigPathFromStruct(mgmtKubeConfig) 51 | if err != nil { 52 | return starlark.None, fmt.Errorf("failed to extract management kubeconfig: %w", err) 53 | } 54 | 55 | // if workload cluster is not supplied, then the resources for the management cluster 56 | // should be enumerated 57 | clusterName := workloadCluster 58 | if clusterName == "" { 59 | config, err := k8s.LoadKubeCfg(mgmtKubeConfigPath) 60 | if err != nil { 61 | return starlark.None, fmt.Errorf("failed to load kube config: %w", err) 62 | } 63 | clusterName, err = config.GetClusterName() 64 | if err != nil { 65 | return starlark.None, fmt.Errorf("cannot find cluster with name %s: %w", workloadCluster, err) 66 | } 67 | } 68 | 69 | bastionIpAddr, err := k8s.FetchBastionIpAddress(clusterName, namespace, mgmtKubeConfigPath) 70 | if err != nil { 71 | return starlark.None, fmt.Errorf("could not fetch jump host addresses: %w", err) 72 | } 73 | 74 | providerConfigPath, err := provider.KubeConfig(mgmtKubeConfigPath, clusterName, namespace) 75 | if err != nil { 76 | return starlark.None, err 77 | } 78 | 79 | nodeAddresses, err := k8s.GetNodeAddresses(ctx, providerConfigPath, toSlice(names), toSlice(labels)) 80 | if err != nil { 81 | return starlark.None, fmt.Errorf("could not fetch host addresses: %w", err) 82 | } 83 | 84 | // dictionary for capa provider struct 85 | capaProviderDict := starlark.StringDict{ 86 | "kind": starlark.String(identifiers.capaProvider), 87 | "transport": starlark.String("ssh"), 88 | "kube_config": starlark.String(providerConfigPath), 89 | } 90 | 91 | // add node info to dictionary 92 | var nodeIps []starlark.Value 93 | for _, node := range nodeAddresses { 94 | nodeIps = append(nodeIps, starlark.String(node)) 95 | } 96 | capaProviderDict["hosts"] = starlark.NewList(nodeIps) 97 | 98 | sshConfigDict := starlark.StringDict{} 99 | sshConfig.ToStringDict(sshConfigDict) 100 | 101 | // modify ssh config jump credentials, if not specified 102 | if _, err := sshConfig.Attr("jump_host"); err != nil { 103 | sshConfigDict["jump_host"] = starlark.String(bastionIpAddr) 104 | } 105 | if _, err := sshConfig.Attr("jump_user"); err != nil { 106 | sshConfigDict["jump_user"] = starlark.String("ubuntu") 107 | } 108 | capaProviderDict[identifiers.sshCfg] = starlarkstruct.FromStringDict(starlark.String(identifiers.sshCfg), sshConfigDict) 109 | 110 | return starlarkstruct.FromStringDict(starlark.String(identifiers.capaProvider), capaProviderDict), nil 111 | } 112 | -------------------------------------------------------------------------------- /ssh/ssh.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package ssh 5 | 6 | import ( 7 | "bytes" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "strings" 12 | "time" 13 | 14 | "github.com/sirupsen/logrus" 15 | "github.com/vladimirvivien/gexe" 16 | "github.com/vladimirvivien/gexe/exec" 17 | "k8s.io/apimachinery/pkg/util/wait" 18 | ) 19 | 20 | type ProxyJumpArgs struct { 21 | User string 22 | Host string 23 | } 24 | 25 | type SSHArgs struct { 26 | User string 27 | Host string 28 | PrivateKeyPath string 29 | Port string 30 | MaxRetries int 31 | ProxyJump *ProxyJumpArgs 32 | } 33 | 34 | // Run runs a command over SSH and returns the result as a string 35 | func Run(args SSHArgs, agent Agent, cmd string) (string, error) { 36 | reader, err := sshRunProc(args, agent, cmd) 37 | if err != nil { 38 | return "", err 39 | } 40 | var result bytes.Buffer 41 | if _, err := result.ReadFrom(reader); err != nil { 42 | return "", err 43 | } 44 | return strings.TrimSpace(result.String()), nil 45 | } 46 | 47 | // RunRead runs a command over SSH and returns an io.Reader for stdout/stderr 48 | func RunRead(args SSHArgs, agent Agent, cmd string) (io.Reader, error) { 49 | return sshRunProc(args, agent, cmd) 50 | } 51 | 52 | func sshRunProc(args SSHArgs, agent Agent, cmd string) (io.Reader, error) { 53 | e := gexe.New() 54 | prog := e.Prog().Avail("ssh") 55 | if len(prog) == 0 { 56 | return nil, errors.New("ssh program not found") 57 | } 58 | 59 | sshCmd, err := makeSSHCmdStr(prog, args) 60 | if err != nil { 61 | return nil, err 62 | } 63 | effectiveCmd := fmt.Sprintf(`%s "%s"`, sshCmd, cmd) 64 | logrus.Debug("ssh.run: ", effectiveCmd) 65 | 66 | if agent != nil { 67 | logrus.Debugf("Adding agent info: %s", agent.GetEnvVariables()) 68 | e = e.Envs(agent.GetEnvVariables()...) 69 | } 70 | 71 | var proc *exec.Proc 72 | maxRetries := args.MaxRetries 73 | if maxRetries == 0 { 74 | maxRetries = 10 75 | } 76 | retries := wait.Backoff{Steps: maxRetries, Duration: time.Millisecond * 80, Jitter: 0.1} 77 | if err := wait.ExponentialBackoff(retries, func() (bool, error) { 78 | p := e.RunProc(effectiveCmd) 79 | if p.Err() != nil { 80 | logrus.Warn(fmt.Sprintf("ssh: failed to connect to %s: error '%s %s': retrying connection", args.Host, p.Err(), p.Result())) 81 | return false, p.Err() 82 | } 83 | proc = p 84 | return true, nil // worked 85 | }); err != nil { 86 | logrus.Debugf("ssh.run failed after %d tries", maxRetries) 87 | return nil, fmt.Errorf("ssh: failed after %d attempt(s): %s", maxRetries, err) 88 | } 89 | 90 | if proc == nil { 91 | return nil, errors.New("ssh.run: did get process result") 92 | } 93 | 94 | return proc.Out(), nil 95 | } 96 | 97 | func makeSSHCmdStr(progName string, args SSHArgs) (string, error) { 98 | if args.User == "" { 99 | return "", errors.New("SSH: user is required") 100 | } 101 | if args.Host == "" { 102 | return "", errors.New("SSH: host is required") 103 | } 104 | 105 | if args.ProxyJump != nil { 106 | if args.ProxyJump.User == "" || args.ProxyJump.Host == "" { 107 | return "", errors.New("SSH: jump user and host are required") 108 | } 109 | } 110 | 111 | sshCmdPrefix := func() string { 112 | return fmt.Sprintf("%s -q -o StrictHostKeyChecking=no", progName) 113 | } 114 | 115 | pkPath := func() string { 116 | if args.PrivateKeyPath != "" { 117 | return fmt.Sprintf("-i %s", args.PrivateKeyPath) 118 | } 119 | return "" 120 | } 121 | 122 | port := func() string { 123 | if args.Port == "" { 124 | return "-p 22" 125 | } 126 | return fmt.Sprintf("-p %s", args.Port) 127 | } 128 | 129 | proxyJump := func() string { 130 | if args.ProxyJump != nil { 131 | return fmt.Sprintf("%s@%s", args.User, args.Host) + ` -o "ProxyCommand ssh -o StrictHostKeyChecking=no -W %h:%p ` + fmt.Sprintf("%s %s@%s\"", pkPath(), args.ProxyJump.User, args.ProxyJump.Host) 132 | } 133 | return "" 134 | } 135 | 136 | // build command as 137 | // ssh -i -P user@host OR 138 | // ssh -i -P user@host -o "ProxyCommand ssh -W %h:%p -i " 139 | cmd := func() string { 140 | cmdStr := fmt.Sprintf("%s %s %s ", sshCmdPrefix(), pkPath(), port()) 141 | 142 | if proxyDetails := proxyJump(); proxyDetails != "" { 143 | cmdStr += proxyDetails 144 | } else { 145 | cmdStr += fmt.Sprintf("%s@%s", args.User, args.Host) 146 | } 147 | 148 | return cmdStr 149 | } 150 | 151 | return cmd(), nil 152 | } 153 | -------------------------------------------------------------------------------- /starlark/kube_config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "github.com/vmware-tanzu/crash-diagnostics/util" 11 | "go.starlark.net/starlark" 12 | "go.starlark.net/starlarkstruct" 13 | ) 14 | 15 | // KubeConfigFn is built-in starlark function that wraps the kwargs into a dictionary value. 16 | // The result is also added to the thread for other built-in to access. 17 | // Starlark: kube_config(path=kubecf/path, [cluster_context=context_name]) 18 | func KubeConfigFn(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 19 | var path, clusterCtxName string 20 | var provider *starlarkstruct.Struct 21 | 22 | if err := starlark.UnpackArgs( 23 | identifiers.kubeCfg, args, kwargs, 24 | "cluster_context?", &clusterCtxName, 25 | "path?", &path, 26 | "capi_provider?", &provider, 27 | ); err != nil { 28 | return starlark.None, fmt.Errorf("%s: %s", identifiers.kubeCfg, err) 29 | } 30 | 31 | // check if only one of the two options are present 32 | if (len(path) == 0 && provider == nil) || (len(path) != 0 && provider != nil) { 33 | return starlark.None, errors.New("need either path or capi_provider") 34 | } 35 | 36 | var extraKubeConfigStr starlark.String 37 | var isKCPProvider bool 38 | 39 | if len(path) == 0 { 40 | val := provider.Constructor() 41 | 42 | if constructor, ok := val.(starlark.String); ok { 43 | constStr := constructor.GoString() 44 | if constStr != identifiers.capvProvider && 45 | constStr != identifiers.capaProvider && 46 | constStr != identifiers.kcpProvider { 47 | return starlark.None, errors.New("unknown capi provider") 48 | } 49 | 50 | if constStr == identifiers.kcpProvider { 51 | kcp_kubeconfig, err := provider.Attr("kcp_kubeconfig") 52 | if err != nil { 53 | return starlark.None, fmt.Errorf("not able to find kcp_kubeconfig for kcp_provider") 54 | } 55 | 56 | extraKubeConfigStr, ok = kcp_kubeconfig.(starlark.String) 57 | if !ok { 58 | return starlark.None, errors.New("could not fetch kubeconfig") 59 | } 60 | 61 | isKCPProvider = true 62 | } 63 | } 64 | 65 | pathVal, err := provider.Attr("kube_config") 66 | if err != nil { 67 | return starlark.None, fmt.Errorf("could not find the kubeconfig attribute: %w", err) 68 | } 69 | pathStr, ok := pathVal.(starlark.String) 70 | if !ok { 71 | return starlark.None, errors.New("could not fetch kubeconfig") 72 | } 73 | path = pathStr.GoString() 74 | 75 | } 76 | 77 | path, err := util.ExpandPath(path) 78 | if err != nil { 79 | return starlark.None, err 80 | } 81 | 82 | stringDictVal := starlark.StringDict{ 83 | "cluster_context": starlark.String(clusterCtxName), 84 | "path": starlark.String(path), 85 | } 86 | 87 | if isKCPProvider { 88 | stringDictVal["kcp_kubeconfig"] = extraKubeConfigStr 89 | } 90 | 91 | return starlarkstruct.FromStringDict(starlark.String(identifiers.kubeCfg), stringDictVal), nil 92 | } 93 | 94 | // addDefaultKubeConf initializes a Starlark Dict with default 95 | // KUBECONFIG configuration data 96 | func addDefaultKubeConf(thread *starlark.Thread) error { 97 | args := []starlark.Tuple{ 98 | {starlark.String("path"), starlark.String(defaults.kubeconfig)}, 99 | } 100 | 101 | conf, err := KubeConfigFn(thread, nil, nil, args) 102 | if err != nil { 103 | return err 104 | } 105 | thread.SetLocal(identifiers.kubeCfg, conf) 106 | return nil 107 | } 108 | 109 | func getKubeConfigPathFromStruct(kubeConfigStructVal *starlarkstruct.Struct) (string, error) { 110 | kvPathVal, err := kubeConfigStructVal.Attr("path") 111 | if err != nil { 112 | return "", fmt.Errorf("failed to extract kubeconfig path: %w", err) 113 | } 114 | kvPathStrVal, ok := kvPathVal.(starlark.String) 115 | if !ok { 116 | return "", errors.New("failed to extract kubeconfig") 117 | } 118 | return kvPathStrVal.GoString(), nil 119 | } 120 | 121 | // getKubeConfigContextNameFromStruct returns the cluster name from the KubeConfig struct 122 | // provided. If filed cluster_context not provided or unable to convert, it is returned 123 | // as an empty context. 124 | func getKubeConfigContextNameFromStruct(kubeConfigStructVal *starlarkstruct.Struct) string { 125 | ctxVal, err := kubeConfigStructVal.Attr("cluster_context") 126 | if err != nil { 127 | return "" 128 | } 129 | ctxName, ok := ctxVal.(starlark.String) 130 | if !ok { 131 | return "" 132 | } 133 | return ctxName.GoString() 134 | } 135 | -------------------------------------------------------------------------------- /examples/windows_capv_provider.crsh: -------------------------------------------------------------------------------- 1 | conf = crashd_config(workdir=args.workdir) 2 | ssh_conf = ssh_config(username="capv", private_key_path=args.private_key) 3 | kube_conf = kube_config(path=args.mc_config) 4 | 5 | 6 | #list out management and cluster nodes 7 | wc_provider=capv_provider( 8 | workload_cluster=args.cluster_name, 9 | ssh_config=ssh_conf, 10 | mgmt_kube_config=kube_conf 11 | ) 12 | 13 | def fetch_workload_provider(iaas, cluster_name, ssh_cfg, kube_cfg, filter_labels): 14 | ns = args.workload_cluster_ns 15 | if iaas == "vsphere": 16 | provider = capv_provider( 17 | workload_cluster=cluster_name, 18 | namespace=ns, 19 | ssh_config=ssh_cfg, 20 | mgmt_kube_config=kube_cfg, 21 | labels=filter_labels 22 | ) 23 | else: 24 | provider = capa_provider( 25 | workload_cluster=cluster_name, 26 | namespace=ns, 27 | ssh_config=ssh_cfg, 28 | mgmt_kube_config=kube_cfg, 29 | labels=filter_labels 30 | ) 31 | return provider 32 | 33 | 34 | def capture_windows_node_diagnostics(nodes, cni): 35 | if cni == antrea: 36 | capture(cmd="Get-Service ovs* | select * ", resources=nodes) 37 | capture(cmd="Get-Service antrea-agent | select * ", resources=nodes) 38 | capture(cmd="Get-Service kube-proxy | select * ", resources=nodes) 39 | capture(cmd="cat c:\\var\\log\\antrea\\antrea-agent.exe.INFO", resources=nodes) 40 | copy_from(path="c:\\openvswitch\\var\\log\\openvswitch\\ovs-vswitchd.log", resources=nodes) 41 | copy_from(path="c:\\openvswitch\\var\\log\\openvswitch\\ovsdb-server.log", resources=nodes) 42 | 43 | capture(cmd="Get-CimInstance -ClassName Win32_LogicalDisk", file_name="disk_info.out", resources=nodes) 44 | capture(cmd="(Get-ItemProperty -Path c:\\windows\\system32\\hal.dll).VersionInfo.FileVersion",file_name="windows_version_info.out", resources=nodes) 45 | capture(cmd="cat C:\\k\\StartKubelet.ps1 ; cat C:\\var\\lib\\kubelet\\kubeadm-flags.env", resources=nodes) 46 | capture(cmd="Get-Service Kubelet | select * ", resources=nodes) 47 | capture(cmd="Get-Service Containerd | select * ", resources=nodes) 48 | capture(cmd="Get-Service Kubelet | select * ", resources=nodes) 49 | capture(cmd="Get-HNSNetwork", resources=nodes) 50 | capture(cmd="& 'c:\\Program Files\\containerd\\crictl.exe' -r 'npipe:////./pipe/containerd-containerd' info", resources=nodes) 51 | capture(cmd="Get-MpPreference | select ExclusionProcess", resources=nodes) 52 | capture(cmd="cat c:\\var\\log\\kubelet\\kubelet.exe.INFO", resources=nodes) 53 | capture(cmd="cat c:\\var\\log\\kube-proxy\\kube-proxy.exe.INFO", resources=nodes) 54 | capture(cmd="cat 'c:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init-unattend.log'", resources=nodes) 55 | capture(cmd="cat 'c:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init.log'", resources=nodes) 56 | copy_from(path="C:\\Windows\\System32\\Winevt\\Logs\\System.evtx", resources=nodes) 57 | copy_from(path="C:\\Windows\\System32\\Winevt\\Logs\\Application.evtx", resources=nodes) 58 | 59 | def capture_node_diagnostics(nodes): 60 | capture(cmd="sudo df -i", resources=nodes) 61 | capture(cmd="sudo crictl info", resources=nodes) 62 | capture(cmd="df -h /var/lib/containerd", resources=nodes) 63 | capture(cmd="sudo systemctl status kubelet", resources=nodes) 64 | capture(cmd="sudo systemctl status containerd", resources=nodes) 65 | capture(cmd="sudo journalctl -xeu kubelet", resources=nodes) 66 | 67 | capture(cmd="sudo cat /var/log/cloud-init-output.log", resources=nodes) 68 | capture(cmd="sudo cat /var/log/cloud-init.log", resources=nodes) 69 | 70 | 71 | #fetch linux nodes 72 | wc_provider_linux = fetch_workload_provider(infra, name, ssh_conf, kube_conf, ["kubernetes.io/os=linux"]) 73 | nodes = resources(provider=wc_provider_linux) 74 | 75 | #fetch windows nodes 76 | wc_provider_windows = fetch_workload_provider(infra, name, ssh_conf, kube_conf, ["kubernetes.io/os=windows"]) 77 | win_nodes = resources(provider=wc_provider_windows) 78 | 79 | capture_node_diagnostics(nodes) 80 | capture_windows_node_diagnostics(win_nodes, args.cni) 81 | 82 | 83 | #add code to collect pod info from cluster 84 | set_defaults(kube_config(capi_provider = wc_provider)) 85 | 86 | pod_ns=["default", "kube-system"] 87 | 88 | kube_capture(what="logs", namespaces=pod_ns) 89 | kube_capture(what="objects", kinds=["pods", "services"], namespaces=pod_ns) 90 | kube_capture(what="objects", kinds=["deployments", "replicasets"], groups=["apps"], namespaces=pod_ns) 91 | 92 | archive(output_file="diagnostics.tar.gz", source_paths=[conf.workdir]) -------------------------------------------------------------------------------- /starlark/copy_to.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package starlark 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | 10 | "github.com/sirupsen/logrus" 11 | "go.starlark.net/starlark" 12 | "go.starlark.net/starlarkstruct" 13 | 14 | "github.com/vmware-tanzu/crash-diagnostics/ssh" 15 | ) 16 | 17 | // copyToFunc is a built-in starlark function that copies file resources from 18 | // the local machine to a specified location on remote compute resources. 19 | // 20 | // If only one argument is provided, it is assumed to be the of file to copy. 21 | // If resources are not provded, copy_to will search the starlark context for one. 22 | // 23 | // Starlark format: copy_to([] [,source_path=, target_path=, resources=resources]) 24 | 25 | func copyToFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { 26 | var sourcePath, targetPath string 27 | var resources *starlark.List 28 | 29 | if err := starlark.UnpackArgs( 30 | identifiers.copyTo, args, kwargs, 31 | "source_path", &sourcePath, 32 | "target_path?", &targetPath, 33 | "resources?", &resources, 34 | ); err != nil { 35 | return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) 36 | } 37 | 38 | if len(sourcePath) == 0 { 39 | return starlark.None, fmt.Errorf("%s: path arg not set", identifiers.copyTo) 40 | } 41 | if len(targetPath) == 0 { 42 | targetPath = sourcePath 43 | } 44 | 45 | if resources == nil { 46 | res, err := getResourcesFromThread(thread) 47 | if err != nil { 48 | return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) 49 | } 50 | resources = res 51 | } 52 | 53 | var agent ssh.Agent 54 | var ok bool 55 | if agentVal := thread.Local(identifiers.sshAgent); agentVal != nil { 56 | agent, ok = agentVal.(ssh.Agent) 57 | if !ok { 58 | return starlark.None, errors.New("unable to fetch ssh-agent") 59 | } 60 | } 61 | 62 | results, err := execCopyTo(sourcePath, targetPath, agent, resources) 63 | if err != nil { 64 | return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) 65 | } 66 | 67 | // build list of struct as result 68 | var resultList []starlark.Value 69 | for _, result := range results { 70 | if len(results) == 1 { 71 | return result.toStarlarkStruct(), nil 72 | } 73 | resultList = append(resultList, result.toStarlarkStruct()) 74 | } 75 | 76 | return starlark.NewList(resultList), nil 77 | } 78 | 79 | func execCopyTo(sourcePath, targetPath string, agent ssh.Agent, resources *starlark.List) ([]commandResult, error) { 80 | if resources == nil { 81 | return nil, fmt.Errorf("%s: missing resources", identifiers.copyFrom) 82 | } 83 | 84 | var results []commandResult 85 | for i := 0; i < resources.Len(); i++ { 86 | val := resources.Index(i) 87 | res, ok := val.(*starlarkstruct.Struct) 88 | if !ok { 89 | return nil, fmt.Errorf("%s: unexpected resource type", identifiers.copyFrom) 90 | } 91 | 92 | val, err := res.Attr("kind") 93 | if err != nil { 94 | return nil, fmt.Errorf("%s: resource.kind: %s", identifiers.copyFrom, err) 95 | } 96 | kind := val.(starlark.String) 97 | 98 | val, err = res.Attr("transport") 99 | if err != nil { 100 | return nil, fmt.Errorf("%s: resource.transport: %s", identifiers.copyFrom, err) 101 | } 102 | transport := val.(starlark.String) 103 | 104 | val, err = res.Attr("host") 105 | if err != nil { 106 | return nil, fmt.Errorf("%s: resource.host: %s", identifiers.copyFrom, err) 107 | } 108 | host := string(val.(starlark.String)) 109 | 110 | switch { 111 | case string(kind) == identifiers.hostResource && string(transport) == "ssh": 112 | result, err := execSCPCopyTo(host, sourcePath, targetPath, agent, res) 113 | if err != nil { 114 | logrus.Errorf("%s: failed to copy to : %s: %s", identifiers.copyTo, sourcePath, err) 115 | } 116 | results = append(results, result) 117 | default: 118 | logrus.Errorf("%s: unsupported or invalid resource kind: %s", identifiers.copyFrom, kind) 119 | continue 120 | } 121 | } 122 | 123 | return results, nil 124 | } 125 | 126 | func execSCPCopyTo(host, sourcePath, targetPath string, agent ssh.Agent, res *starlarkstruct.Struct) (commandResult, error) { 127 | sshCfg := starlarkstruct.FromKeywords(starlarkstruct.Default, makeDefaultSSHConfig()) 128 | if val, err := res.Attr(identifiers.sshCfg); err == nil { 129 | if cfg, ok := val.(*starlarkstruct.Struct); ok { 130 | sshCfg = cfg 131 | } 132 | } 133 | 134 | args, err := getSSHArgsFromCfg(sshCfg) 135 | if err != nil { 136 | return commandResult{}, err 137 | } 138 | args.Host = host 139 | err = ssh.CopyTo(args, agent, sourcePath, targetPath) 140 | return commandResult{resource: args.Host, result: targetPath, err: err}, err 141 | } 142 | --------------------------------------------------------------------------------