├── images ├── logo.png └── logo-small.png ├── pkg └── query │ ├── testdata │ └── xprotect-reports.sql │ ├── pack_test.go │ ├── run.go │ ├── pack.go │ └── query.go ├── go.mod ├── .yamllint ├── .gitignore ├── .github └── workflows │ └── ci.yml ├── go.sum ├── Makefile ├── .golangci.yml ├── README.md ├── LICENSE └── cmd └── osqtool └── main.go /images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chainguard-dev/osqtool/HEAD/images/logo.png -------------------------------------------------------------------------------- /images/logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chainguard-dev/osqtool/HEAD/images/logo-small.png -------------------------------------------------------------------------------- /pkg/query/testdata/xprotect-reports.sql: -------------------------------------------------------------------------------- 1 | -- Returns a list of malware matches from macOS XProtect 2 | -- 3 | -- interval: 1200 4 | -- platform: darwin 5 | SELECT 6 | * 7 | FROM 8 | xprotect_reports; 9 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/chainguard-dev/osqtool 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/fatih/semgroup v1.2.0 7 | github.com/google/go-cmp v0.5.9 8 | k8s.io/klog/v2 v2.90.0 9 | ) 10 | 11 | require ( 12 | github.com/go-logr/logr v1.2.0 // indirect 13 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | braces: 6 | max-spaces-inside: 1 7 | brackets: 8 | max-spaces-inside: 1 9 | comments: disable 10 | comments-indentation: disable 11 | document-start: disable 12 | line-length: 13 | level: warning 14 | max: 160 15 | allow-non-breakable-inline-mappings: true 16 | truthy: disable 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # added by lint-install 18 | out/ 19 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Harden Runner 18 | uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 19 | with: 20 | egress-policy: audit 21 | 22 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 26 | with: 27 | go-version: 'stable' 28 | 29 | - name: Build 30 | run: make out/osqtool 31 | 32 | - name: Lint 33 | run: make lint 34 | 35 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg= 2 | github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8= 3 | github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= 4 | github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 5 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 6 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 7 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 8 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 9 | k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= 10 | k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= 11 | -------------------------------------------------------------------------------- /pkg/query/pack_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "github.com/google/go-cmp/cmp/cmpopts" 8 | ) 9 | 10 | func TestLoad(t *testing.T) { 11 | got, err := Load("testdata/xprotect-reports.sql") 12 | if err != nil { 13 | t.Fatalf("load: %v", err) 14 | } 15 | 16 | want := &Metadata{ 17 | Name: "xprotect-reports", 18 | Query: "SELECT * FROM xprotect_reports;", 19 | Interval: "1200", 20 | Description: "Returns a list of malware matches from macOS XProtect", 21 | Platform: "darwin", 22 | } 23 | 24 | if diff := cmp.Diff(got, want, cmpopts.IgnoreUnexported(Metadata{})); diff != "" { 25 | t.Errorf("Load() got = %v, want %v\n diff: %s", got, want, diff) 26 | } 27 | } 28 | 29 | func TestRender(t *testing.T) { 30 | m := &Metadata{ 31 | Name: "xprotect-reports", 32 | Query: "SELECT * FROM xprotect_reports;", 33 | Interval: "1200", 34 | Platform: "darwin", 35 | Description: "Returns a list of malware matches from macOS XProtect", 36 | } 37 | 38 | got, err := Render(m) 39 | if err != nil { 40 | t.Fatalf("render: %v", err) 41 | } 42 | 43 | want := `-- Returns a list of malware matches from macOS XProtect 44 | -- 45 | -- interval: 1200 46 | -- platform: darwin 47 | 48 | SELECT * FROM xprotect_reports; 49 | ` 50 | if diff := cmp.Diff(got, want); diff != "" { 51 | t.Errorf("Load() got = %v, want %v\n diff: %s", got, want, diff) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # BEGIN: lint-install . 3 | # http://github.com/tinkerbell/lint-install 4 | 5 | .PHONY: lint 6 | lint: _lint 7 | 8 | LINT_ARCH := $(shell uname -m) 9 | LINT_OS := $(shell uname) 10 | LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]') 11 | LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) 12 | 13 | # shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation 14 | ifeq ($(LINT_OS),Darwin) 15 | ifeq ($(LINT_ARCH),arm64) 16 | LINT_ARCH=x86_64 17 | endif 18 | endif 19 | 20 | LINTERS := 21 | FIXERS := 22 | 23 | GOLANGCI_LINT_CONFIG := $(LINT_ROOT)/.golangci.yml 24 | GOLANGCI_LINT_VERSION ?= v1.51.2 25 | GOLANGCI_LINT_BIN := $(LINT_ROOT)/out/linters/golangci-lint-$(GOLANGCI_LINT_VERSION)-$(LINT_ARCH) 26 | $(GOLANGCI_LINT_BIN): 27 | mkdir -p $(LINT_ROOT)/out/linters 28 | rm -rf $(LINT_ROOT)/out/linters/golangci-lint-* 29 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LINT_ROOT)/out/linters $(GOLANGCI_LINT_VERSION) 30 | mv $(LINT_ROOT)/out/linters/golangci-lint $@ 31 | 32 | LINTERS += golangci-lint-lint 33 | golangci-lint-lint: $(GOLANGCI_LINT_BIN) 34 | find . -name go.mod -execdir "$(GOLANGCI_LINT_BIN)" run -c "$(GOLANGCI_LINT_CONFIG)" \; 35 | 36 | FIXERS += golangci-lint-fix 37 | golangci-lint-fix: $(GOLANGCI_LINT_BIN) 38 | find . -name go.mod -execdir "$(GOLANGCI_LINT_BIN)" run -c "$(GOLANGCI_LINT_CONFIG)" --fix \; 39 | 40 | .PHONY: _lint $(LINTERS) 41 | _lint: $(LINTERS) 42 | 43 | .PHONY: fix $(FIXERS) 44 | fix: $(FIXERS) 45 | 46 | # END: lint-install . 47 | 48 | out/osqtool: 49 | mkdir -p out 50 | GOBIN=$(CURDIR)/out go install ./cmd/osqtool/ 51 | -------------------------------------------------------------------------------- /pkg/query/run.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "os/exec" 9 | "runtime" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | type RunResult struct { 18 | IncompatiblePlatform string 19 | Rows []Row 20 | Elapsed time.Duration 21 | } 22 | 23 | type Row map[string]string 24 | 25 | func (r Row) String() string { 26 | var sb strings.Builder 27 | 28 | keys := []string{} 29 | for k := range r { 30 | keys = append(keys, k) 31 | } 32 | sort.Strings(keys) 33 | 34 | for _, k := range keys { 35 | v := r[k] 36 | 37 | text := fmt.Sprintf(`%s:%s `, k, v) 38 | if strings.Contains(v, " ") || strings.Contains(v, ":") { 39 | text = fmt.Sprintf(`%s:'%s' `, k, v) 40 | } 41 | 42 | sb.WriteString(text) 43 | } 44 | 45 | return strings.TrimSpace(sb.String()) 46 | } 47 | 48 | // IsIncompatible returns "" if compatible, or a string of the platform this query is compatible with. 49 | func IsIncompatible(m *Metadata) string { 50 | other := "" 51 | if m.Platform != "" && m.Platform != runtime.GOOS { 52 | if m.Platform == "posix" { 53 | if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { 54 | other = "posix" 55 | } 56 | } else { 57 | other = m.Platform 58 | } 59 | } 60 | return other 61 | } 62 | 63 | func Run(m *Metadata) (*RunResult, error) { 64 | incompatible := IsIncompatible(m) 65 | 66 | cmd := exec.Command("osqueryi", "--json") 67 | stdin, err := cmd.StdinPipe() 68 | if err != nil { 69 | return nil, fmt.Errorf("error: %v", err) 70 | } 71 | 72 | go func() { 73 | defer stdin.Close() 74 | _, err := io.WriteString(stdin, m.Query) 75 | if err != nil { 76 | klog.Errorf("failed tos end data to osqueryi: %w", err) 77 | } 78 | }() 79 | 80 | start := time.Now() 81 | stdout, err := cmd.Output() 82 | elapsed := time.Since(start) 83 | 84 | ignoreError := false 85 | if err != nil { 86 | if ee, ok := err.(*exec.ExitError); ok { 87 | if incompatible != "" && ee.ExitCode() == 1 && bytes.Contains(ee.Stderr, []byte("no such table:")) { 88 | klog.Infof("partial test due to incompatible platform %q: %s", incompatible, strings.TrimSpace(string(ee.Stderr))) 89 | ignoreError = true 90 | } else { 91 | return nil, fmt.Errorf("%s [%w]: %s\nstdin: %s", cmd, err, ee.Stderr, m.Query) 92 | } 93 | } 94 | if !ignoreError { 95 | return nil, fmt.Errorf("%s: %w", cmd, err) 96 | } 97 | } 98 | 99 | rows := []Row{} 100 | err = json.Unmarshal(stdout, &rows) 101 | if err != nil { 102 | klog.Errorf("unable to parse output: %v", err) 103 | } 104 | 105 | return &RunResult{IncompatiblePlatform: incompatible, Rows: rows, Elapsed: elapsed}, nil 106 | } 107 | -------------------------------------------------------------------------------- /pkg/query/pack.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | "regexp" 12 | "strings" 13 | 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | type Pack struct { 18 | Queries map[string]*Metadata `json:"queries,omitempty"` 19 | Discovery map[string]*Metadata `json:"discovery,omitempty"` 20 | 21 | // Refer to obj.HasMember() calls in osquery/config/packs.cpp 22 | Shard int `json:"shard,omitempty"` 23 | Platform string `json:"platform,omitempty"` 24 | Version string `json:"version,omitempty"` 25 | Oncall string `json:"oncall,omitempty"` 26 | } 27 | 28 | // FlattenPacks flattens an array of Pack objects 29 | func FlattenPacks(ps []*Pack) *Pack { 30 | var c *Pack 31 | 32 | for _, p := range ps { 33 | for k, v := range p.Queries { 34 | c.Queries[k] = v 35 | } 36 | for k, v := range p.Discovery { 37 | c.Discovery[k] = v 38 | } 39 | 40 | c.Shard = p.Shard 41 | c.Platform = p.Platform 42 | c.Version = p.Version 43 | c.Oncall = p.Oncall 44 | } 45 | return c 46 | } 47 | 48 | type RenderConfig struct { 49 | SingleQuotes bool 50 | } 51 | 52 | // RenderPack renders an osquery pack file from a set of queries. 53 | func RenderPack(pack *Pack, c *RenderConfig) ([]byte, error) { 54 | out, err := json.MarshalIndent(pack, "", " ") 55 | if err != nil { 56 | return out, err 57 | } 58 | 59 | // This does not yet handle the case where someone double-quote: 60 | // a single quote, for example: mdfind.query="item == 'latest'" 61 | if c.SingleQuotes { 62 | out = bytes.ReplaceAll(out, []byte(`\"`), []byte("'")) 63 | } 64 | out = bytes.ReplaceAll(out, []byte(`\u003e`), []byte(">")) 65 | out = bytes.ReplaceAll(out, []byte(`\u003c`), []byte("<")) 66 | out = bytes.ReplaceAll(out, []byte(`\u0026`), []byte("&")) 67 | return out, nil 68 | 69 | } 70 | 71 | // LoadPack loads and parses an osquery pack file. 72 | func LoadPack(path string) (*Pack, error) { 73 | pack := &Pack{} 74 | var err error 75 | var bs []byte 76 | 77 | if path == "-" { 78 | r := bufio.NewReader(os.Stdin) 79 | bs, err = io.ReadAll(r) 80 | } else { 81 | bs, err = os.ReadFile(path) 82 | } 83 | if err != nil { 84 | return nil, fmt.Errorf("read: %v", err) 85 | } 86 | 87 | // workaround: invalid character '\n' in string escape code 88 | // replace trailing \ with \ 89 | bs = bytes.ReplaceAll(bs, []byte("\\\n"), []byte("\\\\n")) 90 | 91 | // workaround: cannot unmarshal number into Go struct field Metadata.queries.interval of type string 92 | nakedInterval := regexp.MustCompile(`"interval"\s*:\s*(\d+),`) 93 | bs = nakedInterval.ReplaceAll(bs, []byte("\"interval\": \"$1\",")) 94 | 95 | err = json.Unmarshal(bs, pack) 96 | if err != nil { 97 | return nil, fmt.Errorf("unmarshal: %v", err) 98 | } 99 | 100 | // Final repairs 101 | for name, v := range pack.Queries { 102 | v.Name = name 103 | 104 | if pack.Platform != "" && v.Platform == "" { 105 | v.Platform = pack.Platform 106 | } 107 | v.Query = strings.ReplaceAll(v.Query, "\\n", "\n") 108 | 109 | singles := []string{} 110 | for _, line := range strings.Split(v.Query, "\n") { 111 | singles = append(singles, strings.TrimSpace(line)) 112 | } 113 | v.SingleLineQuery = strings.Join(singles, " ") 114 | } 115 | 116 | return pack, nil 117 | } 118 | 119 | // SaveToDirectory saves a map of queries into a directory. 120 | func SaveToDirectory(mm map[string]*Metadata, destination string) error { 121 | for name, m := range mm { 122 | s, err := Render(m) 123 | if err != nil { 124 | return fmt.Errorf("render: %v", err) 125 | } 126 | 127 | bs := []byte(s) 128 | path := filepath.Join(destination, name+".sql") 129 | klog.Infof("Writing %d bytes to %s ...", len(bs), path) 130 | err = os.WriteFile(path, bs, 0o600) 131 | if err != nil { 132 | return fmt.Errorf("write file: %v", err) 133 | } 134 | } 135 | return nil 136 | } 137 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # The default runtime timeout is 1m, which doesn't work well on Github Actions. 3 | timeout: 4m 4 | 5 | # NOTE: This file is populated by the lint-install tool. Local adjustments may be overwritten. 6 | linters-settings: 7 | cyclop: 8 | # NOTE: This is a very high transitional threshold 9 | max-complexity: 37 10 | package-average: 34.0 11 | skip-tests: true 12 | 13 | gocognit: 14 | # NOTE: This is a very high transitional threshold 15 | min-complexity: 98 16 | 17 | dupl: 18 | threshold: 200 19 | 20 | goconst: 21 | min-len: 4 22 | min-occurrences: 5 23 | ignore-tests: true 24 | 25 | gofumpt: 26 | extra-rules: true 27 | 28 | gosec: 29 | excludes: 30 | - G107 # Potential HTTP request made with variable url 31 | - G204 # Subprocess launched with function call as argument or cmd arguments 32 | - G404 # Use of weak random number generator (math/rand instead of crypto/rand 33 | 34 | errorlint: 35 | # these are still common in Go: for instance, exit errors. 36 | asserts: false 37 | # Forcing %w in error wrapping forces authors to make errors part of their package APIs. The decision to make 38 | # an error part of a package API should be a concious decision by the author. 39 | # Also see Hyrums Law. 40 | errorf: false 41 | 42 | exhaustive: 43 | default-signifies-exhaustive: true 44 | 45 | funlen: 46 | lines: 100 47 | statements: 50 48 | 49 | nestif: 50 | min-complexity: 8 51 | 52 | nolintlint: 53 | require-explanation: true 54 | allow-unused: false 55 | require-specific: true 56 | 57 | revive: 58 | ignore-generated-header: true 59 | severity: warning 60 | rules: 61 | - name: atomic 62 | - name: blank-imports 63 | - name: bool-literal-in-expr 64 | - name: confusing-naming 65 | - name: constant-logical-expr 66 | - name: context-as-argument 67 | - name: context-keys-type 68 | - name: deep-exit 69 | - name: defer 70 | - name: range-val-in-closure 71 | - name: range-val-address 72 | - name: dot-imports 73 | - name: error-naming 74 | - name: error-return 75 | - name: error-strings 76 | - name: errorf 77 | - name: exported 78 | - name: identical-branches 79 | - name: if-return 80 | - name: import-shadowing 81 | - name: increment-decrement 82 | - name: indent-error-flow 83 | - name: indent-error-flow 84 | - name: package-comments 85 | - name: range 86 | - name: receiver-naming 87 | - name: redefines-builtin-id 88 | - name: superfluous-else 89 | - name: struct-tag 90 | - name: time-naming 91 | - name: unexported-naming 92 | - name: unexported-return 93 | - name: unnecessary-stmt 94 | - name: unreachable-code 95 | - name: unused-parameter 96 | - name: var-declaration 97 | - name: var-naming 98 | - name: unconditional-recursion 99 | - name: waitgroup-by-value 100 | 101 | staticcheck: 102 | go: "1.18" 103 | 104 | unused: 105 | go: "1.18" 106 | 107 | output: 108 | sort-results: true 109 | 110 | linters: 111 | disable-all: true 112 | enable: 113 | - asciicheck 114 | - bodyclose 115 | - cyclop 116 | - dogsled 117 | - dupl 118 | - durationcheck 119 | - errcheck 120 | - errname 121 | - errorlint 122 | - exhaustive 123 | - exportloopref 124 | - forcetypeassert 125 | - funlen 126 | - gocognit 127 | - goconst 128 | - gocritic 129 | - godot 130 | - gofmt 131 | - gofumpt 132 | - gosec 133 | - goheader 134 | - goimports 135 | - goprintffuncname 136 | - gosimple 137 | - gosec 138 | - govet 139 | - importas 140 | - ineffassign 141 | - makezero 142 | - misspell 143 | - nakedret 144 | - nestif 145 | - nilerr 146 | - noctx 147 | - nolintlint 148 | - predeclared 149 | # disabling for the initial iteration of the linting tool 150 | # - promlinter 151 | - revive 152 | # - rowserrcheck - disabled because of generics, https://github.com/golangci/golangci-lint/issues/2649 153 | # - sqlclosecheck - disabled because of generics, https://github.com/golangci/golangci-lint/issues/2649 154 | - staticcheck 155 | # - structcheck - disabled because of generics, https://github.com/golangci/golangci-lint/issues/2649 156 | - stylecheck 157 | - thelper 158 | - tparallel 159 | - typecheck 160 | - unconvert 161 | - unparam 162 | - unused 163 | # - wastedassign - disabled because of generics, https://github.com/golangci/golangci-lint/issues/2649 164 | - whitespace 165 | 166 | # Disabled linters, due to being misaligned with Go practices 167 | # - exhaustivestruct 168 | - gochecknoglobals 169 | - gochecknoinits 170 | - goconst 171 | # - godox 172 | # - goerr113 173 | # - gomnd 174 | # - lll 175 | # - nlreturn 176 | # - testpackage 177 | # - wsl 178 | # Disabled linters, due to not being relevant to our code base: 179 | # - maligned 180 | # - prealloc "For most programs usage of prealloc will be a premature optimization." 181 | # Disabled linters due to bad error messages or bugs 182 | # - tagliatelle 183 | 184 | issues: 185 | # Excluding configuration per-path, per-linter, per-text and per-source 186 | exclude-rules: 187 | - path: _test\.go 188 | linters: 189 | - dupl 190 | - errcheck 191 | - forcetypeassert 192 | - gocyclo 193 | - gosec 194 | - noctx 195 | 196 | - path: .*cmd.* 197 | linters: 198 | - noctx 199 | 200 | - path: main\.go 201 | linters: 202 | - noctx 203 | 204 | - path: .*cmd.* 205 | text: "deep-exit" 206 | 207 | - path: main\.go 208 | text: "deep-exit" 209 | 210 | # This check is of questionable value 211 | - linters: 212 | - tparallel 213 | text: "call t.Parallel on the top level as well as its subtests" 214 | 215 | # Don't hide lint issues just because there are many of them 216 | max-same-issues: 0 217 | max-issues-per-linter: 0 218 | -------------------------------------------------------------------------------- /pkg/query/query.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strconv" 9 | "strings" 10 | 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | type Metadata struct { 15 | // Refer to q.value.HasMember() calls in osquery/config/packs.cpp 16 | Query string `json:"query"` 17 | Interval string `json:"interval,omitempty"` 18 | Shard int `json:"shard,omitempty"` 19 | Platform string `json:"platform,omitempty"` 20 | Version string `json:"version,omitempty"` 21 | Description string `json:"description,omitempty"` 22 | 23 | Snapshot bool `json:"snapshot,omitempty"` 24 | Removed bool `json:"removed,omitempty"` 25 | DenyList bool `json:"denylist,omitempty"` 26 | 27 | // Custom fields 28 | ExtendedDescription string `json:"extended_description,omitempty"` // not an official field 29 | Value string `json:"value,omitempty"` // not an official field, but used in packs 30 | Name string `json:"-"` 31 | Tags []string `json:"-"` 32 | 33 | SingleLineQuery string `json:"-"` 34 | } 35 | 36 | // LoadFromDir recursively loads osquery queries from a directory. 37 | func LoadFromDir(path string) (map[string]*Metadata, error) { 38 | mm := map[string]*Metadata{} 39 | 40 | err := filepath.Walk(path, 41 | func(path string, info os.FileInfo, err error) error { 42 | if err != nil { 43 | return err 44 | } 45 | if strings.HasSuffix(path, ".sql") { 46 | klog.V(1).Infof("found query: %s", path) 47 | m, err := Load(path) 48 | if err != nil { 49 | return fmt.Errorf("load: %v", err) 50 | } 51 | mm[m.Name] = m 52 | } 53 | return nil 54 | }) 55 | 56 | return mm, err 57 | } 58 | 59 | // Load loads a query from a file. 60 | func Load(path string) (*Metadata, error) { 61 | bs, err := os.ReadFile(path) 62 | if err != nil { 63 | return nil, fmt.Errorf("read: %v", err) 64 | } 65 | 66 | name := strings.ReplaceAll(filepath.Base(path), ".sql", "") 67 | m, err := Parse(name, bs) 68 | if err != nil { 69 | return nil, fmt.Errorf("parse: %v", err) 70 | } 71 | 72 | return m, nil 73 | } 74 | 75 | // Render renders query metadata into a string. 76 | func Render(m *Metadata) (string, error) { 77 | lines := []string{} 78 | 79 | if m.Description != "" { 80 | lines = append(lines, fmt.Sprintf("-- %s", m.Description)) 81 | } 82 | 83 | // TODO: only add divider when necessary 84 | lines = append(lines, "--") 85 | 86 | if m.ExtendedDescription != "" { 87 | for _, ed := range strings.Split(m.ExtendedDescription, "\n") { 88 | lines = append(lines, fmt.Sprintf("-- %s", ed)) 89 | } 90 | lines = append(lines, "-- ") 91 | } 92 | 93 | if m.Interval != "" { 94 | lines = append(lines, fmt.Sprintf("-- interval: %s", m.Interval)) 95 | } 96 | 97 | if m.Platform != "" { 98 | lines = append(lines, fmt.Sprintf("-- platform: %s", m.Platform)) 99 | } 100 | 101 | if m.Shard > 0 { 102 | lines = append(lines, fmt.Sprintf("-- shard: %d", m.Shard)) 103 | } 104 | 105 | if m.Value != "" { 106 | lines = append(lines, fmt.Sprintf("-- value: %s", m.Value)) 107 | } 108 | 109 | if m.Version != "" { 110 | lines = append(lines, fmt.Sprintf("-- version: %s", m.Version)) 111 | } 112 | 113 | lines = append(lines, "") 114 | lines = append(lines, m.Query) 115 | 116 | return strings.Join(lines, "\n") + "\n", nil 117 | } 118 | 119 | // Parse parses query content and returns a Metadata object. 120 | func Parse(name string, bs []byte) (*Metadata, error) { //nolint: funlen // TODO: split into smaller functions 121 | // NOTE: The 'name' can be as simple as the file base path 122 | m := &Metadata{ 123 | Name: name, 124 | } 125 | 126 | out := []string{} 127 | for i, line := range bytes.Split(bs, []byte("\n")) { 128 | s := strings.TrimSuffix(string(line), "\n") 129 | 130 | // Wait a minute buckaroo, are you really trying to parse SQL? Have you considered --flags? 131 | // This is going to require work. 132 | before, after, hasComment := strings.Cut(s, "--") 133 | 134 | // " --x" 135 | if strings.Count(before, `"`)%2 == 1 && strings.Count(after, `"`)%2 == 1 { 136 | hasComment = false 137 | } 138 | // ' --x' 139 | if strings.Count(before, `'`)%2 == 1 && strings.Count(after, `'`)%2 == 1 { 140 | hasComment = false 141 | } 142 | 143 | if !hasComment { 144 | out = append(out, s) 145 | continue 146 | } 147 | 148 | if !strings.HasPrefix(strings.TrimSpace(s), "--") { 149 | out = append(out, before) 150 | continue 151 | } 152 | 153 | // If we are here, we have a leading comment - check for directives 154 | if i == 0 { 155 | m.Description = strings.TrimSpace(after) 156 | } 157 | 158 | after = strings.TrimSpace(after) 159 | directive, content, hasDirective := strings.Cut(strings.TrimSpace(after), ":") 160 | if hasDirective { 161 | content = strings.TrimSpace(content) 162 | } 163 | 164 | // See https://github.com/osquery/osquery/blob/4ee0be8000d59742d4fe86d2cb0a6241b79d11ff/osquery/config/packs.cpp 165 | switch directive { 166 | case "interval": 167 | m.Interval = content 168 | case "platform": 169 | m.Platform = content 170 | case "version": 171 | m.Version = content 172 | case "tags": 173 | m.Tags = strings.Split(content, " ") 174 | case "shard": 175 | shard, err := strconv.Atoi(content) 176 | if err != nil { 177 | return nil, err 178 | } 179 | m.Shard = shard 180 | case "value": 181 | m.Value = content 182 | } 183 | } 184 | 185 | m.Query = strings.TrimSpace(strings.Join(out, "\n")) 186 | 187 | // Single-line query form 188 | trimmed := []string{} 189 | for _, l := range out { 190 | trimmed = append(trimmed, strings.TrimSpace(l)) 191 | } 192 | m.SingleLineQuery = strings.TrimSpace(strings.Join(trimmed, " ")) 193 | 194 | if !strings.HasSuffix(m.Query, ";") { 195 | m.Query += ";" 196 | m.SingleLineQuery += ";" 197 | } 198 | 199 | guessPlatform := "" 200 | 201 | // If the platform field isn't filled in, try to guess via the name 202 | switch { 203 | case strings.HasSuffix(m.Name, "linux"): 204 | guessPlatform = "linux" 205 | case strings.HasSuffix(m.Name, "macos"): 206 | guessPlatform = "darwin" 207 | case strings.HasSuffix(m.Name, "darwin"): 208 | guessPlatform = "darwin" 209 | case strings.HasSuffix(m.Name, "posix"): 210 | guessPlatform = "posix" 211 | case strings.HasSuffix(m.Name, "unix"): 212 | guessPlatform = "posix" 213 | case strings.HasSuffix(m.Name, "windows"): 214 | guessPlatform = "windows" 215 | case strings.HasSuffix(m.Name, "win"): 216 | guessPlatform = "windows" 217 | } 218 | 219 | switch m.Platform { 220 | case "macos": 221 | m.Platform = "darwin" 222 | case "unix": 223 | m.Platform = "posix" 224 | case "win": 225 | m.Platform = "windows" 226 | case "": 227 | m.Platform = guessPlatform 228 | } 229 | 230 | if guessPlatform != "" && m.Platform != guessPlatform { 231 | return m, fmt.Errorf("platform is set to %q, but filename indicates %q", m.Platform, guessPlatform) 232 | } 233 | 234 | return m, nil 235 | } 236 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # osqtool 2 | 3 | [![Actions Status](https://github.com/chainguard-dev/osqtool/workflows/Go/badge.svg)](https://github.com/chainguard-dev/osqtool/actions) 4 | [![Go Report](https://goreportcard.com/badge/github.com/chainguard-dev/osqtool)](https://goreportcard.com/badge/github.com/chainguard-dev/osqtool) 5 | [![Latest Release](https://img.shields.io/github/v/release/chainguard-dev/osqtool?include_prereleases)](https://github.com/chainguard-dev/osqtool/releases/latest) 6 | [![stable](http://badges.github.io/stability-badges/dist/stable.svg)](http://github.com/badges/stability-badges) 7 | 8 | A swiss-army tool for testing, creating, and manipulating [osquery](https://osquery.io/) query packs. 9 | 10 | ![osqtool logo](images/logo-small.png?raw=true "osqtool logo") 11 | 12 | ## Installation 13 | 14 | With [Go](https://go.dev/) v1.20+ installed, run: 15 | 16 | ```shell 17 | go install github.com/chainguard-dev/osqtool/cmd/osqtool@latest 18 | ``` 19 | 20 | ## Usage 21 | 22 | osqtool supports 4 commands: 23 | 24 | * `apply` - programatically manipulate an osquery query pack, for instance, adjusting intervals 25 | * `pack` - create a JSON pack file from a directory of raw SQL files 26 | * `unpack` - extract raw SQL files from a JSON query pack file 27 | * `run` - run an osquery pack file or directory of SQL queries with human and diff-friendly output 28 | * `verify` - verify that the queries in a query pack, directory, or raw SQL file are valid and test well 29 | 30 | ### apply 31 | 32 | Want to take an osquery pack from the internet, but make changes to it programatically? osqtool has you covered: 33 | 34 | ```shell 35 | curl https://raw.githubusercontent.com/osquery/osquery/master/packs/it-compliance.conf \ 36 | | osqtool --min-interval=8h --platforms linux,posix --exclude os_version apply - 37 | ``` 38 | 39 | This will set all queries to an 8-hour interval, remove Windows-specific queries, and exclude a query named `os_version`. 40 | 41 | ### Pack 42 | 43 | Create an osquery pack configuration from a recursive directory of SQL files: 44 | 45 | ```shell 46 | osqtool pack /tmp/osx-attacks 47 | ``` 48 | 49 | Here's the example output: 50 | 51 | ```json 52 | { 53 | "queries": { 54 | "Aobo_Keylogger": { 55 | "query": "select * from launchd where name like 'com.ab.kl%.plist';", 56 | "interval": "3600", 57 | "version": "1.4.5", 58 | "description": "(http://aobo.cc/aobo-mac-os-x-keylogger.html)", 59 | "value": "Artifact used by this malware" 60 | }, 61 | "Backdoor_MAC_Eleanor": { 62 | "query": "SELECT * FROM launchd WHERE name IN ('com.getdropbox.dropbox.integritycheck.plist','com.getdropbox.dropbox.timegrabber.plist','com.getdropbox.dropbox.usercontent.plist');", 63 | "interval": "3600", 64 | "version": "1.4.5", 65 | "description": "(https://blog.malwarebytes.com/cybercrime/2016/07/new-mac-backdoor-malware-eleanor/)", 66 | "value": "Artifact used by this malware" 67 | }, 68 | ... 69 | ``` 70 | 71 | The `pack` command supports the same flags as the `apply` command. In particular, you may find `--exclude`, `--exclude-tags`, and `--verify` useful. 72 | 73 | ### Run 74 | 75 | Run a set of queries! 76 | 77 | ```shell 78 | osqtool run incident-response.conf 79 | ``` 80 | 81 | Example output: 82 | 83 | ```log 84 | block_devices (7 rows) 85 | ---------------------- 86 | block_size:512 label: model: name:/dev/nvme0n1 parent: size:488397168 type: uuid: vendor: 87 | block_size: label: model: name:/dev/nvme0n1p1 parent:/dev/nvme0n1 size:614400 type: uuid: vendor: 88 | block_size: label: model: name:/dev/nvme0n1p2 parent:/dev/nvme0n1 size:415929892 type: uuid: vendor: 89 | block_size: label: model: name:/dev/nvme0n1p3 parent:/dev/nvme0n1 size:71843677 type: uuid: vendor: 90 | block_size:512 label: model:'SD Card Reader' name:/dev/sda parent: size:0 type: uuid: vendor:CalDigit 91 | block_size:512 label: model: name:/dev/dm-0 parent: size:415925796 type: uuid: vendor: 92 | block_size:512 label: model: name:/dev/dm-1 parent: size:71839581 type: uuid: vendor: 93 | 94 | crontab (1 rows) 95 | ---------------- 96 | command:'root run-parts /etc/cron.hourly' day_of_month:* day_of_week:* event: hour:* minute:01 month:* path:/etc/cron.d/0hourly 97 | 98 | disk_encryption (0 rows) 99 | ``` 100 | 101 | ### Unpack 102 | 103 | Extract an osquery pack into a directory of SQL files: 104 | 105 | ```shell 106 | osqtool --output=/tmp/osx-attacks unpack osx-attacks.conf 107 | ``` 108 | 109 | Here is example output: 110 | 111 | ```log 112 | Writing 745 bytes to /tmp/out/OceanLotus_dropped_file_1.sql ... 113 | Writing 268 bytes to /tmp/out/OSX_MaMi_DNS_Servers.sql ... 114 | Writing 328 bytes to /tmp/out/OSX_ColdRoot_RAT_Files.sql ... 115 | Writing 209 bytes to /tmp/out/iWorm.sql ... 116 | 74 queries saved to /tmp/out 117 | ``` 118 | 119 | The `unpack` command supports the same flags as the `apply` command. 120 | 121 | 122 | ### Verify 123 | 124 | Verify that the queries are valid in a pack, SQL file, or directory of SQL files 125 | 126 | ```shell 127 | osqtool verify /tmp/detect 128 | ``` 129 | 130 | Example output: 131 | 132 | ```log 133 | Verifying "high-disk-bytes-written" ... 134 | high-disk-bytes-written" returned 0 rows within 264.361831ms 135 | Verifying "unexpected-shell-parents" ... 136 | "unexpected-shell-parents" failed validation: /sbin/osqueryi --json [exit status 1]: Error: near line 1: near "sh": syntax error 137 | 78 queries found: 55 verified, 10 errored, 13 skipped 138 | "verify" failed: 10 errors occurred: 139 | * xprotect-reports: /sbin/osqueryi --json [exit status 1]: Error: near line 1: no such table: xprotect_reports 140 | ``` 141 | 142 | You can set limits on the number of rows returned, amount of runtime per query, per day, or across the pack, see `--help` for more information. 143 | 144 | ### Common Flags 145 | 146 | Here are the options that are available to `apply`, `unpack`, `pack`, and `verify` 147 | 148 | ``` 149 | -default-interval duration 150 | Interval to use for queries which do not specify one (default 1h0m0s) 151 | -exclude string 152 | Comma-separated list of queries to exclude 153 | -exclude-tags string 154 | Comma-separated list of tags to exclude (default "disabled") 155 | -max-interval duration 156 | Queries can't be scheduled more often than this (default 15s) 157 | -max-query-daily-duration duration 158 | Maximum duration for a single query multiplied by how many times it runs daily (checked during --verify) (default 1h0m0s) 159 | -max-query-duration duration 160 | Maximum query duration (checked during --verify) (default 4s) 161 | -max-results int 162 | Maximum number of results a query may return during verify (default 1000) 163 | -max-total-daily-duration duration 164 | Maximum total query-duration per day across all queries (default 6h0m0s) 165 | -min-interval duration 166 | Queries cant be scheduled less often than this (default 24h0m0s) 167 | -multi-line 168 | output queries is multi-line form. This is accepted by osquery, but technically is invalid JSON. 169 | -output string 170 | Location of output 171 | -platforms string 172 | Comma-separated list of platforms to include 173 | -single-quotes 174 | Render double quotes as single quotes (may corrupt queries) 175 | -skip_headers 176 | If true, avoid header prefixes in the log messages 177 | -tag-intervals string 178 | modifiers to the default-interval based on query tags (default "transient=5m,postmortem=6h,rapid=15s,often=x/4,seldom=2x") 179 | -verify 180 | Verify the output 181 | -workers int 182 | Number of workers to use when verifying results (0 for automatic) 183 | ``` 184 | 185 | At the moment, flags must be declared before the subcommand. `¯\_(ツ)_/¯` 186 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cmd/osqtool/main.go: -------------------------------------------------------------------------------- 1 | // osqtool operates on osquery query and pack files 2 | // 3 | // Copyright 2022 Chainguard, Inc. 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "flag" 11 | "fmt" 12 | "os" 13 | "os/exec" 14 | "runtime" 15 | "sort" 16 | "strconv" 17 | "strings" 18 | "sync/atomic" 19 | "time" 20 | "unicode/utf8" 21 | 22 | "github.com/chainguard-dev/osqtool/pkg/query" 23 | "github.com/fatih/semgroup" 24 | "k8s.io/klog/v2" 25 | ) 26 | 27 | // Config is a struct representation of our flags. 28 | type Config struct { 29 | maxQueryDuration time.Duration 30 | maxQueryDurationPerDay time.Duration 31 | MaxTotalQueryDurationPerDay time.Duration 32 | MinInterval time.Duration 33 | MaxInterval time.Duration 34 | DefaultInterval time.Duration 35 | TagIntervals []string 36 | Exclude []string 37 | ExcludeTags []string 38 | Platforms []string 39 | Workers int 40 | MaxResults int 41 | SingleQuotes bool 42 | MultiLine bool 43 | } 44 | 45 | func main() { 46 | outputFlag := flag.String("output", "", "Location of output") 47 | minIntervalFlag := flag.Duration("max-interval", 20*time.Second, "Queries can't be scheduled more often than this") 48 | multiLineFlag := flag.Bool("multi-line", false, "output queries is multi-line form. This is accepted by osquery, but technically is invalid JSON.") 49 | defaultIntervalFlag := flag.Duration("default-interval", 1*time.Hour, "Interval to use for queries which do not specify one") 50 | tagIntervalsFlag := flag.String("tag-intervals", "transient=6m,persistent=1.25x,postmortem=6h,rapid=20s,often=x/3,seldom=3x", "modifiers to the default-interval based on query tags") 51 | maxIntervalFlag := flag.Duration("min-interval", 24*time.Hour, "Queries cant be scheduled less often than this") 52 | excludeFlag := flag.String("exclude", "", "Comma-separated list of queries to exclude") 53 | excludeTagsFlag := flag.String("exclude-tags", "disabled", "Comma-separated list of tags to exclude") 54 | platformsFlag := flag.String("platforms", "", "Comma-separated list of platforms to include") 55 | workersFlag := flag.Int("workers", 0, "Number of workers to use when verifying results (0 for automatic)") 56 | maxResultsFlag := flag.Int("max-results", 250000, "Maximum number of results a query may return during verify") 57 | singleQuotesFlag := flag.Bool("single-quotes", false, "Render double quotes as single quotes (may corrupt queries)") 58 | maxQueryDurationFlag := flag.Duration("max-query-duration", 4*time.Second, "Maximum query duration (checked during --verify)") 59 | maxQueryDurationPerDayFlag := flag.Duration("max-query-daily-duration", 60*time.Minute, "Maximum duration for a single query multiplied by how many times it runs daily (checked during --verify)") 60 | maxTotalQueryDurationFlag := flag.Duration("max-total-daily-duration", 6*time.Hour, "Maximum total query-duration per day across all queries") 61 | verifyFlag := flag.Bool("verify", false, "Verify queries quickly") 62 | 63 | klog.InitFlags(nil) 64 | flag.Parse() 65 | args := flag.Args() 66 | 67 | if len(args) < 2 { 68 | klog.Exitf("usage: osqtool [apply|pack|run|unpack|verify] ") 69 | } 70 | 71 | action := args[0] 72 | paths := args[1:] 73 | var err error 74 | c := Config{ 75 | maxQueryDuration: *maxQueryDurationFlag, 76 | maxQueryDurationPerDay: *maxQueryDurationPerDayFlag, 77 | MaxTotalQueryDurationPerDay: *maxTotalQueryDurationFlag, 78 | MinInterval: *minIntervalFlag, 79 | MaxInterval: *maxIntervalFlag, 80 | MaxResults: *maxResultsFlag, 81 | DefaultInterval: *defaultIntervalFlag, 82 | TagIntervals: strings.Split(*tagIntervalsFlag, ","), 83 | Exclude: strings.Split(*excludeFlag, ","), 84 | ExcludeTags: strings.Split(*excludeTagsFlag, ","), 85 | Platforms: strings.Split(*platformsFlag, ","), 86 | Workers: *workersFlag, 87 | SingleQuotes: *singleQuotesFlag, 88 | MultiLine: *multiLineFlag, 89 | } 90 | 91 | if c.Workers < 1 { 92 | c.Workers = runtime.NumCPU() 93 | if *verifyFlag || action == "verify" { 94 | klog.Infof("automatically setting verify worker count to %d", c.Workers) 95 | } 96 | } 97 | 98 | if *verifyFlag || action == "verify" { 99 | if _, err := exec.LookPath("osqueryi"); err != nil { 100 | klog.Exit(fmt.Errorf("osqueryi executable not found on the host! Download it from: https://osquery.io/downloads")) 101 | } 102 | 103 | err = Verify(paths, c) 104 | if err != nil { 105 | klog.Exitf("verify failed: %v", err) 106 | } 107 | } 108 | 109 | switch action { 110 | case "apply": 111 | err = Apply(paths, *outputFlag, c) 112 | case "pack": 113 | err = Pack(paths, *outputFlag, c) 114 | case "unpack": 115 | err = Unpack(paths, *outputFlag, c) 116 | case "verify": 117 | err = Verify(paths, c) 118 | case "run": 119 | err = Run(paths, *outputFlag, c) 120 | default: 121 | err = fmt.Errorf("unknown action") 122 | } 123 | if err != nil { 124 | klog.Exitf("%q failed: %v", action, err) 125 | } 126 | } 127 | 128 | // calculateInterval calculates the default interval to use for a query. 129 | func calculateInterval(m *query.Metadata, c Config) int { 130 | tagMap := map[string]bool{} 131 | for _, t := range m.Tags { 132 | tagMap[t] = true 133 | } 134 | 135 | interval := int(c.DefaultInterval.Seconds()) 136 | 137 | for _, k := range c.TagIntervals { 138 | tag, modifier, found := strings.Cut(k, "=") 139 | klog.V(1).Infof("processing tag interval: %s=%s (map: %v) - currently: %d", tag, modifier, tagMap, interval) 140 | 141 | if !found { 142 | klog.Errorf("unparseable tag interval: %v", k) 143 | continue 144 | } 145 | 146 | if !tagMap[tag] { 147 | klog.V(1).Infof("%s is not mentioned by this query, moving on", tag) 148 | continue 149 | } 150 | 151 | if i, err := strconv.Atoi(modifier); err == nil { 152 | klog.V(1).Infof("%s is an int, setting interval to %d", modifier, i) 153 | interval = i 154 | continue 155 | } 156 | 157 | if d, err := time.ParseDuration(modifier); err == nil { 158 | klog.V(1).Infof("%s is a duration, setting interval to %0.f", modifier, d.Seconds()) 159 | interval = int(d.Seconds()) 160 | continue 161 | } 162 | 163 | switch { 164 | case strings.HasSuffix(modifier, "x"): 165 | x, err := strconv.ParseFloat(strings.Trim(modifier, "x"), 64) 166 | if err != nil { 167 | klog.Errorf("unparseable tag multiplier: %v", modifier) 168 | continue 169 | } 170 | 171 | klog.V(1).Infof("multiplying interval by %d", x) 172 | interval = int(float64(interval) * x) 173 | case strings.Contains(modifier, "x/"): 174 | _, divisor, found := strings.Cut(k, "/") 175 | if !found { 176 | klog.Errorf("unparseable tag denominator: %v", k) 177 | continue 178 | } 179 | 180 | if d, err := strconv.ParseFloat(divisor, 64); err == nil { 181 | klog.V(1).Infof("dividing interval by %d", d) 182 | interval = int(float64(interval) / d) 183 | } 184 | default: 185 | klog.Errorf("do not understand modifier: %s", k) 186 | } 187 | } 188 | return interval 189 | } 190 | 191 | // TODO: Move config application to pkg/query. 192 | func applyConfig(mm map[string]*query.Metadata, c Config) error { 193 | klog.V(1).Infof("applying config: %+v", c) 194 | minSeconds := int(c.MinInterval.Seconds()) 195 | maxSeconds := int(c.MaxInterval.Seconds()) 196 | excludeMap := map[string]bool{} 197 | for _, v := range c.Exclude { 198 | if v == "" { 199 | continue 200 | } 201 | excludeMap[v] = true 202 | } 203 | 204 | excludeTagsMap := map[string]bool{} 205 | for _, v := range c.ExcludeTags { 206 | if v != "" { 207 | excludeTagsMap[v] = true 208 | } 209 | } 210 | 211 | platformsMap := map[string]bool{} 212 | for _, v := range c.Platforms { 213 | if v == "" { 214 | continue 215 | } 216 | 217 | platformsMap[v] = true 218 | } 219 | 220 | for name, m := range mm { 221 | if !c.MultiLine { 222 | m.Query = m.SingleLineQuery 223 | } 224 | 225 | if excludeMap[name] { 226 | klog.Infof("Skipping %s,excluded by --exclude", name) 227 | delete(mm, name) 228 | continue 229 | } 230 | 231 | for _, t := range m.Tags { 232 | if excludeTagsMap[t] { 233 | klog.Infof("Skipping %s, excluded by --exclude-tags=%s", name, t) 234 | delete(mm, name) 235 | continue 236 | } 237 | } 238 | 239 | if len(platformsMap) > 0 && m.Platform != "" && !platformsMap[m.Platform] { 240 | klog.Infof("Skipping %s - %q not listed in --platforms", name, m.Platform) 241 | delete(mm, name) 242 | continue 243 | } 244 | 245 | if m.Interval == "" { 246 | interval := calculateInterval(m, c) 247 | klog.V(1).Infof("setting %q interval to %ds", name, interval) 248 | m.Interval = strconv.Itoa(interval) 249 | } 250 | 251 | i, err := strconv.Atoi(m.Interval) 252 | if err != nil { 253 | return fmt.Errorf("%q: failed to parse %q: %w", name, m.Interval, err) 254 | } 255 | 256 | if i > maxSeconds { 257 | klog.Infof("overriding %q interval to %ds (max)", name, maxSeconds) 258 | m.Interval = strconv.Itoa(maxSeconds) 259 | } 260 | if i < minSeconds { 261 | klog.Infof("overriding %q interval to %ds (min)", name, minSeconds) 262 | m.Interval = strconv.Itoa(minSeconds) 263 | } 264 | } 265 | return nil 266 | } 267 | 268 | // Apply applies programattic changes to an osquery pack. 269 | func Apply(sourcePaths []string, output string, c Config) error { 270 | ps := []*query.Pack{} 271 | 272 | for _, path := range sourcePaths { 273 | p, err := query.LoadPack(path) 274 | if err != nil { 275 | return fmt.Errorf("load pack: %v", err) 276 | } 277 | 278 | if err := applyConfig(p.Queries, c); err != nil { 279 | return fmt.Errorf("apply: %w", err) 280 | } 281 | ps = append(ps, p) 282 | } 283 | 284 | p := query.FlattenPacks(ps) 285 | bs, err := query.RenderPack(p, &query.RenderConfig{SingleQuotes: c.SingleQuotes}) 286 | if err != nil { 287 | return fmt.Errorf("render: %v", err) 288 | } 289 | 290 | if output == "" { 291 | _, err = fmt.Println(string(bs)) 292 | return err 293 | } 294 | 295 | return os.WriteFile(output, bs, 0o600) 296 | } 297 | 298 | // Pack creates an osquery pack from a recursive directory of SQL files. 299 | func Pack(sourcePaths []string, output string, c Config) error { 300 | mms := map[string]*query.Metadata{} 301 | for _, path := range sourcePaths { 302 | klog.Infof("Loading from %s ...", path) 303 | mm, err := query.LoadFromDir(path) 304 | if err != nil { 305 | return fmt.Errorf("load from dir %s: %v", path, err) 306 | } 307 | 308 | if err := applyConfig(mm, c); err != nil { 309 | return fmt.Errorf("apply: %w", err) 310 | } 311 | for k, v := range mm { 312 | mms[k] = v 313 | } 314 | } 315 | 316 | klog.Infof("Packing %d queries into %s ...", len(mms), output) 317 | bs, err := query.RenderPack(&query.Pack{Queries: mms}, &query.RenderConfig{SingleQuotes: c.SingleQuotes}) 318 | if err != nil { 319 | return fmt.Errorf("render: %v", err) 320 | } 321 | 322 | if output == "" { 323 | _, err = fmt.Println(string(bs)) 324 | return err 325 | } 326 | 327 | return os.WriteFile(output, bs, 0o600) 328 | } 329 | 330 | // Unpack extracts SQL files from an osquery pack. 331 | func Unpack(sourcePaths []string, destPath string, c Config) error { 332 | if destPath == "" { 333 | destPath = "." 334 | } 335 | 336 | mms := map[string]*query.Metadata{} 337 | for _, path := range sourcePaths { 338 | p, err := query.LoadPack(path) 339 | if err != nil { 340 | return fmt.Errorf("load pack %s: %v", path, err) 341 | } 342 | 343 | if err := applyConfig(p.Queries, c); err != nil { 344 | return fmt.Errorf("apply: %w", err) 345 | } 346 | 347 | for k, v := range p.Queries { 348 | mms[k] = v 349 | } 350 | 351 | } 352 | 353 | err := query.SaveToDirectory(mms, destPath) 354 | if err != nil { 355 | return fmt.Errorf("save to dir: %v", err) 356 | } 357 | fmt.Printf("%d queries saved to %s\n", len(mms), destPath) 358 | return nil 359 | } 360 | 361 | // dailyQueryDuration returns what the total duration for a query would be for a day. 362 | func dailyQueryDuration(interval string, d time.Duration) (time.Duration, int, error) { 363 | i, err := strconv.Atoi(interval) 364 | if err != nil { 365 | return time.Duration(0), 0, err 366 | } 367 | 368 | runs := 86400 / i 369 | return time.Duration(runs) * d, runs, nil 370 | } 371 | 372 | func loadAndApply(paths []string, c Config) (map[string]*query.Metadata, error) { 373 | mm := map[string]*query.Metadata{} 374 | 375 | for _, path := range paths { 376 | s, err := os.Stat(path) 377 | if err != nil { 378 | return nil, fmt.Errorf("stat: %w", err) 379 | } 380 | 381 | loaded := map[string]*query.Metadata{} 382 | switch { 383 | case s.IsDir(): 384 | loaded, err = query.LoadFromDir(path) 385 | if err != nil { 386 | return mm, fmt.Errorf("load from dir %s: %w", path, err) 387 | } 388 | case strings.Contains(path, ".conf"): 389 | p, err := query.LoadPack(path) 390 | if err != nil { 391 | return mm, fmt.Errorf("load pack %s: %w", path, err) 392 | } 393 | loaded = p.Queries 394 | default: 395 | m, err := query.Load(path) 396 | if err != nil { 397 | return mm, fmt.Errorf("load %s: %w", path, err) 398 | } 399 | loaded[m.Name] = m 400 | } 401 | 402 | for k, v := range loaded { 403 | if mm[k] != nil { 404 | return mm, fmt.Errorf("conflict: %q already loaded", k) 405 | } 406 | mm[k] = v 407 | } 408 | 409 | klog.Infof("Loaded %d queries from %s", len(loaded), path) 410 | } 411 | 412 | klog.Infof("Applying configuration to %d queries: %+v", len(mm), c) 413 | if err := applyConfig(mm, c); err != nil { 414 | return mm, fmt.Errorf("apply: %w", err) 415 | } 416 | 417 | return mm, nil 418 | } 419 | 420 | // Run runs the queries within a directory or pack. 421 | func Run(path []string, output string, c Config) error { 422 | mm, err := loadAndApply(path, c) 423 | if err != nil { 424 | return err 425 | } 426 | 427 | f := os.Stdout 428 | if output != "" && output != "-" { 429 | f, err = os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0o700) 430 | if err != nil { 431 | 432 | return fmt.Errorf("unable to open output: %s", err) 433 | } 434 | } 435 | 436 | errs := []error{} 437 | qs := []*query.Metadata{} 438 | for _, q := range mm { 439 | qs = append(qs, q) 440 | } 441 | 442 | sort.Slice(qs, func(i, j int) bool { return qs[i].Name < qs[j].Name }) 443 | lastRows := -1 444 | 445 | // TODO: Parallelize. Output must be sorted for diffing 446 | for _, m := range qs { 447 | m := m 448 | name := m.Name 449 | 450 | if cw := query.IsIncompatible(m); cw != "" { 451 | klog.V(1).Infof("skipping incompatible query: %s (%s)", name, cw) 452 | continue 453 | } 454 | 455 | vf, verr := query.Run(m) 456 | if verr != nil { 457 | klog.Errorf("%q failed: %v", name, verr) 458 | errs = append(errs, verr) 459 | continue 460 | } 461 | 462 | // TODO: Consider CSV output 463 | header := fmt.Sprintf("%s (%d rows)", name, len(vf.Rows)) 464 | 465 | // If this is a big entry after a short entry, add a space 466 | if lastRows == 0 && len(vf.Rows) > 0 { 467 | fmt.Fprintln(f, "") 468 | } 469 | fmt.Fprintln(f, header) 470 | 471 | lastRows = len(vf.Rows) 472 | if len(vf.Rows) == 0 { 473 | continue 474 | } 475 | 476 | divider := strings.Repeat("-", utf8.RuneCountInString(header)) 477 | fmt.Fprintln(f, divider) 478 | for _, v := range vf.Rows { 479 | fmt.Fprintln(f, v) 480 | } 481 | fmt.Fprintln(f, "") 482 | } 483 | 484 | return errors.Join(errs...) 485 | } 486 | 487 | // Verify verifies the queries within a directory or pack. 488 | func Verify(path []string, c Config) error { 489 | mm, err := loadAndApply(path, c) 490 | if err != nil { 491 | return err 492 | } 493 | 494 | var ( 495 | verified, partial uint64 496 | totalQueryDuration time.Duration 497 | totalRuns int64 498 | ) 499 | 500 | sg := semgroup.NewGroup(context.Background(), int64(c.Workers)) 501 | 502 | for name, m := range mm { 503 | m := m 504 | name := name 505 | 506 | sg.Go(func() error { 507 | klog.Infof("Verifying: %q ", name) 508 | vf, verr := query.Run(m) 509 | if verr != nil { 510 | klog.Errorf("%q failed validation: %v", name, verr) 511 | return fmt.Errorf("%s: %w", name, verr) 512 | } 513 | 514 | // Short-circuit out of remaining tests if the query is not compatible with the local platform 515 | if vf.IncompatiblePlatform != "" { 516 | atomic.AddUint64(&partial, 1) 517 | return nil 518 | } 519 | 520 | if vf.Elapsed > c.maxQueryDuration { 521 | return fmt.Errorf("%q: %s exceeds --max-query-duration=%s", name, vf.Elapsed.Round(time.Millisecond), c.maxQueryDuration) 522 | } 523 | 524 | queryDurationPerDay, runsPerDay, err := dailyQueryDuration(m.Interval, vf.Elapsed) 525 | if err != nil { 526 | return fmt.Errorf("%q: failed to parse interval: %v", name, err) 527 | } 528 | 529 | atomic.AddInt64((*int64)(&totalQueryDuration), int64(queryDurationPerDay)) 530 | atomic.AddInt64((&totalRuns), int64(runsPerDay)) 531 | 532 | if queryDurationPerDay > c.maxQueryDurationPerDay { 533 | return fmt.Errorf("%q: %s exceeds --max-daily-query-duration=%s (%d runs * %s)", name, queryDurationPerDay.Round(time.Second), c.maxQueryDurationPerDay, runsPerDay, vf.Elapsed.Round(time.Millisecond)) 534 | } 535 | 536 | if len(vf.Rows) > c.MaxResults { 537 | shortResult := []string{} 538 | for _, r := range vf.Rows { 539 | shortResult = append(shortResult, r.String()) 540 | } 541 | if len(shortResult) >= 10 { 542 | shortResult = shortResult[0:10] 543 | shortResult = append(shortResult, "...") 544 | } 545 | 546 | return fmt.Errorf("%q: %d results exceeds --max-results=%d:\n %s", name, len(vf.Rows), c.MaxResults, strings.Join(shortResult, "\n ")) 547 | } 548 | 549 | klog.Infof("%q returned %d rows in %s, daily cost for interval %s (%d runs): %s", name, len(vf.Rows), vf.Elapsed.Round(time.Millisecond), m.Interval, runsPerDay, queryDurationPerDay.Round(time.Second)) 550 | atomic.AddUint64(&verified, 1) 551 | return nil 552 | }) 553 | } 554 | 555 | errs := []error{} 556 | // Someday this might return new go errors 557 | errs = append(errs, sg.Wait()) 558 | errored := uint64(len(errs)) 559 | 560 | if verified == 0 { 561 | errs = append(errs, fmt.Errorf("0 queries were fully verified")) 562 | } 563 | 564 | if totalQueryDuration > c.MaxTotalQueryDurationPerDay { 565 | errs = append(errs, fmt.Errorf("total query duration per day (%s) exceeds --max-total-daily-duration=%s", totalQueryDuration.Round(time.Second), c.MaxTotalQueryDurationPerDay)) 566 | } 567 | 568 | klog.Infof("%d queries found: %d verified, %d errored, %d partial", len(mm), verified, errored, partial) 569 | klog.Infof("total daily query runs: %d", totalRuns) 570 | klog.Infof("total daily execution time: %s", totalQueryDuration) 571 | 572 | return errors.Join(errs...) 573 | } 574 | --------------------------------------------------------------------------------