├── .github └── workflows │ ├── codeql-analysis.yml │ ├── dependency-scanning.yml │ ├── lint.yml │ ├── push.yml │ ├── release.yaml │ └── secret-scanning.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yaml ├── AGENT.md ├── LICENSE ├── Makefile ├── README.md ├── TODO.md ├── cmd ├── experiments │ ├── function_finder │ │ ├── function_finder │ │ └── main.go │ ├── run-examples.sh │ ├── ts-docs │ │ ├── README.md │ │ ├── main.go │ │ ├── sample-api.md │ │ ├── sample.ts │ │ └── ts-docs.sh │ └── typescript_analyzer │ │ ├── main.go │ │ └── typescript_analyzer └── oak │ ├── commands │ ├── legacy.go │ ├── root.go │ └── run.go │ ├── doc │ ├── create-query.md │ ├── glaze-output.md │ └── glob.md │ ├── main.go │ └── queries │ ├── equals.yaml │ ├── example1.yaml │ ├── go │ ├── comments.yaml │ ├── consts.yaml │ ├── definitions.yaml │ └── structs.yaml │ ├── php │ ├── classes.yaml │ ├── functions.yaml │ └── wp │ │ └── filters.yaml │ └── typescript │ ├── definitions.yaml │ ├── functions.yaml │ ├── tests.yaml │ └── types.yaml ├── contexts ├── dsl.txt ├── generate-api-query.sh ├── goal.txt ├── goals │ ├── parser.txt │ └── spec.txt ├── golang-example.txt ├── golang-types.txt ├── instructions │ └── just-code.txt ├── prompt.sh ├── test.go ├── tree-sitter-api.txt ├── tree-sitter-doc.md ├── tree-sitter-predicates-examples.md └── tree-sitter-query.md ├── go.mod ├── go.sum ├── lefthook.yml ├── pinocchio └── oak │ └── create-command.yaml ├── pkg ├── api │ └── query_builder.go ├── cmds │ ├── cmd.go │ ├── glazed.go │ ├── layers │ │ └── oak.yaml │ └── writer.go ├── doc │ ├── doc.go │ └── topics │ │ ├── 01-programmatic-oak-api.md │ │ └── 02-tree-dump-formats.md ├── lang.go ├── queries.go └── tree-sitter │ ├── dump.go │ ├── dump │ ├── dump.go │ ├── json.go │ ├── text.go │ ├── xml.go │ └── yaml.go │ └── tree-sitter.go ├── prompto ├── oak │ └── go-definitions └── typescript │ └── definitions ├── queries ├── create-description.yaml ├── describe.txt ├── descriptions.txt ├── example1.yaml ├── examples │ └── go │ │ └── returns.yaml ├── go-func-declaration.txt ├── go-spec-fixed.yaml ├── go-spec-fixed2.yaml ├── go-spec.yaml ├── it.txt ├── match-should.txt ├── match-should2.txt ├── match-should3.txt ├── should.yaml ├── spec.txt ├── spec.yaml ├── spec2.yaml ├── test-constant.txt └── test.yaml ├── test-inputs ├── constants.js ├── ecs.hcl ├── go-example.go ├── php │ ├── all.php │ ├── class.php │ └── finalClass.php ├── returns.go ├── should4.js ├── test.go └── typescript │ ├── App.tsx │ ├── Component.tsx │ ├── const-fn.ts │ ├── enum.ts │ ├── interface.ts │ └── types.ts └── ttmp └── 2025-05-22 ├── 01-design-ideas-for-an-elegant-oak-api.md ├── 02-design-for-an-elegant-oak-api.md └── 03-proposals-for-dump-tree-output-format.md /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL Analysis" 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | schedule: 9 | - cron: '0 0 * * 0' # Run weekly on Sunday at midnight 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-latest 15 | permissions: 16 | security-events: write 17 | 18 | steps: 19 | - name: Checkout repository 20 | uses: actions/checkout@v4 21 | 22 | - name: Initialize CodeQL 23 | uses: github/codeql-action/init@v3 24 | with: 25 | languages: go 26 | 27 | - name: Perform CodeQL Analysis 28 | uses: github/codeql-action/analyze@v3 -------------------------------------------------------------------------------- /.github/workflows/dependency-scanning.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/dependency-scanning.yml 2 | name: Dependency Scanning 3 | 4 | on: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main ] 9 | schedule: 10 | - cron: '0 0 * * 0' # Run weekly on Sunday at midnight 11 | 12 | jobs: 13 | dependency-review: 14 | name: Dependency Review 15 | runs-on: ubuntu-latest 16 | if: github.event_name == 'pull_request' 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | 21 | - name: Dependency Review 22 | uses: actions/dependency-review-action@v4 23 | with: 24 | fail-on-severity: high 25 | 26 | govulncheck: 27 | name: Go Vulnerability Check 28 | runs-on: ubuntu-latest 29 | steps: 30 | - name: Checkout code 31 | uses: actions/checkout@v4 32 | 33 | - name: Set up Go 34 | uses: actions/setup-go@v5 35 | with: 36 | go-version: '1.24' 37 | 38 | - name: Install govulncheck 39 | run: go install golang.org/x/vuln/cmd/govulncheck@latest 40 | 41 | - name: Run govulncheck 42 | run: govulncheck ./... 43 | 44 | nancy: 45 | name: Nancy Vulnerability Scan 46 | runs-on: ubuntu-latest 47 | steps: 48 | - name: Checkout code 49 | uses: actions/checkout@v4 50 | 51 | - name: Set up Go 52 | uses: actions/setup-go@v5 53 | with: 54 | go-version: '1.24' 55 | 56 | - name: Install Nancy 57 | run: go install github.com/sonatype-nexus-community/nancy@latest 58 | 59 | - name: Run Nancy 60 | run: go list -json -deps ./... | nancy sleuth 61 | 62 | gosec: 63 | name: GoSec Security Scan 64 | runs-on: ubuntu-latest 65 | steps: 66 | - name: Checkout code 67 | uses: actions/checkout@v4 68 | 69 | - name: Set up Go 70 | uses: actions/setup-go@v5 71 | with: 72 | go-version: '1.24' 73 | 74 | - name: Run Gosec Security Scanner 75 | uses: securego/gosec@master 76 | with: 77 | args: -exclude=G101,G304,G301,G306,G204 -exclude-dir=.history ./... -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | # this is copied over from goreleaser/goreleaser/.github/workflows/lint.yml 2 | name: golangci-lint 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | branches: 8 | - main 9 | pull_request: 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | golangci: 15 | permissions: 16 | contents: read 17 | pull-requests: read 18 | name: lint 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | with: 23 | fetch-depth: 0 24 | - run: git fetch --force --tags 25 | - uses: actions/setup-go@v5 26 | with: 27 | go-version: '>=1.19.5' 28 | cache: true 29 | - name: golangci-lint 30 | uses: golangci/golangci-lint-action@v7 31 | with: 32 | version: v2.0.2 33 | args: --timeout=5m -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: golang-pipeline 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | - run: git fetch --force --tags 17 | - uses: actions/setup-go@v5 18 | with: 19 | go-version: '>=1.19.5' 20 | cache: true 21 | - 22 | name: Run unit tests 23 | run: go test ./... 24 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | 4 | push: 5 | # run only against tags 6 | tags: 7 | - '*' 8 | 9 | permissions: 10 | contents: write 11 | # packages: write 12 | # issues: write 13 | 14 | 15 | jobs: 16 | goreleaser: 17 | runs-on: ubuntu-latest 18 | environment: release 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 23 | - run: git fetch --force --tags 24 | - uses: actions/setup-go@v5 25 | with: 26 | go-version: '>=1.19.5' 27 | cache: true 28 | 29 | - name: Import GPG key 30 | id: import_gpg 31 | uses: crazy-max/ghaction-import-gpg@v6 32 | with: 33 | gpg_private_key: ${{ secrets.GO_GO_GOLEMS_SIGN_KEY }} 34 | passphrase: ${{ secrets.GO_GO_GOLEMS_SIGN_PASSPHRASE }} 35 | fingerprint: "6EBE1DF0BDF48A1BBA381B5B79983EF218C6ED7E" 36 | 37 | - uses: goreleaser/goreleaser-action@v6 38 | with: 39 | distribution: goreleaser 40 | version: latest 41 | args: release --clean 42 | env: 43 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 44 | COSIGN_PWD: ${{ secrets.COSIGN_PWD }} 45 | TAP_GITHUB_TOKEN: ${{ secrets.RELEASE_ACTION_PAT }} 46 | GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} 47 | FURY_TOKEN: ${{ secrets.FURY_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/secret-scanning.yml: -------------------------------------------------------------------------------- 1 | name: Secret Scanning 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | trufflehog: 11 | name: TruffleHog Secret Scan 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: TruffleHog OSS 20 | uses: trufflesecurity/trufflehog@main 21 | with: 22 | path: ./ 23 | base: ${{ github.event.repository.default_branch }} 24 | head: HEAD -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | dist/ 17 | ./oak 18 | 19 | .history 20 | .specstory 21 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | linters: 4 | default: none 5 | enable: 6 | # defaults 7 | - errcheck 8 | - govet 9 | - ineffassign 10 | - staticcheck 11 | - unused 12 | # stuff I'm adding 13 | - exhaustive 14 | # - gochecknoglobals 15 | # - gochecknoinits 16 | - nonamedreturns 17 | - predeclared 18 | exclusions: 19 | paths: 20 | - test-inputs/ 21 | - contexts/ 22 | rules: 23 | - linters: 24 | - staticcheck 25 | text: 'SA1019: cli.CreateProcessorLegacy' 26 | settings: 27 | errcheck: 28 | exclude-functions: 29 | - (io.Closer).Close 30 | - fmt.Fprintf 31 | - fmt.Fprintln 32 | formatters: 33 | enable: 34 | - gofmt 35 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: oak 3 | 4 | before: 5 | hooks: 6 | # You may remove this if you don't use go modules. 7 | - go mod tidy 8 | # you may remove this if you don't need go generate 9 | - go generate ./... 10 | 11 | builds: 12 | # - id: darwin-arm64 13 | # main: ./cmd/oak 14 | # binary: oak 15 | # goos: 16 | # - darwin 17 | # goarch: 18 | # - arm64 19 | # env: 20 | # - PKG_CONFIG_SYSROOT_DIR=/sysroot/macos/arm64 21 | # - PKG_CONFIG_PATH=/sysroot/macos/arm64/usr/local/lib/pkgconfig 22 | # - CC=clang 23 | # - CXX=clang++ 24 | # flags: 25 | # - -mod=readonly 26 | # ldflags: 27 | # - -s -w -X main.version={{.Version}} 28 | - id: darwin-amd64 29 | main: ./cmd/oak 30 | binary: oak 31 | goos: 32 | - darwin 33 | goarch: 34 | - amd64 35 | env: 36 | - PKG_CONFIG_SYSROOT_DIR=/sysroot/macos/amd64 37 | - PKG_CONFIG_PATH=/sysroot/macos/amd64/usr/local/lib/pkgconfig 38 | - CC=o64-clang 39 | - CXX=o64-clang++ 40 | flags: 41 | - -mod=readonly 42 | ldflags: 43 | - -s -w -X main.version={{.Version}} 44 | - id: linux-armhf 45 | main: ./cmd/oak 46 | binary: oak 47 | goos: 48 | - linux 49 | goarch: 50 | - arm 51 | goarm: 52 | - 7 53 | env: 54 | - CC=arm-linux-gnueabihf-gcc 55 | - CXX=arm-linux-gnueabihf-g++ 56 | - CGO_CFLAGS=--sysroot=/sysroot/linux/armhf 57 | - CGO_LDFLAGS=--sysroot=/sysroot/linux/armhf 58 | - PKG_CONFIG_SYSROOT_DIR=/sysroot/linux/armhf 59 | - PKG_CONFIG_PATH=/sysroot/linux/armhf/opt/vc/lib/pkgconfig:/sysroot/linux/armhf/usr/lib/arm-linux-gnueabihf/pkgconfig:/sysroot/linux/armhf/usr/lib/pkgconfig:/sysroot/linux/armhf/usr/local/lib/pkgconfig 60 | flags: 61 | - -mod=readonly 62 | ldflags: 63 | - -s -w -X main.version={{.Version}} 64 | checksum: 65 | name_template: 'checksums.txt' 66 | snapshot: 67 | name_template: "{{ incpatch .Version }}-next" 68 | changelog: 69 | sort: asc 70 | filters: 71 | exclude: 72 | - '^docs:' 73 | - '^test:' 74 | brews: 75 | - name: oak 76 | description: "oak is a tool" 77 | homepage: "https://github.com/go-go-golems/oak" 78 | repository: 79 | owner: go-go-golems 80 | name: homebrew-go-go-go 81 | token: "{{ .Env.TAP_GITHUB_TOKEN }}" 82 | 83 | nfpms: 84 | - 85 | id: packages 86 | 87 | vendor: GO GO GOLEMS 88 | homepage: https://github.com/go-go-golems/ 89 | maintainer: Manuel Odendahl 90 | 91 | description: |- 92 | oak is a tool to run tree-sitter queries against a codebase and template format the results. 93 | 94 | license: MIT 95 | 96 | # Formats to be generated. 97 | formats: 98 | # - apk 99 | - deb 100 | - rpm 101 | 102 | # Version Release. 103 | release: 1 104 | 105 | # Section. 106 | section: default 107 | 108 | # Priority. 109 | priority: extra 110 | 111 | # Custom configuration applied only to the Deb packager. 112 | deb: 113 | # Lintian overrides 114 | lintian_overrides: 115 | - statically-linked-binary 116 | - changelog-file-missing-in-native-package 117 | 118 | publishers: 119 | - name: fury.io 120 | # by specifying `packages` id here goreleaser will only use this publisher 121 | # with artifacts identified by this id 122 | ids: 123 | - packages 124 | dir: "{{ dir .ArtifactPath }}" 125 | cmd: curl -F package=@{{ .ArtifactName }} https://{{ .Env.FURY_TOKEN }}@push.fury.io/go-go-golems/ 126 | 127 | # modelines, feel free to remove those if you don't want/use them: 128 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 129 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 130 | -------------------------------------------------------------------------------- /AGENT.md: -------------------------------------------------------------------------------- 1 | # AGENT.md 2 | 3 | ## Project Structure 4 | 5 | - only a single go.mod file at the root 6 | - package name is github.com/go-go-golems/oak at the root 7 | 8 | ## Code Style Guidelines 9 | 10 | - Go: Uses gofmt, go 1.23+, github.com/pkg/errors for error wrapping 11 | - Go: Uses zerolog for logging, cobra for CLI, viper for config 12 | - Go: Follow standard naming (CamelCase for exported, camelCase for unexported) 13 | - Python: PEP 8 formatting, uses logging module for structured logging 14 | - Python: Try/except blocks with specific exceptions and error logging 15 | - Use interfaces to define behavior, prefer structured concurrency 16 | - Pre-commit hooks use lefthook (configured in lefthook.yml) 17 | 18 | 19 | When implementing go interfaces, use the var _ Interface = &Foo{} to make sure the interface is always implemented correctly. 20 | When building web applications, use htmx, bootstrap and the templ templating language. 21 | Always use a context argument when appropriate. 22 | Use cobra for command-line applications. 23 | Use the "defaults" package name, instead of "default" package name, as it's reserved in go. 24 | Use github.com/pkg/errors for wrapping errors. 25 | When starting goroutines, use errgroup. 26 | 27 | 28 | 29 | Use bun, react and rtk-query. Use typescript. 30 | Use bootstrap for styling. 31 | 32 | 33 | 34 | If me or you the LLM agent seem to go down too deep in a debugging/fixing rabbit hole in our conversations, remind me to take a breath and think about the bigger picture instead of hacking away. Say: "I think I'm stuck, let's TOUCH GRASS". IMPORTANT: Don't try to fix errors by yourself more than twice in a row. Then STOP. Don't do anything else. 35 | 36 | 37 | 38 | Run the format_file tool at the end of each response. 39 | % 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Manuel Odendahl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: lint lintmax gosec govulncheck test build goreleaser tag-major tag-minor tag-patch release bump-glazed install 2 | 3 | all: 4 | 5 | VERSION=v0.1.0 6 | 7 | docker-lint: 8 | docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v2.0.2 golangci-lint run -v 9 | 10 | lint: 11 | golangci-lint run -v 12 | 13 | lintmax: 14 | golangci-lint run -v --max-same-issues=100 15 | 16 | gosec: 17 | go install github.com/securego/gosec/v2/cmd/gosec@latest 18 | gosec -exclude=G101,G304,G301,G306,G204 -exclude-dir=.history ./... 19 | 20 | govulncheck: 21 | go install golang.org/x/vuln/cmd/govulncheck@latest 22 | govulncheck ./... 23 | 24 | test: 25 | go test ./... 26 | 27 | build: 28 | go generate ./... 29 | go build ./... 30 | 31 | goreleaser: 32 | goreleaser release --skip=sign --snapshot --clean 33 | 34 | tag-major: 35 | git tag $(shell svu major) 36 | 37 | tag-minor: 38 | git tag $(shell svu minor) 39 | 40 | tag-patch: 41 | git tag $(shell svu patch) 42 | 43 | release: 44 | git push --tags 45 | GOPROXY=proxy.golang.org go list -m github.com/go-go-golems/oak@$(shell svu current) 46 | 47 | bump-glazed: 48 | go get github.com/go-go-golems/glazed@latest 49 | go get github.com/go-go-golems/clay@latest 50 | go mod tidy 51 | 52 | OAK_BINARY=$(shell which oak) 53 | install: 54 | go build -o ./dist/oak ./cmd/oak && \ 55 | cp ./dist/oak $(OAK_BINARY) 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GO GO OAK 2 | 3 | ![image](https://user-images.githubusercontent.com/128441/233886270-13d0cdd7-ca86-4bea-982a-ffee978b2dd7.png) 4 | 5 | 6 | --- 7 | 8 | Use tree-sitter to run queries against programming language files. 9 | 10 | ``` 11 | ____ ___ ____ ___ ____ ___ _ _____ __ __ ____ 12 | / ___| / _ \ / ___| / _ \ / ___| / _ \ | | | ____|| \/ |/ ___| 13 | | | _ | | | | | | _ | | | | | | _ | | | || | | _| | |\/| |\___ \ 14 | | |_| || |_| | | |_| || |_| | | |_| || |_| || |___ | |___ | | | | ___) | 15 | \____| \___/ \____| \___/ \____| \___/ |_____||_____||_| |_||____/ 16 | 17 | _ _ ____ _____ ___ _ _ __ _____ ___ 18 | | | | |/ ___| | ____| / _ \ / \ | |/ / |_ _|/ _ \ 19 | | | | |\___ \ | _| | | | | / _ \ | ' / | | | | | | 20 | | |_| | ___) || |___ | |_| |/ ___ \ | . \ | | | |_| | 21 | \___/ |____/ |_____| \___//_/ \_\|_|\_\ |_| \___/ 22 | 23 | ____ ____ ___ _ _ ____ ___ ____ ____ _____ ____ 24 | | __ ) | _ \ |_ _|| \ | | / ___| / _ \ | _ \ | _ \ | ____|| _ \ 25 | | _ \ | |_) | | | | \| || | _ | | | || |_) || | | || _| | |_) | 26 | | |_) || _ < | | | |\ || |_| | | |_| || _ < | |_| || |___ | _ < 27 | |____/ |_| \_\|___||_| \_| \____| \___/ |_| \_\|____/ |_____||_| \_\ 28 | 29 | _____ ___ ____ _ _ _ ___ ____ 30 | |_ _|/ _ \ / ___|| | | | / \ / _ \ / ___| 31 | | | | | | | | | | |_| | / _ \ | | | |\___ \ 32 | | | | |_| | | |___ | _ | / ___ \| |_| | ___) |_ 33 | |_| \___/ \____||_| |_|/_/ \_\\___/ |____/(_) 34 | 35 | ``` 36 | 37 | ## Overview 38 | 39 | Oak allows the user to provide [tree-sitter](https://tree-sitter.github.io/tree-sitter/) queries 40 | in a YAML file and use the resulting captures to expand a go template. 41 | 42 | ## Background 43 | 44 | When prompting LLMs for programming, it is very useful to provide some context about 45 | the code that you want to generate, for example out of your current codebase. 46 | 47 | Just copy pasting code gets you really far, but it eats a lot of tokens, and often 48 | confuses the LLM. Minimal prompts are often much more effective (see [Exploring coding with LLMs](https://share.descript.com/view/CDetEUb5doZ)). 49 | In order to quickly generate minimal prompts out of an existing codebase, we can use `oak`, 50 | which follows the pattern of tools like [glaze](https://github.com/go-go-golems/glazed), 51 | [sqleton](https://github.com/go-go-golems/sqleton) or [pinocchio](https://github.com/go-go-golems/geppetto) 52 | which allow the user to declare "commands" in a YAML file. 53 | 54 | ## Getting started 55 | 56 | After installing oak by downloading the proper release binary, you can start creating oak commands 57 | by storing them in your command repository (a directory containing all your oak queries). 58 | 59 | To get started, use `oak help create-query` to get an introduction to creating a new verb. 60 | 61 | You can also run [example1](./cmd/oak/queries/example1.yaml) against [a go file](./test-inputs/test.go) 62 | to get a list of imports and function declarations. 63 | 64 | ``` 65 | ❯ oak example1 ./test-inputs/test.go 66 | File: ./test-inputs/test.go 67 | 68 | Function Declarations: 69 | - foo(s string) 70 | - main() 71 | - someFunction() 72 | - printString(s string) 73 | 74 | Import Statements: 75 | - path: "fmt" 76 | ``` 77 | 78 | Commands can be run against whole directories or with explicit globs. For more information, use `oak help glob`. 79 | 80 | ``` 81 | ❯ oak example1 --glob **/queries.go . ./test-inputs/test.go 82 | File: ./test-inputs/test.go 83 | 84 | Function Declarations: 85 | ... 86 | 87 | File: pkg/queries.go 88 | 89 | Function Declarations: 90 | ... 91 | ``` 92 | 93 | ## Rendering the query templates 94 | 95 | Queries are themselves go templates that will get expanded based on the command-line flags 96 | defined in the oak YAML file. 97 | 98 | You can print the queries to get a preview of what they would look like rendered. 99 | 100 | For example, for the following YAML: 101 | 102 | ```yaml 103 | name: equals 104 | short: Find expressions where the right side equals a number. 105 | flags: 106 | - name: number 107 | type: int 108 | help: The number to compare against 109 | default: 1 110 | language: go 111 | 112 | queries: 113 | - name: testPredicate 114 | query: | 115 | (binary_expression 116 | left: (_) @left 117 | right: (_) @right 118 | (#eq? @right {{ .number }})) 119 | 120 | template: | 121 | {{ range .testPredicate.Matches }} 122 | - {{ .left.Text }} - {{.right.Text}}{{ end }} 123 | 124 | ``` 125 | 126 | We can either use the default value: 127 | 128 | ``` 129 | ❯ oak equals --print-queries 130 | - name: testPredicate 131 | query: | 132 | (binary_expression 133 | left: (_) @left 134 | right: (_) @right 135 | (#eq? @right 1)) 136 | ``` 137 | 138 | or provide a custom number: 139 | 140 | ``` 141 | ❯ oak equals --print-queries --number 23 142 | - name: testPredicate 143 | query: | 144 | (binary_expression 145 | left: (_) @left 146 | right: (_) @right 147 | (#eq? @right 23)) 148 | ``` 149 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | - [ ] Use the filefilter from clay to filter files -------------------------------------------------------------------------------- /cmd/experiments/function_finder/function_finder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-go-golems/oak/a488f21ac94c1fe2fc7fc1490c06b75c88bc3c7a/cmd/experiments/function_finder/function_finder -------------------------------------------------------------------------------- /cmd/experiments/function_finder/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/go-go-golems/oak/pkg/api" 9 | ) 10 | 11 | // Function represents a function found in the source code 12 | type Function struct { 13 | Name string 14 | Parameters string 15 | IsExported bool 16 | SourceFile string 17 | LineNumber int 18 | } 19 | 20 | func main() { 21 | if len(os.Args) < 2 { 22 | fmt.Println("Usage: function_finder ") 23 | os.Exit(1) 24 | } 25 | 26 | filePath := os.Args[1] 27 | 28 | // Create a query builder for Go code 29 | query := api.NewQueryBuilder( 30 | api.WithLanguage("go"), 31 | api.WithQuery("functions", ` 32 | (function_declaration 33 | name: (identifier) @functionName 34 | parameters: (parameter_list) @parameters 35 | body: (block)? @body) 36 | `), 37 | api.WithQuery("methods", ` 38 | (method_declaration 39 | receiver: (parameter_list) @receiver 40 | name: (field_identifier) @methodName 41 | parameters: (parameter_list) @parameters 42 | body: (block)? @body) 43 | `), 44 | ) 45 | 46 | // 1. Template-based output 47 | fmt.Println("\n=== Template Output ===") 48 | templateResult, err := query.RunWithTemplate( 49 | context.Background(), 50 | ` 51 | # Functions in {{ .Language }} Files 52 | 53 | {{ range $file, $results := .ResultsByFile }} 54 | ## {{ $file }} 55 | 56 | {{ if index $results "functions" }} 57 | ### Functions 58 | {{ range (index $results "functions").Matches }} 59 | - func {{ (index . "functionName").Text }}{{ (index . "parameters").Text }} 60 | {{ end }} 61 | {{ end }} 62 | 63 | {{ if index $results "methods" }} 64 | ### Methods 65 | {{ range (index $results "methods").Matches }} 66 | - func {{ (index . "receiver").Text }} {{ (index . "methodName").Text }}{{ (index . "parameters").Text }} 67 | {{ end }} 68 | {{ end }} 69 | {{ end }} 70 | `, 71 | api.WithFiles([]string{filePath}), 72 | ) 73 | if err != nil { 74 | fmt.Printf("Error: %s\n", err) 75 | os.Exit(1) 76 | } 77 | 78 | fmt.Println(templateResult) 79 | 80 | // 2. Programmatic processing 81 | fmt.Println("\n=== Programmatic Output ===") 82 | functionsResult, err := query.RunWithProcessor( 83 | context.Background(), 84 | func(results api.QueryResults) (any, error) { 85 | var functions []Function 86 | 87 | for fileName, fileResults := range results { 88 | // Process regular functions 89 | if funcResults, ok := fileResults["functions"]; ok { 90 | for _, match := range funcResults.Matches { 91 | fnName := match["functionName"].Text 92 | params := match["parameters"].Text 93 | functions = append(functions, Function{ 94 | Name: fnName, 95 | Parameters: params, 96 | IsExported: isExported(fnName), 97 | SourceFile: fileName, 98 | LineNumber: int(match["functionName"].StartPoint.Row) + 1, 99 | }) 100 | } 101 | } 102 | 103 | // Process methods 104 | if methodResults, ok := fileResults["methods"]; ok { 105 | for _, match := range methodResults.Matches { 106 | methName := match["methodName"].Text 107 | params := match["parameters"].Text 108 | functions = append(functions, Function{ 109 | Name: methName, 110 | Parameters: params, 111 | IsExported: isExported(methName), 112 | SourceFile: fileName, 113 | LineNumber: int(match["methodName"].StartPoint.Row) + 1, 114 | }) 115 | } 116 | } 117 | } 118 | 119 | return functions, nil 120 | }, 121 | api.WithFiles([]string{filePath}), 122 | ) 123 | if err != nil { 124 | fmt.Printf("Error: %s\n", err) 125 | os.Exit(1) 126 | } 127 | 128 | // Type assertion for the result 129 | functions, ok := functionsResult.([]Function) 130 | if !ok { 131 | fmt.Println("Error: could not convert result to []Function") 132 | os.Exit(1) 133 | } 134 | 135 | // Print statistics 136 | fmt.Printf("Found %d functions/methods:\n", len(functions)) 137 | 138 | // Count exported vs non-exported functions 139 | exportedCount := 0 140 | for _, fn := range functions { 141 | if fn.IsExported { 142 | exportedCount++ 143 | } 144 | } 145 | 146 | fmt.Printf("- Exported: %d\n", exportedCount) 147 | fmt.Printf("- Unexported: %d\n", len(functions)-exportedCount) 148 | 149 | // Print details 150 | fmt.Println("\nFunction Details:") 151 | for i, fn := range functions { 152 | exportedStr := "unexported" 153 | if fn.IsExported { 154 | exportedStr = "exported" 155 | } 156 | 157 | fmt.Printf("%d. %s (%s) at %s:%d\n", 158 | i+1, 159 | fn.Name+fn.Parameters, 160 | exportedStr, 161 | fn.SourceFile, 162 | fn.LineNumber, 163 | ) 164 | } 165 | } 166 | 167 | // isExported checks if a function/method name is exported (starts with uppercase) 168 | func isExported(name string) bool { 169 | if len(name) == 0 { 170 | return false 171 | } 172 | 173 | firstChar := name[0] 174 | return firstChar >= 'A' && firstChar <= 'Z' 175 | } 176 | -------------------------------------------------------------------------------- /cmd/experiments/run-examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | echo "Building examples..." 8 | cd function_finder && go build 9 | cd ../typescript_analyzer && go build 10 | cd .. 11 | 12 | echo -e "\n============================================" 13 | echo "Running Function Finder on Go example:" 14 | echo -e "============================================\n" 15 | ./function_finder/function_finder ../../test-inputs/go-example.go 16 | 17 | echo -e "\n============================================" 18 | echo "Running TypeScript Analyzer on Component.tsx:" 19 | echo -e "============================================\n" 20 | ./typescript_analyzer/typescript_analyzer ../../test-inputs/typescript/Component.tsx 21 | 22 | echo -e "\n============================================" 23 | echo "Running TypeScript Analyzer on App.tsx:" 24 | echo -e "============================================\n" 25 | ./typescript_analyzer/typescript_analyzer ../../test-inputs/typescript/App.tsx -------------------------------------------------------------------------------- /cmd/experiments/ts-docs/README.md: -------------------------------------------------------------------------------- 1 | # TypeScript/JavaScript API Documentation Generator 2 | 3 | This tool uses the Oak API to automatically generate API reference documentation in Markdown format for TypeScript and JavaScript files. 4 | 5 | ## Features 6 | 7 | - Detects function declarations, arrow functions, and class methods 8 | - Extracts docstrings and parameter information 9 | - Formats JSDoc comments (@param, @returns) as Markdown 10 | - Handles TypeScript type annotations 11 | - Detects exported vs non-exported functions 12 | - Creates a table of contents with links to each function 13 | 14 | ## Usage 15 | 16 | ```bash 17 | # Generate documentation for a single file 18 | go run main.go path/to/file.ts 19 | 20 | # Generate documentation for all TS/JS files in a directory 21 | go run main.go path/to/directory 22 | ``` 23 | 24 | ## Output 25 | 26 | The tool generates Markdown output with the following structure: 27 | 28 | ``` 29 | # API Reference 30 | 31 | ## Table of Contents 32 | 33 | - [File1](#file1) 34 | - [function1](#function1) 35 | - [function2](#function2) 36 | 37 | ## File1 38 | 39 | ### function1 40 | 41 | _Exported_ 42 | 43 | Function description from docstring 44 | 45 | ```typescript 46 | function1(param1: type1, param2: type2): returnType 47 | ``` 48 | 49 | **Parameters:** 50 | 51 | - `param1` - _type1_ 52 | - `param2` - _type2_ 53 | 54 | **Returns:** _returnType_ 55 | 56 | _Defined in [file.ts:10]_ 57 | ``` 58 | 59 | ## Implementation Details 60 | 61 | The tool uses Oak's tree-sitter integration to parse TypeScript/JavaScript code and extract: 62 | 63 | 1. Function declarations 64 | 2. Arrow function expressions (both exported and non-exported) 65 | 3. Class method definitions 66 | 4. Comments associated with functions 67 | 5. Parameter types and return types 68 | 69 | For each function, it generates properly formatted Markdown documentation including: 70 | 71 | - Function signature with parameters and return type 72 | - Parameter list with types 73 | - Return type information 74 | - Export status 75 | - Source location 76 | 77 | ## Dependencies 78 | 79 | - Oak API for tree-sitter parsing 80 | - Cobra for command-line interface 81 | -------------------------------------------------------------------------------- /cmd/experiments/ts-docs/sample-api.md: -------------------------------------------------------------------------------- 1 | # sample API Reference 2 | 3 | ## Table of Contents 4 | 5 | - [.](#) 6 | - [formatUser](#formatuser) 7 | - [add](#add) 8 | - [capitalize](#capitalize) 9 | - [constructor](#constructor) 10 | - [setProperty](#setproperty) 11 | - [getProperty](#getproperty) 12 | 13 | ## . 14 | 15 | ### formatUser 16 | 17 | *Exported* 18 | 19 | Formats a user object into a display string 20 | 21 | ```typescript 22 | formatUser(user: { 23 | name: string; 24 | age?: number; 25 | roles: string[]; 26 | }): string 27 | ``` 28 | 29 | **Parameters:** 30 | 31 | - `user` - *{ 32 | name: string; 33 | age?: number; 34 | roles: string[]; 35 | }* 36 | 37 | **Returns:** *string* 38 | 39 | *Defined in [.:18]* 40 | 41 | ### add 42 | 43 | Adds two numbers together and returns the result. 44 | - **a:** First number to add 45 | - **b:** Second number to add 46 | **Returns:** Sum of the two numbers 47 | 48 | ```typescript 49 | add(a: number, b: number): number 50 | ``` 51 | 52 | **Parameters:** 53 | 54 | - `a` - *number* 55 | - `b` - *number* 56 | 57 | **Returns:** *number* 58 | 59 | *Defined in [.:11]* 60 | 61 | ### capitalize 62 | 63 | A private helper function that is not exported 64 | 65 | ```typescript 66 | capitalize(str: string): string 67 | ``` 68 | 69 | **Parameters:** 70 | 71 | - `str` - *string* 72 | 73 | **Returns:** *string* 74 | 75 | *Defined in [.:29]* 76 | 77 | ### constructor 78 | 79 | Creates a new UserProfile instance 80 | - **username:** The username for this profile 81 | - **email:** Optional email address 82 | 83 | ```typescript 84 | constructor(public username: string, private email?: string) 85 | ``` 86 | 87 | **Parameters:** 88 | 89 | - `public username` - *string* 90 | - `private email?` - *string* 91 | 92 | *Defined in [.:45]* 93 | 94 | ### setProperty 95 | 96 | Sets a property on the user profile 97 | - **key:** Property name 98 | - **value:** Property value 99 | 100 | ```typescript 101 | setProperty(key: string, value: any): void 102 | ``` 103 | 104 | **Parameters:** 105 | 106 | - `key` - *string* 107 | - `value` - *any* 108 | 109 | **Returns:** *void* 110 | 111 | *Defined in [.:52]* 112 | 113 | ### getProperty 114 | 115 | Gets a property from the user profile 116 | - **key:** Property name to retrieve 117 | - **defaultValue:** Value to return if property doesn't exist 118 | 119 | ```typescript 120 | getProperty(key: string, defaultValue?: T): T | undefined 121 | ``` 122 | 123 | **Parameters:** 124 | 125 | - `key` - *string* 126 | - `defaultValue?` - *T* 127 | 128 | **Returns:** *T | undefined* 129 | 130 | *Defined in [.:61]* 131 | 132 | 133 | -------------------------------------------------------------------------------- /cmd/experiments/ts-docs/sample.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A simple utility module that demonstrates TypeScript functionality. 3 | */ 4 | 5 | /** 6 | * Adds two numbers together and returns the result. 7 | * @param a First number to add 8 | * @param b Second number to add 9 | * @returns Sum of the two numbers 10 | */ 11 | export function add(a: number, b: number): number { 12 | return a + b; 13 | } 14 | 15 | /** 16 | * Formats a user object into a display string 17 | */ 18 | export const formatUser = (user: { 19 | name: string; 20 | age?: number; 21 | roles: string[]; 22 | }): string => { 23 | const age = user.age ? ` (${user.age})` : ""; 24 | const roles = user.roles.join(", "); 25 | return `${user.name}${age} - ${roles}`; 26 | }; 27 | 28 | // A private helper function that is not exported 29 | function capitalize(str: string): string { 30 | if (!str) return str; 31 | return str.charAt(0).toUpperCase() + str.slice(1); 32 | } 33 | 34 | /** 35 | * User profile class that manages user data 36 | */ 37 | export class UserProfile { 38 | private data: Record = {}; 39 | 40 | /** 41 | * Creates a new UserProfile instance 42 | * @param username The username for this profile 43 | * @param email Optional email address 44 | */ 45 | constructor(public username: string, private email?: string) {} 46 | 47 | /** 48 | * Sets a property on the user profile 49 | * @param key Property name 50 | * @param value Property value 51 | */ 52 | setProperty(key: string, value: any): void { 53 | this.data[key] = value; 54 | } 55 | 56 | /** 57 | * Gets a property from the user profile 58 | * @param key Property name to retrieve 59 | * @param defaultValue Value to return if property doesn't exist 60 | */ 61 | getProperty(key: string, defaultValue?: T): T | undefined { 62 | return this.data[key] ?? defaultValue; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /cmd/experiments/ts-docs/ts-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script runs the ts-docs tool and writes the output to a markdown file 4 | 5 | if [ $# -lt 1 ]; then 6 | echo "Usage: $0 [output.md]" 7 | echo "If output.md is not specified, it will use the basename of the input file/directory + -api.md" 8 | exit 1 9 | fi 10 | 11 | INPUT="$1" 12 | 13 | # Determine output filename 14 | if [ $# -ge 2 ]; then 15 | OUTPUT="$2" 16 | else 17 | # Default output file based on input name 18 | BASENAME=$(basename "$INPUT") 19 | FILENAME="${BASENAME%.*}" 20 | OUTPUT="${FILENAME}-api.md" 21 | fi 22 | 23 | # Run the tool and save output 24 | echo "Generating API documentation for $INPUT..." 25 | go run main.go "$INPUT" >"$OUTPUT" 26 | 27 | if [ $? -eq 0 ]; then 28 | echo "API documentation written to $OUTPUT" 29 | echo "Word count: $(wc -w <"$OUTPUT") words" 30 | echo "Function count: $(grep -c '^###' "$OUTPUT") functions" 31 | else 32 | echo "Error generating documentation" 33 | exit 1 34 | fi 35 | -------------------------------------------------------------------------------- /cmd/experiments/typescript_analyzer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/go-go-golems/oak/pkg/api" 11 | ) 12 | 13 | // Component represents a React component in TypeScript 14 | type Component struct { 15 | Name string 16 | Type string // 'function', 'arrow', 'class' 17 | Props string 18 | IsExported bool 19 | HasChildren bool 20 | SourceFile string 21 | LineNumber int 22 | } 23 | 24 | func main() { 25 | if len(os.Args) < 2 { 26 | fmt.Println("Usage: typescript_analyzer ") 27 | os.Exit(1) 28 | } 29 | 30 | path := os.Args[1] 31 | 32 | var runOption api.RunOption 33 | fileInfo, err := os.Stat(path) 34 | if err != nil { 35 | fmt.Printf("Error: %s\n", err) 36 | os.Exit(1) 37 | } 38 | 39 | if fileInfo.IsDir() { 40 | // Use glob for directory 41 | runOption = api.WithGlob(filepath.Join(path, "**/*.{ts,tsx}")) 42 | } else { 43 | // Use specific file 44 | runOption = api.WithFiles([]string{path}) 45 | } 46 | 47 | // Create a query builder for TypeScript 48 | query := api.NewQueryBuilder( 49 | api.WithLanguage("tsx"), 50 | api.WithQuery("functionComponents", ` 51 | (function_declaration 52 | name: (identifier) @componentName 53 | parameters: (formal_parameters 54 | (required_parameter 55 | pattern: (object_pattern)? @props)) @parameters 56 | body: (statement_block)? @body) 57 | `), 58 | api.WithQuery("arrowComponents", ` 59 | (export_statement 60 | (lexical_declaration 61 | (variable_declarator 62 | name: (identifier) @componentName 63 | value: (arrow_function 64 | parameters: (formal_parameters 65 | (required_parameter 66 | pattern: (object_pattern)? @props)) @parameters 67 | body: (statement_block)? @body)))) 68 | `), 69 | api.WithQuery("constArrowComponents", ` 70 | (lexical_declaration 71 | (variable_declarator 72 | name: (identifier) @componentName 73 | value: (arrow_function 74 | parameters: (formal_parameters 75 | (required_parameter 76 | pattern: (object_pattern)? @props)) @parameters 77 | body: (statement_block)? @body))) 78 | `), 79 | ) 80 | 81 | // Template-based output 82 | fmt.Println("=== React Components Analysis ===") 83 | templateResult, err := query.RunWithTemplate( 84 | context.Background(), 85 | ` 86 | # React Components in TypeScript Files 87 | 88 | {{ range $file, $results := .ResultsByFile }} 89 | ## {{ $file }} 90 | 91 | {{ if index $results "functionComponents" }} 92 | ### Function Components 93 | {{ range (index $results "functionComponents").Matches }} 94 | - function {{ (index . "componentName").Text }}{{ (index . "parameters").Text }} 95 | {{ end }} 96 | {{ end }} 97 | 98 | {{ if index $results "arrowComponents" }} 99 | ### Exported Arrow Components 100 | {{ range (index $results "arrowComponents").Matches }} 101 | - export const {{ (index . "componentName").Text }} = {{ (index . "parameters").Text }} => {...} 102 | {{ end }} 103 | {{ end }} 104 | 105 | {{ if index $results "constArrowComponents" }} 106 | ### Const Arrow Components 107 | {{ range (index $results "constArrowComponents").Matches }} 108 | - const {{ (index . "componentName").Text }} = {{ (index . "parameters").Text }} => {...} 109 | {{ end }} 110 | {{ end }} 111 | {{ end }} 112 | `, 113 | runOption, 114 | ) 115 | if err != nil { 116 | fmt.Printf("Error: %s\n", err) 117 | os.Exit(1) 118 | } 119 | 120 | fmt.Println(templateResult) 121 | 122 | // Programmatic processing 123 | fmt.Println("\n=== Component Statistics ===") 124 | componentsResult, err := query.RunWithProcessor( 125 | context.Background(), 126 | func(results api.QueryResults) (any, error) { 127 | var components []Component 128 | 129 | for fileName, fileResults := range results { 130 | // Process function components 131 | if funcResults, ok := fileResults["functionComponents"]; ok { 132 | for _, match := range funcResults.Matches { 133 | compName := match["componentName"].Text 134 | props := match["props"].Text 135 | components = append(components, Component{ 136 | Name: compName, 137 | Type: "function", 138 | Props: props, 139 | IsExported: isExported(compName), 140 | HasChildren: hasChildren(props), 141 | SourceFile: fileName, 142 | LineNumber: int(match["componentName"].StartPoint.Row) + 1, 143 | }) 144 | } 145 | } 146 | 147 | // Process exported arrow components 148 | if arrowResults, ok := fileResults["arrowComponents"]; ok { 149 | for _, match := range arrowResults.Matches { 150 | compName := match["componentName"].Text 151 | props := match["props"].Text 152 | components = append(components, Component{ 153 | Name: compName, 154 | Type: "arrow", 155 | Props: props, 156 | IsExported: true, // These are always exported 157 | HasChildren: hasChildren(props), 158 | SourceFile: fileName, 159 | LineNumber: int(match["componentName"].StartPoint.Row) + 1, 160 | }) 161 | } 162 | } 163 | 164 | // Process const arrow components 165 | if constResults, ok := fileResults["constArrowComponents"]; ok { 166 | for _, match := range constResults.Matches { 167 | compName := match["componentName"].Text 168 | props := match["props"].Text 169 | components = append(components, Component{ 170 | Name: compName, 171 | Type: "arrow", 172 | Props: props, 173 | IsExported: false, // These are not exported 174 | HasChildren: hasChildren(props), 175 | SourceFile: fileName, 176 | LineNumber: int(match["componentName"].StartPoint.Row) + 1, 177 | }) 178 | } 179 | } 180 | } 181 | 182 | return components, nil 183 | }, 184 | runOption, 185 | ) 186 | if err != nil { 187 | fmt.Printf("Error: %s\n", err) 188 | os.Exit(1) 189 | } 190 | 191 | // Type assertion for the result 192 | components, ok := componentsResult.([]Component) 193 | if !ok { 194 | fmt.Println("Error: could not convert result to []Component") 195 | os.Exit(1) 196 | } 197 | 198 | // Print statistics 199 | fmt.Printf("Found %d React components:\n", len(components)) 200 | 201 | // Count by type 202 | funcCount := 0 203 | arrowCount := 0 204 | exportedCount := 0 205 | childrenCount := 0 206 | 207 | for _, comp := range components { 208 | switch comp.Type { 209 | case "function": 210 | funcCount++ 211 | case "arrow": 212 | arrowCount++ 213 | } 214 | 215 | if comp.IsExported { 216 | exportedCount++ 217 | } 218 | 219 | if comp.HasChildren { 220 | childrenCount++ 221 | } 222 | } 223 | 224 | fmt.Printf("- Function components: %d\n", funcCount) 225 | fmt.Printf("- Arrow function components: %d\n", arrowCount) 226 | fmt.Printf("- Exported components: %d\n", exportedCount) 227 | fmt.Printf("- Components with children: %d\n", childrenCount) 228 | 229 | // Print details of components with children 230 | if childrenCount > 0 { 231 | fmt.Println("\nComponents that accept children:") 232 | for _, comp := range components { 233 | if comp.HasChildren { 234 | exportedStr := "" 235 | if comp.IsExported { 236 | exportedStr = "exported " 237 | } 238 | 239 | fmt.Printf("- %s (%s%s component) at %s:%d\n", 240 | comp.Name, 241 | exportedStr, 242 | comp.Type, 243 | comp.SourceFile, 244 | comp.LineNumber, 245 | ) 246 | } 247 | } 248 | } 249 | } 250 | 251 | // isExported checks if a component name is exported (starts with uppercase) 252 | func isExported(name string) bool { 253 | if len(name) == 0 { 254 | return false 255 | } 256 | 257 | firstChar := name[0] 258 | return firstChar >= 'A' && firstChar <= 'Z' 259 | } 260 | 261 | // hasChildren checks if props include children 262 | func hasChildren(props string) bool { 263 | return strings.Contains(props, "children") || strings.Contains(props, "props") 264 | } 265 | -------------------------------------------------------------------------------- /cmd/experiments/typescript_analyzer/typescript_analyzer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-go-golems/oak/a488f21ac94c1fe2fc7fc1490c06b75c88bc3c7a/cmd/experiments/typescript_analyzer/typescript_analyzer -------------------------------------------------------------------------------- /cmd/oak/commands/legacy.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" 9 | "github.com/go-go-golems/oak/pkg" 10 | cmds2 "github.com/go-go-golems/oak/pkg/cmds" 11 | tree_sitter "github.com/go-go-golems/oak/pkg/tree-sitter" 12 | sitter "github.com/smacker/go-tree-sitter" 13 | "github.com/spf13/cobra" 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | func RegisterLegacyCommands(rootCmd *cobra.Command) { 18 | var queryFile string 19 | var templateFile string 20 | 21 | queryCmd := &cobra.Command{ 22 | Use: "query", 23 | Short: "SitterQuery a source code file with a plain sitter query", 24 | Args: cobra.MinimumNArgs(1), 25 | Run: func(cmd *cobra.Command, args []string) { 26 | query, err := readFileOrStdin(queryFile) 27 | cobra.CheckErr(err) 28 | 29 | language, err := cmd.Flags().GetString("language") 30 | cobra.CheckErr(err) 31 | queryName, err := cmd.Flags().GetString("query-name") 32 | cobra.CheckErr(err) 33 | 34 | for _, inputFile := range args { 35 | var lang *sitter.Language 36 | if language != "" { 37 | lang, err = pkg.LanguageNameToSitterLanguage(language) 38 | cobra.CheckErr(err) 39 | } else { 40 | lang, err = pkg.FileNameToSitterLanguage(inputFile) 41 | cobra.CheckErr(err) 42 | } 43 | 44 | if queryName == "" { 45 | queryName = "main" 46 | } 47 | 48 | description := glazed_cmds.NewCommandDescription("query") 49 | 50 | oak := cmds2.NewOakWriterCommand(description, 51 | cmds2.WithQueries(tree_sitter.SitterQuery{ 52 | Name: queryName, 53 | Query: string(query), 54 | }), 55 | cmds2.WithSitterLanguage(lang), 56 | cmds2.WithTemplate(templateFile)) 57 | 58 | sourceCode, err := readFileOrStdin(inputFile) 59 | cobra.CheckErr(err) 60 | 61 | ctx := context.Background() 62 | tree, err := oak.Parse(ctx, nil, sourceCode) 63 | cobra.CheckErr(err) 64 | 65 | if lang == nil { 66 | lang, err = oak.GetLanguage() 67 | cobra.CheckErr(err) 68 | } 69 | 70 | results, err := tree_sitter.ExecuteQueries(lang, tree.RootNode(), oak.Queries, sourceCode) 71 | cobra.CheckErr(err) 72 | 73 | // render template if provided 74 | if templateFile != "" { 75 | s, err := oak.RenderWithTemplateFile(results, templateFile) 76 | println(s) 77 | cobra.CheckErr(err) 78 | } else { 79 | matches := []map[string]string{} 80 | for _, result := range results { 81 | for _, match := range result.Matches { 82 | match_ := map[string]string{} 83 | for k, v := range match { 84 | // this really should be glazed output 85 | match_[k] = fmt.Sprintf("%s (%s)", v.Text, v.Type) 86 | } 87 | matches = append(matches, match_) 88 | } 89 | } 90 | err = yaml.NewEncoder(os.Stdout).Encode(matches) 91 | cobra.CheckErr(err) 92 | } 93 | } 94 | }, 95 | } 96 | queryCmd.Flags().StringVarP(&queryFile, "query-file", "q", "", "SitterQuery file path") 97 | err := queryCmd.MarkFlagRequired("query-file") 98 | cobra.CheckErr(err) 99 | 100 | queryCmd.Flags().String("query-name", "", "SitterQuery name") 101 | queryCmd.Flags().String("language", "", "Language name") 102 | 103 | queryCmd.Flags().StringVarP(&templateFile, "template", "t", "", "Template file path") 104 | 105 | parseCmd := &cobra.Command{ 106 | Use: "parse", 107 | Short: "Parse a source code file", 108 | Run: func(cmd *cobra.Command, args []string) { 109 | language, err := cmd.Flags().GetString("language") 110 | cobra.CheckErr(err) 111 | 112 | // Get dump format options 113 | dumpFormat, err := cmd.Flags().GetString("dump-format") 114 | cobra.CheckErr(err) 115 | showBytes, err := cmd.Flags().GetBool("show-bytes") 116 | cobra.CheckErr(err) 117 | showContent, err := cmd.Flags().GetBool("show-content") 118 | cobra.CheckErr(err) 119 | showAttributes, err := cmd.Flags().GetBool("show-attributes") 120 | cobra.CheckErr(err) 121 | skipWhitespace, err := cmd.Flags().GetBool("skip-whitespace") 122 | cobra.CheckErr(err) 123 | 124 | for _, inputFile := range args { 125 | var lang *sitter.Language 126 | if language != "" { 127 | lang, err = pkg.LanguageNameToSitterLanguage(language) 128 | cobra.CheckErr(err) 129 | } else { 130 | lang, err = pkg.FileNameToSitterLanguage(inputFile) 131 | cobra.CheckErr(err) 132 | } 133 | 134 | description := glazed_cmds.NewCommandDescription("parse") 135 | 136 | oak := cmds2.NewOakWriterCommand( 137 | description, 138 | cmds2.WithSitterLanguage(lang), 139 | cmds2.WithTemplate(templateFile)) 140 | 141 | sourceCode, err := readFileOrStdin(inputFile) 142 | cobra.CheckErr(err) 143 | 144 | ctx := context.Background() 145 | tree, err := oak.Parse(ctx, nil, sourceCode) 146 | cobra.CheckErr(err) 147 | 148 | // Use the enhanced dumping with custom format if specified 149 | if dumpFormat != "" { 150 | var format tree_sitter.DumpFormat 151 | switch dumpFormat { 152 | case "text": 153 | format = tree_sitter.FormatText 154 | case "xml": 155 | format = tree_sitter.FormatXML 156 | case "json": 157 | format = tree_sitter.FormatJSON 158 | case "yaml": 159 | format = tree_sitter.FormatYAML 160 | default: 161 | format = tree_sitter.FormatText 162 | } 163 | 164 | options := tree_sitter.DumpOptions{ 165 | ShowBytes: showBytes, 166 | ShowContent: showContent, 167 | ShowAttributes: showAttributes, 168 | SkipWhitespace: skipWhitespace, 169 | } 170 | 171 | err = oak.DumpTreeToWriter(tree, sourceCode, os.Stdout, format, options) 172 | cobra.CheckErr(err) 173 | } else { 174 | // Use the original DumpTree for backward compatibility 175 | oak.DumpTree(tree) 176 | } 177 | } 178 | }, 179 | } 180 | 181 | parseCmd.Flags().String("language", "", "Language name") 182 | // Add dump format flags 183 | parseCmd.Flags().String("dump-format", "", "Output format for the tree dump (text, xml, json, yaml)") 184 | parseCmd.Flags().Bool("show-bytes", false, "Show byte offsets in the tree dump") 185 | parseCmd.Flags().Bool("show-content", true, "Show node content in the tree dump") 186 | parseCmd.Flags().Bool("show-attributes", true, "Show node attributes in the tree dump") 187 | parseCmd.Flags().Bool("skip-whitespace", true, "Skip whitespace-only nodes in the tree dump") 188 | 189 | rootCmd.AddCommand(parseCmd) 190 | rootCmd.AddCommand(queryCmd) 191 | } 192 | -------------------------------------------------------------------------------- /cmd/oak/commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "embed" 5 | "fmt" 6 | "os" 7 | 8 | clay "github.com/go-go-golems/clay/pkg" 9 | clay_commandmeta "github.com/go-go-golems/clay/pkg/cmds/commandmeta" 10 | clay_repositories "github.com/go-go-golems/clay/pkg/cmds/repositories" 11 | "github.com/go-go-golems/clay/pkg/repositories" 12 | "github.com/go-go-golems/glazed/pkg/cli" 13 | glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" 14 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 15 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 16 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 17 | "github.com/go-go-golems/glazed/pkg/help" 18 | "github.com/go-go-golems/glazed/pkg/types" 19 | cmds2 "github.com/go-go-golems/oak/pkg/cmds" 20 | "github.com/spf13/cobra" 21 | "github.com/spf13/viper" 22 | ) 23 | 24 | var RootCmd = &cobra.Command{ 25 | Use: "oak", 26 | Short: "oak runs tree-sitter queries against your source code", 27 | } 28 | 29 | func InitRootCmd(docFS embed.FS) (*help.HelpSystem, error) { 30 | helpSystem := help.NewHelpSystem() 31 | err := helpSystem.LoadSectionsFromFS(docFS, ".") 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | helpSystem.SetupCobraRootCommand(RootCmd) 37 | 38 | err = clay.InitViper("oak", RootCmd) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | RootCmd.AddCommand(RunCommandCmd) 44 | return helpSystem, nil 45 | } 46 | 47 | func InitAllCommands(helpSystem *help.HelpSystem, queriesFS embed.FS) error { 48 | repositoryPaths := viper.GetStringSlice("repositories") 49 | 50 | defaultDirectory := "$HOME/.oak/queries" 51 | _, err := os.Stat(os.ExpandEnv(defaultDirectory)) 52 | if err == nil { 53 | repositoryPaths = append(repositoryPaths, os.ExpandEnv(defaultDirectory)) 54 | } 55 | 56 | loader := &cmds2.OakCommandLoader{} 57 | repositories_ := createRepositories(repositoryPaths, loader, queriesFS) 58 | 59 | allCommands, err := repositories.LoadRepositories( 60 | helpSystem, 61 | RootCmd, 62 | repositories_, 63 | cli.WithCobraShortHelpLayers(layers.DefaultSlug, cmds2.OakSlug), 64 | ) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | glazeCmd := &cobra.Command{ 70 | Use: "glaze", 71 | Short: "Run commands and output results as structured data", 72 | } 73 | RootCmd.AddCommand(glazeCmd) 74 | 75 | oakGlazedLoader := &cmds2.OakGlazedCommandLoader{} 76 | repositories_ = createRepositories(repositoryPaths, oakGlazedLoader, queriesFS) 77 | 78 | _, err = repositories.LoadRepositories( 79 | helpSystem, 80 | glazeCmd, 81 | repositories_, 82 | cli.WithCobraShortHelpLayers(layers.DefaultSlug, cmds2.OakSlug), 83 | ) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | // Create and add the unified command management group 89 | commandManagementCmd, err := clay_commandmeta.NewCommandManagementCommandGroup( 90 | allCommands, 91 | clay_commandmeta.WithListAddCommandToRowFunc(func( 92 | command glazed_cmds.Command, 93 | row types.Row, 94 | parsedLayers *layers.ParsedLayers, 95 | ) ([]types.Row, error) { 96 | ret := []types.Row{row} 97 | switch c := command.(type) { 98 | case *cmds2.OakCommand: 99 | row.Set("language", c.Language) 100 | row.Set("queries", c.Queries) 101 | row.Set("type", "oak") 102 | case *cmds2.OakWriterCommand: 103 | row.Set("language", c.Language) 104 | row.Set("type", "oak-writer") 105 | case *alias.CommandAlias: 106 | row.Set("type", "alias") 107 | row.Set("aliasFor", c.AliasFor) 108 | default: 109 | if _, ok := row.Get("type"); !ok { 110 | row.Set("type", "unknown") 111 | } 112 | } 113 | return ret, nil 114 | }), 115 | ) 116 | if err != nil { 117 | return fmt.Errorf("failed to initialize command management commands: %w", err) 118 | } 119 | RootCmd.AddCommand(commandManagementCmd) 120 | 121 | // Create and add the repositories command group 122 | RootCmd.AddCommand(clay_repositories.NewRepositoriesGroupCommand()) 123 | 124 | return nil 125 | } 126 | 127 | func createRepositories(repositoryPaths []string, loader loaders.CommandLoader, queriesFS embed.FS) []*repositories.Repository { 128 | directories := []repositories.Directory{ 129 | { 130 | FS: queriesFS, 131 | RootDirectory: "queries", 132 | RootDocDirectory: "queries/doc", 133 | Name: "oak", 134 | SourcePrefix: "embed", 135 | }} 136 | 137 | for _, repositoryPath := range repositoryPaths { 138 | dir := os.ExpandEnv(repositoryPath) 139 | // check if dir exists 140 | if fi, err := os.Stat(dir); os.IsNotExist(err) || !fi.IsDir() { 141 | continue 142 | } 143 | directories = append(directories, repositories.Directory{ 144 | FS: os.DirFS(dir), 145 | RootDirectory: ".", 146 | RootDocDirectory: "doc", 147 | Name: dir, 148 | WatchDirectory: dir, 149 | SourcePrefix: "file", 150 | }) 151 | } 152 | 153 | repositories_ := []*repositories.Repository{ 154 | repositories.NewRepository( 155 | repositories.WithDirectories(directories...), 156 | repositories.WithCommandLoader(loader), 157 | ), 158 | } 159 | return repositories_ 160 | } 161 | -------------------------------------------------------------------------------- /cmd/oak/commands/run.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" 11 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 12 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 13 | cmds2 "github.com/go-go-golems/oak/pkg/cmds" 14 | tree_sitter "github.com/go-go-golems/oak/pkg/tree-sitter" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | var RunCommandCmd = &cobra.Command{ 19 | Use: "run-command", 20 | Short: "Run a command from a file", 21 | Args: cobra.MinimumNArgs(2), 22 | Run: func(cmd *cobra.Command, args []string) { 23 | queryFile, err := filepath.Abs(args[0]) 24 | if err != nil { 25 | cobra.CheckErr(err) 26 | } 27 | 28 | loader := &cmds2.OakCommandLoader{} 29 | fs_, queryFile, err := loaders.FileNameToFsFilePath(queryFile) 30 | if err != nil { 31 | cobra.CheckErr(err) 32 | } 33 | cmds_, err := loader.LoadCommands(fs_, queryFile, []glazed_cmds.CommandDescriptionOption{}, []alias.Option{}) 34 | cobra.CheckErr(err) 35 | if len(cmds_) != 1 { 36 | cobra.CheckErr(fmt.Errorf("expected exactly one command")) 37 | } 38 | oak, ok := cmds_[0].(*cmds2.OakWriterCommand) 39 | if !ok { 40 | cobra.CheckErr(fmt.Errorf("expected OakWriterCommand")) 41 | } 42 | 43 | for _, inputFile := range args[1:] { 44 | sourceCode, err := readFileOrStdin(inputFile) 45 | cobra.CheckErr(err) 46 | 47 | ctx := context.Background() 48 | tree, err := oak.Parse(ctx, nil, sourceCode) 49 | cobra.CheckErr(err) 50 | 51 | lang, err := oak.GetLanguage() 52 | cobra.CheckErr(err) 53 | 54 | results, err := tree_sitter.ExecuteQueries(lang, tree.RootNode(), oak.Queries, sourceCode) 55 | cobra.CheckErr(err) 56 | 57 | s, err := oak.Render(results) 58 | cobra.CheckErr(err) 59 | 60 | fmt.Println(s) 61 | } 62 | }, 63 | } 64 | 65 | func readFileOrStdin(filename string) ([]byte, error) { 66 | if filename == "-" { 67 | return io.ReadAll(os.Stdin) 68 | } 69 | b, err := os.ReadFile(filename) 70 | return b, err 71 | } 72 | -------------------------------------------------------------------------------- /cmd/oak/doc/create-query.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Create a source query with oak 3 | Slug: create-query 4 | Topics: 5 | - oak 6 | - query 7 | IsTemplate: false 8 | IsTopLevel: true 9 | ShowPerDefault: true 10 | SectionType: Tutorial 11 | --- 12 | 13 | ## Creating a query file 14 | 15 | To create a query verb with oak, you need to create a YAML file that 16 | provides a high-level [glaze](https://github.com/go-go-golems/glazed) Command 17 | description consisting of: 18 | 19 | - name (the verb used to run your query) 20 | - short (a short description of the verb) 21 | - long (optional, a long form description of the verb) 22 | - flags (a list of glaze.Command flags description the command-line flags the verb provides) 23 | 24 | Arguments to the verb are a list of source files. 25 | 26 | Furthermore, the YAML file should provide the following oak specific fields: 27 | 28 | - language (the name of the grammar to be used) 29 | - queries (a list of queries that have two fields: name and query) 30 | - template (the template used to render the results) 31 | 32 | This file should be stored in one of the repositories configured for oak (by default, `~/.oak/queries`). 33 | The subdirectory the file is in will be translated to subverbs. For example, a file in `~/.oak/queries/foo/bar/bla.yaml` 34 | will be mapped to the verb `oak foo bar bla`. 35 | 36 | Further "repositories" (directories containing query files) can be added by editing the configuration file 37 | `~/.oak/config.yaml` and adding a `repositories` field (a simple list of locations). 38 | 39 | ## Example query 40 | 41 | Here is a simple query file that prints function declarations and import statements for go files. 42 | 43 | ```yaml 44 | name: example1 45 | short: A simple example to extract go imports and functions from a go file 46 | 47 | flags: 48 | - name: verbose 49 | type: bool 50 | help: Output all results 51 | default: false 52 | 53 | language: go 54 | queries: 55 | - name: functionDeclarations 56 | query: | 57 | (function_declaration 58 | name: (identifier) @name 59 | parameters: (parameter_list) @parameters 60 | body: (block)) 61 | - name: importStatements 62 | query: | 63 | (import_declaration 64 | (import_spec_list [ 65 | (import_spec 66 | (package_identifier) @name 67 | path: (interpreted_string_literal) @path) 68 | (import_spec 69 | path: (interpreted_string_literal) @path) 70 | ])) 71 | 72 | template: | 73 | {{ range $file, $results := .ResultsByFile -}} 74 | File: {{ $file }} 75 | 76 | {{ with $results -}} 77 | Function Declarations: 78 | {{- range .functionDeclarations.Matches }} 79 | - {{ .name.Text }}{{ .parameters.Text }} {{ end }} 80 | 81 | Import Statements: 82 | {{ range .importStatements.Matches -}} 83 | - {{ if .name }}name: {{ .name.Text }}, {{end -}} path: {{ .path.Text }} 84 | {{ end }} 85 | {{ end -}} 86 | {{ end -}} 87 | 88 | {{ if .verbose -}} 89 | Results:{{ range $v := .Results }} 90 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 91 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 92 | {{end}}{{ end }} 93 | {{ end -}} 94 | ``` 95 | 96 | ## Templated queries 97 | 98 | Queries are themselves go templates and will be rendered by passing the flags data from glazed. 99 | 100 | You can print the queries to get a preview of what they would look like rendered. 101 | 102 | For example, for the following YAML: 103 | 104 | ```yaml 105 | name: equals 106 | short: Find expressions where the right side equals a number. 107 | flags: 108 | - name: number 109 | type: int 110 | help: The number to compare against 111 | default: 1 112 | language: go 113 | 114 | queries: 115 | - name: testPredicate 116 | query: | 117 | (binary_expression 118 | left: (_) @left 119 | right: (_) @right 120 | (#eq? @right {{ .number }})) 121 | 122 | template: | 123 | {{ range .testPredicate.Matches }} 124 | - {{ .left.Text }} - {{.right.Text}}{{ end }} 125 | 126 | ``` 127 | 128 | We can either use the default value: 129 | 130 | ``` 131 | ❯ oak equals --print-queries 132 | - name: testPredicate 133 | query: | 134 | (binary_expression 135 | left: (_) @left 136 | right: (_) @right 137 | (#eq? @right 1)) 138 | ``` 139 | 140 | or provide a custom number: 141 | 142 | ``` 143 | ❯ oak equals --print-queries --number 23 144 | - name: testPredicate 145 | query: | 146 | (binary_expression 147 | left: (_) @left 148 | right: (_) @right 149 | (#eq? @right 23)) 150 | ``` 151 | 152 | ## Command execution 153 | 154 | To call the command, run `oak` with the verb path given by the subdirectory structure of the command location 155 | within its repository directory. The verb accepts the flags defined in the YAML file as well as a few additional ones 156 | preconfigured by oak and glaze. To get a full list of accepted flags, run `oak CMD --help`: 157 | 158 | ``` 159 | ❯ oak example1 --help 160 | 161 | example1 - A simple example to extract go imports and functions from a go 162 | file 163 | 164 | For more help, run: oak help example1 165 | 166 | ## Usage: 167 | 168 | oak example1 [flags] 169 | 170 | ## Flags: 171 | 172 | --create-alias Create a CLI alias for the query 173 | --create-cliopatra Print the CLIopatra YAML for the command 174 | --create-command Create a new command for the query, with the 175 | defaults updated 176 | -h, --help help for example1 177 | --verbose Output all results 178 | 179 | ## Global flags: 180 | 181 | --config Path to config file (default ~/.oak/config.yml) 182 | --log-file Log file (default: stderr) 183 | --log-format Log format (json, text) (default "text") 184 | --log-level Log level (debug, info, warn, error, fatal) (default 185 | "info") 186 | --with-caller Log caller 187 | ``` 188 | 189 | oak will then: 190 | - iterate over the given input files 191 | - parse each one using treesitter and the configured language grammar into a new AST 192 | - run each query against that AST 193 | - the query results are stored under the name of the file 194 | - these results are themselves a map where the results of each query are stored under the query name 195 | - the individual fields in the result are the captured values in the treesitter query 196 | - finally, the results are rendered using the configured template 197 | - the results are passed as the "Results" field 198 | - the command line flags are also passed to the result template -------------------------------------------------------------------------------- /cmd/oak/doc/glaze-output.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Output matches as structured data 3 | Slug: glaze-output 4 | Commands: 5 | - oak 6 | - glaze 7 | IsTemplate: false 8 | IsTopLevel: true 9 | ShowPerDefault: true 10 | SectionType: GeneralTopic 11 | --- 12 | 13 | ## Output query results as structured data 14 | 15 | To output query results as raw, structured data that can further be processed 16 | by the middlewares provided by [glazed](https://github.com/go-go-golems/glazed), 17 | use the `glaze` verb followed by the name of your command. 18 | 19 | It will output the query results as an array of objects. 20 | 21 | ``` 22 | ❯ oak glaze example1 test-inputs/test.go 23 | +---------------------+----------------------+------------+----------------------------+--------------+ 24 | | file | query | capture | type | text | 25 | +---------------------+----------------------+------------+----------------------------+--------------+ 26 | | test-inputs/test.go | functionDeclarations | name | identifier | foo | 27 | | test-inputs/test.go | functionDeclarations | parameters | parameter_list | (s string) | 28 | | test-inputs/test.go | functionDeclarations | name | identifier | main | 29 | | test-inputs/test.go | functionDeclarations | parameters | parameter_list | () | 30 | | test-inputs/test.go | functionDeclarations | name | identifier | someFunction | 31 | | test-inputs/test.go | functionDeclarations | parameters | parameter_list | () | 32 | | test-inputs/test.go | functionDeclarations | name | identifier | printString | 33 | | test-inputs/test.go | functionDeclarations | parameters | parameter_list | (s string) | 34 | | test-inputs/test.go | importStatements | path | interpreted_string_literal | "fmt" | 35 | +---------------------+----------------------+------------+----------------------------+--------------+ 36 | ``` 37 | 38 | You can then use all the familiar glaze flags to manipulate the output, for example outputting the results as JSON: 39 | 40 | ``` 41 | ❯ oak glaze example1 test-inputs/test.go --output json --fields type,text,capture 42 | [ 43 | { 44 | "capture": "parameters", 45 | "type": "parameter_list", 46 | "text": "(s string)" 47 | }, 48 | { 49 | "capture": "name", 50 | "type": "identifier", 51 | "text": "foo" 52 | }, 53 | ... 54 | ``` -------------------------------------------------------------------------------- /cmd/oak/doc/glob.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Parsing multiple files recursively 3 | Slug: glob 4 | Topics: 5 | - oak 6 | Commands: 7 | - oak 8 | Flags: 9 | - recurse 10 | - glob 11 | IsTemplate: false 12 | IsTopLevel: true 13 | ShowPerDefault: true 14 | SectionType: GeneralTopic 15 | --- 16 | 17 | ## Parsing multiple files 18 | 19 | Often, it is useful to run an oak command against a directory recursively, or even against a full repository. 20 | You can use the `--recurse` and `--glob` files to do so. 21 | 22 | `--recurse` will find all the files whose ending matches the default line endings for the configured language of the 23 | command. For example `--recurse` on a go command will find all files ending in `.go`. 24 | 25 | ``` 26 | ❯ oak example1 --recurse test-inputs 27 | File: test-inputs/test.go 28 | 29 | Function Declarations: 30 | - foo(s string) 31 | - main() 32 | - someFunction() 33 | - printString(s string) 34 | 35 | Import Statements: 36 | - path: "fmt" 37 | 38 | ``` 39 | 40 | `--glob` will find all files whose name matches the provided glob patterns (multiple patterns can be provided 41 | either by passing the flag multiple times, or providing a comma-separated list). 42 | 43 | ``` 44 | ❯ oak example1 --glob 'test-inputs/*.go,**/pkg/queries.go' . 45 | File: pkg/queries.go 46 | 47 | Function Declarations: 48 | ... 49 | 50 | File: test-inputs/test.go 51 | 52 | Function Declarations: 53 | ... 54 | 55 | 56 | ``` 57 | 58 | You can still pass filenames explicitly amongst the list of directories that will be searched recursively: 59 | 60 | ``` 61 | ❯ go run ./cmd/oak example1 --glob **/queries.go . ./test-inputs/test.go 62 | File: ./test-inputs/test.go 63 | 64 | Function Declarations: 65 | ... 66 | 67 | File: pkg/queries.go 68 | 69 | Function Declarations: 70 | ... 71 | ``` -------------------------------------------------------------------------------- /cmd/oak/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "embed" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/go-go-golems/glazed/pkg/cli" 10 | glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" 11 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 12 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 13 | "github.com/go-go-golems/oak/cmd/oak/commands" 14 | cmds2 "github.com/go-go-golems/oak/pkg/cmds" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | //go:embed doc/* 19 | var docFS embed.FS 20 | 21 | //go:embed queries/* 22 | var queriesFS embed.FS 23 | 24 | func main() { 25 | // first, check if the are "run-command file.yaml", 26 | // because we need to load the file and then run the command itself. 27 | // we need to do this before cobra, because we don't know which flags to load yet 28 | if len(os.Args) >= 3 && os.Args[1] == "run" && os.Args[2] != "--help" { 29 | // load the command 30 | loader := &cmds2.OakCommandLoader{} 31 | 32 | filePath, err := filepath.Abs(os.Args[2]) 33 | if err != nil { 34 | fmt.Printf("Could not get absolute path: %v\n", err) 35 | os.Exit(1) 36 | } 37 | fs_, filePath, err := loaders.FileNameToFsFilePath(filePath) 38 | if err != nil { 39 | fmt.Printf("Could not get absolute path: %v\n", err) 40 | os.Exit(1) 41 | } 42 | cmds, err := loader.LoadCommands( 43 | fs_, filePath, 44 | []glazed_cmds.CommandDescriptionOption{}, []alias.Option{}, 45 | ) 46 | if err != nil { 47 | fmt.Printf("Could not load command: %v\n", err) 48 | os.Exit(1) 49 | } 50 | if len(cmds) != 1 { 51 | fmt.Printf("Expected exactly one command, got %d", len(cmds)) 52 | } 53 | 54 | writerCommand, ok := cmds[0].(glazed_cmds.WriterCommand) 55 | if !ok { 56 | fmt.Printf("Expected GlazeCommand, got %T", cmds[0]) 57 | os.Exit(1) 58 | } 59 | 60 | cobraCommand, err := cli.BuildCobraCommandFromWriterCommand(writerCommand) 61 | if err != nil { 62 | fmt.Printf("Could not build cobra command: %v\n", err) 63 | os.Exit(1) 64 | } 65 | 66 | _, err = commands.InitRootCmd(docFS) 67 | cobra.CheckErr(err) 68 | 69 | commands.RootCmd.AddCommand(cobraCommand) 70 | restArgs := os.Args[3:] 71 | os.Args = append([]string{os.Args[0], cobraCommand.Use}, restArgs...) 72 | } else { 73 | helpSystem, err := commands.InitRootCmd(docFS) 74 | cobra.CheckErr(err) 75 | 76 | err = commands.InitAllCommands(helpSystem, queriesFS) 77 | cobra.CheckErr(err) 78 | } 79 | 80 | commands.RegisterLegacyCommands(commands.RootCmd) 81 | 82 | err := commands.RootCmd.Execute() 83 | cobra.CheckErr(err) 84 | } 85 | -------------------------------------------------------------------------------- /cmd/oak/queries/equals.yaml: -------------------------------------------------------------------------------- 1 | name: equals 2 | short: Find expressions where the right side equals a number. 3 | flags: 4 | - name: number 5 | type: int 6 | help: The number to compare against 7 | default: 1 8 | language: go 9 | 10 | queries: 11 | - name: testPredicate 12 | query: | 13 | (binary_expression 14 | left: (_) @left 15 | right: (_) @right 16 | (#eq? @right {{ .number }})) 17 | 18 | template: | 19 | {{ range .testPredicate.Matches }} 20 | - {{ .left.Text }} - {{.right.Text}}{{ end }} 21 | 22 | -------------------------------------------------------------------------------- /cmd/oak/queries/example1.yaml: -------------------------------------------------------------------------------- 1 | name: example1 2 | short: A simple example to extract go imports and functions from a go file 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | 10 | language: go 11 | queries: 12 | - name: functionDeclarations 13 | query: | 14 | (function_declaration 15 | name: (identifier) @name 16 | parameters: (parameter_list) @parameters 17 | body: (block)) 18 | - name: importStatements 19 | query: | 20 | (import_declaration 21 | (import_spec_list [ 22 | (import_spec 23 | (package_identifier) @name 24 | path: (interpreted_string_literal) @path) 25 | (import_spec 26 | path: (interpreted_string_literal) @path) 27 | ])) 28 | 29 | template: | 30 | {{ range $file, $results := .ResultsByFile -}} 31 | File: {{ $file }} 32 | 33 | {{ with $results -}} 34 | Function Declarations: 35 | {{- range .functionDeclarations.Matches }} 36 | - {{ .name.Text }}{{ .parameters.Text }} {{ end }} 37 | 38 | Import Statements: 39 | {{ range .importStatements.Matches -}} 40 | - {{ if .name }}name: {{ .name.Text }}, {{end -}} path: {{ .path.Text }} 41 | {{ end }} 42 | {{ end -}} 43 | {{ end -}} 44 | 45 | {{ if .verbose -}} 46 | Results:{{ range $v := .Results }} 47 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 48 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 49 | {{end}}{{ end }} 50 | {{ end -}} -------------------------------------------------------------------------------- /cmd/oak/queries/go/comments.yaml: -------------------------------------------------------------------------------- 1 | name: comments 2 | short: Extract all commands in a file 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | 10 | language: go 11 | queries: 12 | - name: comments 13 | query: | 14 | ((comment)* @body) 15 | 16 | 17 | template: | 18 | {{ range $file, $results := .ResultsByFile -}} 19 | File: {{ $file }} 20 | 21 | {{ with $results -}} 22 | {{- range .comments.Matches -}} 23 | --- 24 | {{ .body.Text | indentBlock 2}}{{ end -}}{{- end -}}{{- end -}} 25 | 26 | {{ if .verbose -}} 27 | Results:{{ range $v := .Results }} 28 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 29 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 30 | {{end}}{{ end }} 31 | {{ end -}} 32 | -------------------------------------------------------------------------------- /cmd/oak/queries/go/consts.yaml: -------------------------------------------------------------------------------- 1 | name: consts 2 | short: An example to extract go constant specifications 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: type 10 | type: string 11 | help: Only output constant types matching type 12 | - name: only_public 13 | type: bool 14 | help: When true, only output public constants 15 | default: false 16 | - name: name 17 | type: string 18 | help: Only output constants matching name 19 | 20 | language: go 21 | queries: 22 | - name: constSpecs 23 | query: | 24 | ( 25 | (comment)* @comment . 26 | (const_spec 27 | name: (identifier) @constName 28 | type: (type_identifier) @constType 29 | value: (expression_list 30 | (interpreted_string_literal) @constValue)) 31 | {{ if .type }}(#eq? @constType "{{.type}}"){{end}} 32 | {{ if .name }}(#eq? @constName "{{.name}}"){{end}} 33 | {{ if .only_public }}(#match? @constName "^[A-Z]"){{end}} 34 | ) 35 | 36 | template: | 37 | {{ range $file, $results := .ResultsByFile -}} 38 | File: {{ $file }} 39 | {{ with $results -}} 40 | {{- range .constSpecs.Matches }} 41 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 42 | const {{.constName.Text}} {{ .constType.Text }} = {{ .constValue.Text }}{{ end }} 43 | {{ end -}} 44 | {{ end -}} 45 | 46 | {{ if .verbose -}} 47 | Results:{{ range $v := .Results }} 48 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 49 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 50 | {{end}}{{ end }} 51 | {{ end -}} -------------------------------------------------------------------------------- /cmd/oak/queries/go/definitions.yaml: -------------------------------------------------------------------------------- 1 | name: definitions 2 | short: A simple example to extract go functions and methods, structs and interfaces 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: name 10 | type: string 11 | help: Only output methods with a receiver or types matching name 12 | - name: function_name 13 | type: string 14 | help: Only output methods and functions matching name 15 | - name: definition_type 16 | type: stringList 17 | help: Only output definitions matching types (struct, interface, function, method, alias) 18 | - name: with_body 19 | type: bool 20 | help: When true, output the function body, else just output the function declaration and parameters 21 | default: false 22 | - name: only_public 23 | type: bool 24 | help: When true, only output public functions and methods 25 | default: false 26 | 27 | language: go 28 | queries: 29 | - name: typeAliasDeclarations 30 | query: | 31 | {{ if (or (not .definition_type) (has "alias" .definition_type)) }} 32 | ( 33 | (comment)* @comment . 34 | (type_declaration 35 | (type_spec 36 | name: (type_identifier) @typeName 37 | type: (type_identifier) @typeAlias)) 38 | {{ if .name }}(#eq? @typeName "{{.name}}"){{end}} 39 | {{ if .only_public }}(#match? @typeName "^[A-Z]"){{end}} 40 | ) 41 | {{ end }} 42 | 43 | - name: structDeclarations 44 | query: | 45 | {{ if (or (not .definition_type) (has "struct" .definition_type)) }} 46 | ( 47 | (comment)* @comment . 48 | (type_declaration 49 | (type_spec 50 | name: (type_identifier) @structName 51 | type: (struct_type) @structBody)) 52 | {{ if .name }}(#eq? @structName "{{.name}}"){{end}} 53 | {{ if .only_public }}(#match? @structName "^[A-Z]"){{end}} 54 | ) 55 | {{ end }} 56 | 57 | - name: interfaceDeclarations 58 | query: | 59 | {{ if (or (not .definition_type) (has "interface" .definition_type)) }} 60 | ((comment)* @comment . 61 | (type_declaration 62 | (type_spec 63 | name: (type_identifier) @interfaceName 64 | type: (interface_type) @interfaceBody)) 65 | {{ if .name }}(#eq? @interfaceName "{{.name}}"){{end}} 66 | {{ if .only_public }}(#match? @interfaceName "^[A-Z]"){{end}} 67 | ) 68 | {{ end }} 69 | 70 | - name: functionDeclarations 71 | query: | 72 | {{ if (or (not .definition_type) (has "function" .definition_type)) }} 73 | ((comment)* @comment . 74 | (function_declaration 75 | name: (identifier) @name 76 | parameters: (parameter_list)? @parameters 77 | result: (_)? @result 78 | body: (block) @body) 79 | {{ if .name }}(#eq? @name "{{.name}}"){{end}} 80 | {{ if .function_name }}(#eq? @name "{{.function_name}}"){{end}} 81 | {{ if .only_public }}(#match? @name "^[A-Z]"){{end}} 82 | ) 83 | {{end}} 84 | 85 | - name: methodDeclarations 86 | query: | 87 | {{ if (or (not .definition_type) (has "method" .definition_type)) }} 88 | ((comment)* @comment . 89 | (method_declaration 90 | receiver: (parameter_list 91 | [ 92 | (parameter_declaration 93 | name: (identifier) @receiverName 94 | type: (pointer_type (type_identifier) @typeName)) 95 | (parameter_declaration 96 | name: (identifier) @receiverName 97 | type: (type_identifier) @typeName) 98 | (parameter_declaration 99 | name: (identifier) @receiverName 100 | type: (pointer_type (generic_type type: (type_identifier) @typeName))) 101 | (parameter_declaration 102 | name: (identifier) @receiverName 103 | type: (generic_type type: (type_identifier) @typeName)) 104 | ]) @receiver 105 | 106 | name: (field_identifier) @name 107 | parameters: (parameter_list)? @parameters 108 | result: (_)? @result 109 | body: (block) @body) 110 | {{ if .name }}(#eq? @typeName "{{.name}}"){{end}} 111 | {{ if .function_name }}(#eq? @name "{{.function_name}}"){{end}} 112 | {{ if .only_public }}(#match? @name "^[A-Z]"){{end}} 113 | ) 114 | {{end}} 115 | 116 | template: | 117 | {{ range $file, $results := .ResultsByFile -}} 118 | {{ $hasResults := 119 | (and 120 | (gt (len $.ResultsByFile) 1) 121 | (or 122 | (and $results.typeAliasDeclarations (gt (len $results.typeAliasDeclarations.Matches) 0)) 123 | (and $results.functionDeclarations (gt (len $results.functionDeclarations.Matches) 0)) 124 | (and $results.interfaceDeclarations (gt (len $results.interfaceDeclarations.Matches) 0)) 125 | (and $results.structDeclarations (gt (len $results.structDeclarations.Matches) 0)) 126 | (and $results.methodDeclarations (gt (len $results.methodDeclarations.Matches) 0)) 127 | ) 128 | ) 129 | }} 130 | {{ with $results -}} 131 | {{ if $hasResults -}}File: {{ $file }}{{ end }} 132 | {{- range .typeAliasDeclarations.Matches }} 133 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 134 | type {{.typeName.Text}} {{ .typeAlias.Text }}{{ end }} 135 | {{- range .structDeclarations.Matches }} 136 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 137 | type {{.structName.Text}} {{ .structBody.Text }}{{ end }} 138 | {{ range .interfaceDeclarations.Matches -}} 139 | {{ if .comment }}{{ .comment.Text |indentBlock 2}} 140 | {{ end }} 141 | type {{ .interfaceName.Text }} {{ .interfaceBody.Text }} {{ end }} 142 | {{- range .functionDeclarations.Matches }} 143 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 144 | func {{ .name.Text }}{{ .parameters.Text }} {{ .result.Text }} 145 | {{- if $.with_body }} {{ .body.Text}}{{ end }} {{ end }} 146 | {{- range .methodDeclarations.Matches }} 147 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 148 | func {{.receiver.Text}} {{ .name.Text }}{{ .parameters.Text }} {{ .result.Text }} 149 | {{- if $.with_body }} {{ .body.Text}}{{end}}{{ end -}} 150 | {{ end -}} 151 | {{ end -}} 152 | 153 | {{ if .verbose -}} 154 | Results:{{ range $v := .Results }} 155 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 156 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 157 | {{end}}{{ end }} 158 | {{ end -}} 159 | -------------------------------------------------------------------------------- /cmd/oak/queries/go/structs.yaml: -------------------------------------------------------------------------------- 1 | name: structs 2 | short: Extract all structs and interfaces in a go file 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | 10 | language: go 11 | queries: 12 | - name: structDeclarations 13 | query: | 14 | ((comment)* @structComment . 15 | (type_declaration 16 | (type_spec 17 | name: (type_identifier) @structName 18 | type: (struct_type) @structBody))) 19 | 20 | - name: interfaceDeclarations 21 | query: | 22 | (type_declaration 23 | (type_spec 24 | name: (type_identifier) @interfaceName 25 | type: (interface_type) @interfaceBody)) 26 | 27 | template: | 28 | {{ range $file, $results := .ResultsByFile -}} 29 | File: {{ $file }} 30 | 31 | {{ with $results -}} 32 | Struct Declarations: 33 | {{- range .structDeclarations.Matches }} 34 | {{ if .structComment }}{{ .structComment.Text }} {{ end }} 35 | type {{.structName.Text}} {{ .structBody.Text | indent 2 }} 36 | {{ end }} 37 | 38 | Interface Declarations: 39 | {{ range .interfaceDeclarations.Matches -}} 40 | {{ if .interfaceComment }}{{ .interfaceComment.Text }} {{ end }} 41 | type {{ .interfaceName.Text }} {{ .interfaceBody.Text | indent 2}} 42 | {{ end }} 43 | {{ end -}} 44 | {{ end -}} 45 | 46 | {{ if .verbose -}} 47 | Results:{{ range $v := .Results }} 48 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 49 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 50 | {{end}}{{ end }} 51 | {{ end -}} 52 | -------------------------------------------------------------------------------- /cmd/oak/queries/php/classes.yaml: -------------------------------------------------------------------------------- 1 | name: classes 2 | short: Extract PHP classes 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: class 10 | type: string 11 | help: Only output classes matching name 12 | - name: list 13 | type: bool 14 | help: When true, output the list of type names 15 | default: false 16 | - name: with_comments 17 | type: bool 18 | help: When true, output the comments, else ignore them 19 | default: false 20 | - name: with_body 21 | type: bool 22 | help: When true, output the class body, else just output the class declaration 23 | default: false 24 | - name: count 25 | type: int 26 | help: Limit the number of types to output 27 | default: 0 28 | - name: offset 29 | type: int 30 | help: Offset the number of tests to output 31 | default: 0 32 | 33 | language: php 34 | queries: 35 | - name: classDeclarations 36 | query: | 37 | ( 38 | (comment)* @comment . 39 | (class_declaration 40 | (class_modifier)? @modifier 41 | name: (name) @className 42 | body: (declaration_list) @body 43 | ) 44 | {{ if .class }}(#eq? @className "{{ .class }}"){{end}} 45 | ) 46 | 47 | template: | 48 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 49 | {{ range $file, $results := .ResultsByFile }} 50 | {{ if (gt (len $.ResultsByFile) 1) -}}File: {{ $file }}{{ end }} 51 | {{- with $results -}} 52 | {{ if $.list -}} 53 | {{- $count := 0 }} 54 | {{- range .classDeclarations.Matches -}} 55 | {{ $gt := (gt (add $count 1) $.offset) -}} 56 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 57 | {{ $and := (and $gt $lt) -}} 58 | {{- if or $skipLimit $and }} 59 | - {{ .className.Text }} 60 | {{- end }} 61 | {{- $count = add $count 1 }} 62 | {{- end }} 63 | {{- else -}} 64 | {{- $count := 0 }} 65 | {{- range .classDeclarations.Matches -}} 66 | {{ $gt := (gt (add $count 1) $.offset) -}} 67 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 68 | {{ $and := (and $gt $lt) -}} 69 | {{- if or $skipLimit $and }} 70 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 71 | {{ if .modifier }}{{ .modifier.Text }} {{ end }}class {{.className.Text}} {{ if $.with_body }}= { 72 | {{ .body.Text | indentBlock 2 -}} 73 | }{{ end }} 74 | {{ end }} 75 | {{ $count = add $count 1 }} 76 | {{- end }} 77 | {{ end -}} 78 | {{ end }} 79 | {{ end }} 80 | 81 | {{ if .verbose -}} 82 | 83 | Results:{{ range $k, $v := .Results }} 84 | {{ $k }}: {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 85 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 86 | {{end}}{{ end }} 87 | {{ end -}} 88 | -------------------------------------------------------------------------------- /cmd/oak/queries/php/functions.yaml: -------------------------------------------------------------------------------- 1 | name: functions 2 | short: Extract PHP functions defined using export const 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: function 10 | type: string 11 | help: Only output functions matching name 12 | - name: with_body 13 | type: bool 14 | help: When true, output the function body, else just output the function declaration and parameters 15 | default: false 16 | - name: list 17 | type: bool 18 | help: When true, output the list of function names 19 | default: false 20 | - name: with_comments 21 | type: bool 22 | help: When true, output the comments, else ignore them 23 | default: false 24 | - name: count 25 | type: int 26 | help: Limit the number of functions to output 27 | default: 0 28 | - name: offset 29 | type: int 30 | help: Offset the number of tests to output 31 | default: 0 32 | - name: with_private 33 | type: bool 34 | help: When true, output private functions 35 | default: false 36 | 37 | language: php 38 | queries: 39 | - name: functionDeclarations 40 | query: | 41 | ( 42 | (comment)* @comment . 43 | (function_definition 44 | name: (name) @functionName 45 | parameters: (formal_parameters)? @parameters 46 | body: (compound_statement)? @body) 47 | {{ if .function }}(#eq? @functionName "{{.function}}"){{end}} 48 | ) 49 | 50 | - name: methodDeclarations 51 | query: | 52 | ( 53 | (comment)* @comment . 54 | (method_declaration 55 | name: (name) @functionName 56 | parameters: (formal_parameters)? @parameters 57 | body: (compound_statement)? @body) 58 | {{ if .function }}(#eq? @functionName "{{.function}}"){{end}} 59 | ) 60 | 61 | template: | 62 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 63 | {{ range $file, $results := .ResultsByFile }} 64 | {{ $hasResults := 65 | (and 66 | (gt (len $.ResultsByFile) 1) 67 | (or 68 | (gt (len $results.functionDeclarations.Matches) 0) 69 | (gt (len $results.methodDeclarations.Matches) 0) 70 | ) 71 | ) 72 | }} 73 | {{ if $hasResults -}}File: {{ $file }}{{ end }} 74 | {{ with $results -}} 75 | {{ if $.list -}} 76 | {{- $count := 0 }} 77 | {{- range .functionDeclarations.Matches -}} 78 | {{ $gt := (gt (add $count 1) $.offset) -}} 79 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 80 | {{ $and := (and $gt $lt) -}} 81 | {{- if or $skipLimit $and }} 82 | - {{ .functionName.Text }} 83 | {{- end }} 84 | {{- $count = add $count 1 }} 85 | {{- end }} 86 | {{- range .methodDeclarations.Matches -}} 87 | {{ $gt := (gt (add $count 1) $.offset) -}} 88 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 89 | {{ $and := (and $gt $lt) -}} 90 | {{- if or $skipLimit $and }} 91 | - {{ .functionName.Text }} 92 | {{- end }} 93 | {{- $count = add $count 1 }} 94 | {{- end }} 95 | {{- else -}} 96 | {{- $count := 0 }} 97 | {{- range .functionDeclarations.Matches -}} 98 | {{ $gt := (gt (add $count 1) $.offset) -}} 99 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 100 | {{ $and := (and $gt $lt) -}} 101 | {{- if or $skipLimit $and }} 102 | {{- if and .comment $.with_comments }}{{ .comment.Text |indentBlock 2}}{{end}} 103 | function {{.functionName.Text}} {{ .parameters.Text }} {{ if $.with_body }}{{ .body.Text | indent 2 }}{{ end }} 104 | {{- end }} 105 | {{- $count = add $count 1 }} 106 | {{- end }} 107 | {{- range .methodDeclarations.Matches -}} 108 | {{ $gt := (gt (add $count 1) $.offset) -}} 109 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 110 | {{ $and := (and $gt $lt) -}} 111 | {{- if or $skipLimit $and }} 112 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 113 | function {{ .functionName.Text }} {{ .parameters.Text }} {{ if $.with_body }}{{ .body.Text | indent 2 }}{{ end }} 114 | {{- end }} 115 | {{- $count = add $count 1 }} 116 | {{- end }} 117 | {{ end -}} 118 | {{ end }} 119 | {{ end }} 120 | 121 | {{ if .verbose -}} 122 | 123 | Results:{{ range $v := .Results }} 124 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 125 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 126 | {{end}}{{ end }} 127 | {{ end -}} 128 | -------------------------------------------------------------------------------- /cmd/oak/queries/php/wp/filters.yaml: -------------------------------------------------------------------------------- 1 | name: filters 2 | short: A command to list the filter calls to WordPress filters 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: match_action 10 | type: string 11 | help: only output action calls matching action 12 | - name: match_filter 13 | type: string 14 | help: only output filter calls matching filters 15 | - name: concise 16 | type: bool 17 | help: When true, output the concise version of the results 18 | default: false 19 | - name: only_filters 20 | type: bool 21 | help: When true, only output filter calls 22 | default: false 23 | - name: only_actions 24 | type: bool 25 | help: When true, only output action calls 26 | default: false 27 | 28 | 29 | language: php 30 | queries: 31 | - name: filterCalls 32 | query: | 33 | ( 34 | ( 35 | (function_call_expression 36 | function: (_) @function 37 | arguments: (arguments . (string) @filter ) @arguments 38 | ) 39 | (#eq? @function "apply_filters") 40 | {{ if .match_filter }}(#eq? @filter "{{.match_filter}}"){{end}} 41 | ) @fullexpression 42 | ) 43 | - name: actionCalls 44 | query: | 45 | ( 46 | (function_call_expression 47 | function: (_) @function 48 | arguments: (arguments . (string) @action) @arguments 49 | ) 50 | (#eq? @function "do_action") 51 | {{ if .match_action }}(#eq? @action "{{.match_action}}"){{end}} 52 | ) 53 | - name: filterRegistrations 54 | query: | 55 | ( 56 | ( 57 | (function_call_expression 58 | function: (_) @function 59 | arguments: (arguments . (string) @filter ) @arguments 60 | ) 61 | (#eq? @function "add_filter") 62 | {{ if .match_filter }}(#eq? @filter "{{.match_filter}}"){{end}} 63 | ) @fullexpression 64 | ) 65 | - name: actionRegistrations 66 | query: | 67 | ( 68 | (function_call_expression 69 | function: (_) @function 70 | arguments: (arguments . (string) @action) @arguments 71 | ) 72 | (#eq? @function "add_action") 73 | {{ if .match_action }}(#eq? @action "{{.match_action}}"){{end}} 74 | ) 75 | 76 | template: | 77 | {{ range $file, $results := .ResultsByFile -}} 78 | File: {{ $file }} 79 | {{ with $results -}} 80 | {{- if not $.only_actions }}# Filters 81 | {{ range .filterCalls.Matches }} 82 | Filter: {{.filter.Text}} 83 | {{ if not $.concise }}{{ if .arguments.Text }}Full expression: {{ .arguments.Text | indent 2 }}{{- end }}{{- end }} {{ end }} 84 | {{- range .filterRegistrations.Matches }} 85 | RegisterFilter: {{.filter.Text}} 86 | {{ if not $.concise }}{{ if .arguments.Text }}Full expression: {{ .arguments.Text | indent 2 }}{{- end }}{{- end }} {{ end }} 87 | {{- end }} 88 | {{- if not $.only_filters }} 89 | # Actions 90 | {{ range .actionCalls.Matches }} 91 | Action: {{.action.Text}} 92 | {{ if not $.concise }}{{ if .arguments.Text }}Full expression: {{ .arguments.Text | indent 2 }}{{- end }}{{- end }} {{ end -}} 93 | {{- range .actionRegistrations.Matches }} 94 | RegisterAction: {{.action.Text}} 95 | {{ if not $.concise }}{{ if .arguments.Text }}Full expression: {{ .arguments.Text | indent 2 }}{{- end }}{{- end }}{{ end -}} 96 | {{- end }} 97 | {{ end -}} 98 | {{ end -}} 99 | 100 | {{ if .verbose -}} 101 | Results:{{ range $v := .Results }} 102 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 103 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 104 | {{end}}{{ end }} 105 | {{ end -}} -------------------------------------------------------------------------------- /cmd/oak/queries/typescript/definitions.yaml: -------------------------------------------------------------------------------- 1 | name: definitions 2 | short: Extract TypeScript constant definitions using export const 3 | # Created with chatgpt 4 | # https://chat.openai.com/share/bf81616c-792b-45fc-9a74-667cd656dc14 5 | flags: 6 | - name: verbose 7 | type: bool 8 | help: Output all results 9 | default: false 10 | - name: name 11 | type: string 12 | help: Only output constants matching name 13 | - name: with_body 14 | type: bool 15 | help: When true, output the constant value, else just output the constant declaration 16 | default: false 17 | - name: list 18 | type: bool 19 | help: When true, output the list of constant names 20 | default: false 21 | - name: with_private 22 | type: bool 23 | help: When true, output private constants as well 24 | default: false 25 | - name: with_comments 26 | type: bool 27 | help: When true, output the comments, else ignore them 28 | default: false 29 | - name: count 30 | type: int 31 | help: Limit the number of constants to output 32 | default: 0 33 | - name: offset 34 | type: int 35 | help: Offset the number of constants to output 36 | default: 0 37 | 38 | language: tsx 39 | queries: 40 | - name: exportConstDeclarations 41 | # this will match arrow function consts twice, as we match both the arrow_function and the _. 42 | # Not sure how this can be solved, but we already know that this matching language is not great. 43 | query: | 44 | ( 45 | (comment)* @comment . 46 | {{ if not $.with_private -}} 47 | (export_statement 48 | {{ end -}} 49 | [ 50 | (function_declaration 51 | name: (identifier) @name 52 | parameters: (_) @parameters 53 | body: (_)? @constValue 54 | 55 | ) 56 | (lexical_declaration 57 | (variable_declarator 58 | name: (identifier) @name 59 | value: [ 60 | (_ !parameters) @constValue 61 | (arrow_function 62 | parameters: (formal_parameters)? @parameters 63 | body: (statement_block)? @body)? 64 | ] 65 | )) 66 | (class_declaration 67 | name: (type_identifier) @name 68 | (class_heritage)? @heritage 69 | (class_body) @constValue 70 | ) 71 | ] 72 | {{- if not $.with_private -}} ) {{ end -}} 73 | {{ if .name }}(#eq? @name "{{.name}}"){{end}} 74 | ) 75 | 76 | template: | 77 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 78 | {{ range $file, $results := .ResultsByFile }} 79 | {{ $hasResults := 80 | (and 81 | (gt (len $.ResultsByFile) 1) 82 | (gt (len $results.exportConstDeclarations.Matches) 0) 83 | ) 84 | }} 85 | {{ if $hasResults -}}File: {{ $file }}{{ end }} 86 | {{ with $results -}} 87 | {{ if $.list -}} 88 | {{- $count := 0 }} 89 | {{- range .exportConstDeclarations.Matches -}} 90 | {{ $gt := (gt (add $count 1) $.offset) -}} 91 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 92 | {{ $and := (and $gt $lt) -}} 93 | {{- if or $skipLimit $and }} 94 | - {{ .name.Text }} 95 | {{- end }} 96 | {{- $count = add $count 1 }} 97 | {{- end }} 98 | {{- else -}} 99 | {{- $count := 0 }} 100 | {{- range .exportConstDeclarations.Matches -}} 101 | {{ $gt := (gt (add $count 1) $.offset) -}} 102 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 103 | {{ $and := (and $gt $lt) -}} 104 | {{- if or $skipLimit $and }} 105 | {{- if and .comment $.with_comments }}{{ .comment.Text |indentBlock 2}}{{end}} 106 | export const {{ .name.Text }} {{if .parameters}}{{.parameters.Text}}{{end}}{{if .heritage }}{{.heritage.Text}}{{end}} {{ if $.with_body }}= {{ .constValue.Text }}{{ end }} 107 | {{- end }} 108 | {{- $count = add $count 1 }} 109 | {{- end }} 110 | {{ end -}} 111 | {{ end }} 112 | {{ end }} 113 | 114 | {{ if .verbose -}} 115 | 116 | Results:{{ range $v := .Results }} 117 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 118 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 119 | {{end}}{{ end }} 120 | {{ end -}} 121 | -------------------------------------------------------------------------------- /cmd/oak/queries/typescript/functions.yaml: -------------------------------------------------------------------------------- 1 | name: functions 2 | short: Extract TypeScript functions defined using export const 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: function 10 | type: string 11 | help: Only output functions matching name 12 | - name: with_body 13 | type: bool 14 | help: When true, output the function body, else just output the function declaration and parameters 15 | default: false 16 | - name: list 17 | type: bool 18 | help: When true, output the list of function names 19 | default: false 20 | - name: with_comments 21 | type: bool 22 | help: When true, output the comments, else ignore them 23 | default: false 24 | - name: count 25 | type: int 26 | help: Limit the number of functions to output 27 | default: 0 28 | - name: offset 29 | type: int 30 | help: Offset the number of tests to output 31 | default: 0 32 | - name: with_private 33 | type: bool 34 | help: When true, output private functions 35 | default: false 36 | 37 | language: tsx 38 | queries: 39 | - name: functionDeclarations 40 | query: | 41 | ( 42 | (comment)* @comment . 43 | (function_declaration 44 | name: (identifier) @functionName 45 | parameters: (formal_parameters)? @parameters 46 | body: (statement_block)? @body) 47 | {{ if .function }}(#eq? @functionName "{{.function}}"){{end}} 48 | ) 49 | 50 | - name: arrowFunctionDeclarations 51 | query: | 52 | ( 53 | (comment)* @comment . 54 | {{ if not $.with_private -}} 55 | (export_statement 56 | {{ end -}} 57 | (lexical_declaration 58 | (variable_declarator 59 | name: (identifier) @functionName 60 | value: (arrow_function 61 | parameters: (formal_parameters)? @parameters 62 | body: (statement_block)? @body))) 63 | {{- if not $.with_private -}} ) {{ end -}} 64 | {{ if .function }}(#eq? @functionName "{{.function}}"){{end}} 65 | ) 66 | 67 | template: | 68 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 69 | {{ range $file, $results := .ResultsByFile }} 70 | {{ $hasResults := 71 | (and 72 | (gt (len $.ResultsByFile) 1) 73 | (or 74 | (gt (len $results.functionDeclarations.Matches) 0) 75 | (gt (len $results.arrowFunctionDeclarations.Matches) 0) 76 | ) 77 | ) 78 | }} 79 | {{ if $hasResults -}}File: {{ $file }}{{ end }} 80 | {{ with $results -}} 81 | {{ if $.list -}} 82 | {{- $count := 0 }} 83 | {{- range .functionDeclarations.Matches -}} 84 | {{ $gt := (gt (add $count 1) $.offset) -}} 85 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 86 | {{ $and := (and $gt $lt) -}} 87 | {{- if or $skipLimit $and }} 88 | - {{ .functionName.Text }} 89 | {{- end }} 90 | {{- $count = add $count 1 }} 91 | {{- end }} 92 | {{- range .arrowFunctionDeclarations.Matches -}} 93 | {{ $gt := (gt (add $count 1) $.offset) -}} 94 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 95 | {{ $and := (and $gt $lt) -}} 96 | {{- if or $skipLimit $and }} 97 | - {{ .functionName.Text }} 98 | {{- end }} 99 | {{- $count = add $count 1 }} 100 | {{- end }} 101 | {{- else -}} 102 | {{- $count := 0 }} 103 | {{- range .functionDeclarations.Matches -}} 104 | {{ $gt := (gt (add $count 1) $.offset) -}} 105 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 106 | {{ $and := (and $gt $lt) -}} 107 | {{- if or $skipLimit $and }} 108 | {{- if and .comment $.with_comments }}{{ .comment.Text |indentBlock 2}}{{end}} 109 | function {{.functionName.Text}} {{ .parameters.Text }} {{ if $.with_body }}{{ .body.Text | indent 2 }}{{ end }} 110 | {{- end }} 111 | {{- $count = add $count 1 }} 112 | {{- end }} 113 | {{- range .arrowFunctionDeclarations.Matches -}} 114 | {{ $gt := (gt (add $count 1) $.offset) -}} 115 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 116 | {{ $and := (and $gt $lt) -}} 117 | {{- if or $skipLimit $and }} 118 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 119 | export const {{ .functionName.Text }} = {{ .parameters.Text }}{{ if $.with_body }} => {{ .body.Text }}{{ end }} 120 | {{- end }} 121 | {{- $count = add $count 1 }} 122 | {{- end }} 123 | {{ end -}} 124 | {{ end }} 125 | {{ end }} 126 | 127 | {{ if .verbose -}} 128 | 129 | Results:{{ range $v := .Results }} 130 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 131 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 132 | {{end}}{{ end }} 133 | {{ end -}} 134 | -------------------------------------------------------------------------------- /cmd/oak/queries/typescript/tests.yaml: -------------------------------------------------------------------------------- 1 | name: tests 2 | short: Extract unit tests from a file 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: with_body 10 | type: bool 11 | help: When true, output the test body, else just output the test declaration 12 | default: false 13 | - name: list 14 | type: bool 15 | help: When true, output the list of test names 16 | default: false 17 | - name: with_comments 18 | type: bool 19 | help: When true, output the comments, else ignore them 20 | default: false 21 | - name: count 22 | type: int 23 | help: Limit the number of tests to output 24 | default: 0 25 | - name: offset 26 | type: int 27 | help: Offset the number of tests to output 28 | default: 0 29 | 30 | language: typescript 31 | queries: 32 | - name: testDeclarations 33 | query: | 34 | (call_expression 35 | function: (identifier) @functionName 36 | arguments: (arguments 37 | (string) @testName 38 | (arrow_function 39 | body: (statement_block)? @body)) 40 | (#eq? @functionName "test") 41 | ) 42 | 43 | template: | 44 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 45 | {{ range $file, $results := .ResultsByFile }} 46 | File: {{ $file }} 47 | {{ with $results -}} 48 | {{ if $.list -}} 49 | {{- $count := 0 }} 50 | {{- range .testDeclarations.Matches -}} 51 | {{ $gt := (gt (add $count 1) $.offset) -}} 52 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 53 | {{ $and := (and $gt $lt) -}} 54 | {{- if or $skipLimit $and }} 55 | - {{ .testName.Text }} 56 | {{- end }} 57 | {{- $count = add $count 1 }} 58 | {{- end }} 59 | {{- else -}} 60 | {{- $count := 0 }} 61 | {{- range .testDeclarations.Matches -}} 62 | {{ $gt := (gt (add $count 1) $.offset) -}} 63 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 64 | {{ $and := (and $gt $lt) -}} 65 | {{- if or $skipLimit $and }} 66 | {{- if and .comment $.with_comments }}{{ .comment.Text |indentBlock 2}}{{end}} 67 | {{ .functionName.Text }}({{ .testName.Text }}{{ if $.with_body }}, () => {{ .body.Text | indent 2 }} 68 | {{ end }}) 69 | {{- end }} 70 | {{- $count = add $count 1 }} 71 | {{- end }} 72 | {{ end -}} 73 | {{ end }} 74 | {{ end }} 75 | 76 | {{ if .verbose -}} 77 | 78 | Results:{{ range $v := .Results }} 79 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 80 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 81 | {{end}}{{ end }} 82 | {{ end -}} 83 | -------------------------------------------------------------------------------- /cmd/oak/queries/typescript/types.yaml: -------------------------------------------------------------------------------- 1 | name: types 2 | short: Extract TypeScript type declarations 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | - name: type 10 | type: string 11 | help: Only output types matching name 12 | - name: list 13 | type: bool 14 | help: When true, output the list of type names 15 | default: false 16 | - name: with_comments 17 | type: bool 18 | help: When true, output the comments, else ignore them 19 | default: false 20 | - name: count 21 | type: int 22 | help: Limit the number of types to output 23 | default: 0 24 | - name: offset 25 | type: int 26 | help: Offset the number of tests to output 27 | default: 0 28 | 29 | language: tsx 30 | queries: 31 | - name: typeDeclarations 32 | query: | 33 | ( 34 | (comment)* @comment . 35 | ( 36 | type_alias_declaration 37 | name: (type_identifier) @typeName 38 | value: (_) @body 39 | ) 40 | {{ if .type }}(#eq? @typeName "{{ .type }}"){{end}} 41 | 42 | ) 43 | - name: interfaceDeclarations 44 | query: | 45 | ( 46 | (comment)* @comment . 47 | ( 48 | interface_declaration 49 | name: (type_identifier) @interfaceName 50 | body: (_) @body 51 | ) 52 | {{ if .type }}(#eq? @interfaceName "{{ .type }}"){{end}} 53 | ) 54 | - name: enumDeclarations 55 | query: | 56 | ( 57 | (comment)* @comment . 58 | ( 59 | enum_declaration 60 | name: (identifier) @enumName 61 | body: (_) @body 62 | ) 63 | {{ if .type }}(#eq? @enumName "{{ .type }}"){{end}} 64 | ) 65 | 66 | template: | 67 | {{ $skipLimit := (and (eq $.count 0) (eq $.offset 0)) }} 68 | {{ range $file, $results := .ResultsByFile }} 69 | {{ if (gt (len $.ResultsByFile) 1) -}}File: {{ $file }}{{ end }} 70 | {{- with $results -}} 71 | {{ if $.list -}} 72 | {{- $count := 0 }} 73 | {{- range .typeDeclarations.Matches -}} 74 | {{ $gt := (gt (add $count 1) $.offset) -}} 75 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 76 | {{ $and := (and $gt $lt) -}} 77 | {{- if or $skipLimit $and }} 78 | - {{ .typeName.Text }} 79 | {{- end }} 80 | {{- $count = add $count 1 }} 81 | {{- end }} 82 | {{- range .interfaceDeclarations.Matches -}} 83 | {{ $gt := (gt (add $count 1) $.offset) -}} 84 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 85 | {{ $and := (and $gt $lt) -}} 86 | {{- if or $skipLimit $and }} 87 | - {{ .interfaceName.Text }} 88 | {{- end }} 89 | {{- $count = add $count 1 }} 90 | {{- end }} 91 | {{- range .enumDeclarations.Matches -}} 92 | {{ $gt := (gt (add $count 1) $.offset) -}} 93 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 94 | {{ $and := (and $gt $lt) -}} 95 | {{- if or $skipLimit $and }} 96 | - {{ .enumName.Text }} 97 | {{- end }} 98 | {{- $count = add $count 1 }} 99 | {{- end }} 100 | {{- else -}} 101 | {{- $count := 0 }} 102 | {{- range .typeDeclarations.Matches -}} 103 | {{ $gt := (gt (add $count 1) $.offset) -}} 104 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 105 | {{ $and := (and $gt $lt) -}} 106 | {{- if or $skipLimit $and }} 107 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 108 | type {{.typeName.Text}} = { 109 | {{ .body.Text | indentBlock 2 -}} 110 | } 111 | {{ end }} 112 | {{ $count = add $count 1 }} 113 | {{- end }} 114 | {{- range .interfaceDeclarations.Matches -}} 115 | {{ $gt := (gt (add $count 1) $.offset) -}} 116 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 117 | {{ $and := (and $gt $lt) -}} 118 | {{- if or $skipLimit $and }} 119 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 120 | interface {{.interfaceName.Text}} = { 121 | {{ .body.Text | indentBlock 2 -}} 122 | } 123 | {{ end }} 124 | {{ $count = add $count 1 }} 125 | {{- end }} 126 | {{- range .enumDeclarations.Matches -}} 127 | {{ $gt := (gt (add $count 1) $.offset) -}} 128 | {{ $lt := (or (eq $.count 0) (lt $count (add $.offset $.count))) -}} 129 | {{ $and := (and $gt $lt) -}} 130 | {{- if or $skipLimit $and }} 131 | {{- if and .comment $.with_comments }}{{ .comment.Text | indentBlock 2}}{{end}} 132 | enum {{.enumName.Text}} = { 133 | {{ .body.Text | indentBlock 2 -}} 134 | } 135 | {{ end }} 136 | {{ $count = add $count 1 }} 137 | {{- end }} 138 | {{ end -}} 139 | {{ end }} 140 | {{ end }} 141 | 142 | {{ if .verbose -}} 143 | 144 | Results:{{ range $v := .Results }} 145 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 146 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 147 | {{end}}{{ end }} 148 | {{ end -}} 149 | -------------------------------------------------------------------------------- /contexts/dsl.txt: -------------------------------------------------------------------------------- 1 | Here is an example of our input DSL. 2 | 3 | It contains a list of tree-sitter queries used to query an input AST in the given language. 4 | The named captures can then be used to render the template. 5 | 6 | ```yaml 7 | language: go 8 | 9 | queries: 10 | - name: functionDeclarations 11 | query: | 12 | (function_declaration 13 | name: (identifier) @name 14 | parameters: (parameter_list) 15 | body: (block) 16 | (comment) @comment) 17 | - name: importStatements 18 | query: | 19 | (import_declaration 20 | (import_spec 21 | (identifier) @importName)) 22 | - name: variableDeclarations 23 | query: | 24 | (variable_declaration 25 | (identifier) @varName) 26 | 27 | template: | 28 | Function Declarations: 29 | {{ range .functionDeclarations }} 30 | - Name: {{ .name.text }} 31 | Comment: {{ .comment.text }} 32 | {{ end }} 33 | 34 | Import Statements: 35 | {{ range .importStatements }} 36 | - Name: {{ .importName.text }} 37 | {{ end }} 38 | 39 | Variable Declarations: 40 | {{ range .variableDeclarations }} 41 | - Name: {{ .varName.text }} 42 | {{ end }} 43 | ``` -------------------------------------------------------------------------------- /contexts/generate-api-query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # tree-sitter parse contexts/test.go | sed -e 's/\[.*\] - \[.*\]//' -e 's/ )/)/' | pbcopy 4 | 5 | #pinocchio mine do \ 6 | # --intro "Here is the tree-sitter query API documentation." \ 7 | # --context-file contexts/golang-example.txt \ 8 | # --context-file contexts/tree-sitter-query.md \ 9 | # --context-file contexts/dsl.txt \ 10 | # --goal-file contexts/goals/spec.txt 11 | 12 | #pinocchio mine do \ 13 | # --intro "Here is the tree-sitter query API documentation." \ 14 | # --context-file contexts/tree-sitter-query.md \ 15 | # --context-file contexts/dsl.txt \ 16 | # --goal "Write a YAML DSL file that covers all the examples in the tree-sitter query documentation." 17 | 18 | pinocchio mine do \ 19 | --context-file contexts/golang-example.txt \ 20 | --goal "Rewrite the following YAML DSL file that contains tree-sitter queries to work against the GO AST." \ 21 | --instructions-file queries/spec.yaml 22 | 23 | #pinocchio mine do \ 24 | # --context-file contexts/golang-example.txt \ 25 | # --goal "extract all node types from the go tree-sitter AST and return as a bullet list." 26 | 27 | pinocchio mine do \ 28 | --context-file contexts/golang-types.txt \ 29 | --goal "Use the go AST types to fix the tree-sitter queries in following YAML, because the node types often don't exist." \ 30 | --instructions-file queries/go-spec.yaml 31 | 32 | pinocchio mine do \ 33 | --context-file contexts/golang-example.txt \ 34 | --goal "Fix the tree-sitter queries in following YAML, because there are often missing intermediate nodes." \ 35 | --instructions-file queries/go-spec-fixed.yaml 36 | 37 | # go run ./cmd/oak run --query-file queries/go-spec-fixed2.yaml --input-file test-inputs/test.go 2>&1 | grep level | jq -r ".query" 38 | 39 | pinocchio mine do \ 40 | --context-file queries/go-spec-fixed2.yaml \ 41 | --goal "Use the tree-sitter queries listed above to generate a go file covering all the cases." -------------------------------------------------------------------------------- /contexts/goal.txt: -------------------------------------------------------------------------------- 1 | You are an expert go programmer. 2 | 3 | Write a go program that uses treesitter to parse a go file, and run a simple query against the resulting AST. 4 | -------------------------------------------------------------------------------- /contexts/goals/parser.txt: -------------------------------------------------------------------------------- 1 | Write: 2 | - the necessary structures to parse the DSL into a struct OakCommand 3 | - a NewOakCommandFromReader method to load a DSL from YAML 4 | - a method ExecuteQuery(tree, sourceCode) on OakCommand to return a map[string]Result 5 | - a method RenderTemplate(results) on OakCommand to render a template with the results -------------------------------------------------------------------------------- /contexts/goals/spec.txt: -------------------------------------------------------------------------------- 1 | Generate a DSL YAML file that covers all queries, using the go AST nodes: 2 | 3 | - Fields 4 | - Negated Fields 5 | - Anonymous Nodes 6 | - Capturing Nodes 7 | - Quantification Operators 8 | - Grouping Sibling Nodes 9 | - Alternations 10 | - Wildcard Node 11 | - Anchors 12 | - Predicates 13 | 14 | -------------------------------------------------------------------------------- /contexts/golang-example.txt: -------------------------------------------------------------------------------- 1 | Here is an example of a tree-sitter AST resulting from a golang file: 2 | 3 | (source_file 4 | (package_clause 5 | (package_identifier)) 6 | (import_declaration 7 | (import_spec 8 | path: (interpreted_string_literal))) 9 | (type_declaration 10 | (type_spec 11 | name: (type_identifier) 12 | type: (struct_type 13 | (field_declaration_list 14 | (field_declaration 15 | name: (field_identifier) 16 | type: (type_identifier) 17 | tag: (raw_string_literal)) 18 | (field_declaration 19 | name: (field_identifier) 20 | type: (type_identifier)))))) 21 | (function_declaration 22 | name: (identifier) 23 | parameters: (parameter_list 24 | (parameter_declaration 25 | name: (identifier) 26 | type: (type_identifier))) 27 | result: (type_identifier) 28 | body: (block 29 | (return_statement 30 | (expression_list 31 | (binary_expression 32 | left: (identifier) 33 | right: (int_literal)))))) 34 | (method_declaration 35 | receiver: (parameter_list 36 | (parameter_declaration 37 | name: (identifier) 38 | type: (pointer_type 39 | (type_identifier)))) 40 | name: (field_identifier) 41 | parameters: (parameter_list) 42 | body: (block 43 | (if_statement 44 | condition: (true) 45 | consequence: (block 46 | (return_statement))) 47 | (short_var_declaration 48 | left: (expression_list 49 | (identifier)) 50 | right: (expression_list 51 | (int_literal))) 52 | (assignment_statement 53 | left: (expression_list 54 | (selector_expression 55 | operand: (identifier) 56 | field: (field_identifier))) 57 | right: (expression_list 58 | (identifier))) 59 | (assignment_statement 60 | left: (expression_list 61 | (identifier)) 62 | right: (expression_list 63 | (call_expression 64 | function: (identifier) 65 | arguments: (argument_list 66 | (identifier))))) 67 | (for_statement 68 | (for_clause 69 | initializer: (short_var_declaration 70 | left: (expression_list 71 | (identifier)) 72 | right: (expression_list 73 | (int_literal))) 74 | condition: (binary_expression 75 | left: (identifier) 76 | right: (int_literal)) 77 | update: (inc_statement 78 | (identifier))) 79 | body: (block 80 | (inc_statement 81 | (identifier)))) 82 | (if_statement 83 | condition: (binary_expression 84 | left: (identifier) 85 | right: (int_literal)) 86 | consequence: (block 87 | (return_statement))) 88 | (call_expression 89 | function: (selector_expression 90 | operand: (identifier) 91 | field: (field_identifier)) 92 | arguments: (argument_list 93 | (interpreted_string_literal) 94 | (selector_expression 95 | operand: (identifier) 96 | field: (field_identifier)) 97 | (selector_expression 98 | operand: (identifier) 99 | field: (field_identifier)))))) 100 | (function_declaration 101 | name: (identifier) 102 | parameters: (parameter_list) 103 | body: (block 104 | (call_expression 105 | function: (selector_expression 106 | operand: (identifier) 107 | field: (field_identifier)) 108 | arguments: (argument_list 109 | (interpreted_string_literal)))))) 110 | -------------------------------------------------------------------------------- /contexts/golang-types.txt: -------------------------------------------------------------------------------- 1 | Here is a list of the tree-sitter AST node types for the go language: 2 | - source_file 3 | - package_clause 4 | - package_identifier 5 | - import_declaration 6 | - import_spec 7 | - interpreted_string_literal 8 | - type_declaration 9 | - type_spec 10 | - type_identifier 11 | - struct_type 12 | - field_declaration_list 13 | - field_declaration 14 | - field_identifier 15 | - method_declaration 16 | - parameter_list 17 | - parameter_declaration 18 | - identifier 19 | - pointer_type 20 | - block 21 | - if_statement 22 | - true 23 | - return_statement 24 | - short_var_declaration 25 | - expression_list 26 | - int_literal 27 | - assignment_statement 28 | - selector_expression 29 | - field 30 | - for_statement 31 | - for_clause 32 | - binary_expression 33 | - inc_statement 34 | - call_expression 35 | - function 36 | - argument_list -------------------------------------------------------------------------------- /contexts/instructions/just-code.txt: -------------------------------------------------------------------------------- 1 | Only output the function. No prose, no package imports, no explanations, no main function, just code. -------------------------------------------------------------------------------- /contexts/prompt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pinocchio mine do \ 4 | --context-file contexts/goal.txt \ 5 | --context-file contexts/dsl.txt \ 6 | --context-file contexts/tree-sitter-predicates-examples.txt \ 7 | --instructions-file contexts/instructions/just-code.txt \ 8 | --goal-file contexts/goals/parser.txt -------------------------------------------------------------------------------- /contexts/test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | type Foo struct { 6 | a int `json:"a" yaml:"a"` 7 | b int 8 | } 9 | 10 | func foo(a int) int { 11 | return a + 1 12 | } 13 | 14 | func (f *Foo) Bar() { 15 | if true { 16 | return 17 | } 18 | a := 1 19 | f.a = a 20 | a = foo(a) 21 | for i := 0; i < 10; i++ { 22 | a++ 23 | } 24 | if a > 10 { 25 | return 26 | } 27 | v := map[string]int{} 28 | _ = v 29 | fmt.Printf("bar %d %d", f.a, f.b) 30 | } 31 | 32 | func main() { 33 | fmt.Println("foobar") 34 | } 35 | -------------------------------------------------------------------------------- /contexts/tree-sitter-api.txt: -------------------------------------------------------------------------------- 1 | Here is the API of the smacker/tree-sitter library. 2 | 3 | type BaseTree 4 | func (t *BaseTree) Close() 5 | type EditInput 6 | type Input 7 | type InputEncoding 8 | type IterMode 9 | type Iterator 10 | func NewIterator(n *Node, mode IterMode) *Iterator 11 | func NewNamedIterator(n *Node, mode IterMode) *Iterator 12 | func (iter *Iterator) ForEach(fn func(*Node) error) error 13 | func (iter *Iterator) Next() (*Node, error) 14 | type Language 15 | func NewLanguage(ptr unsafe.Pointer) *Language 16 | func (l *Language) FieldName(idx int) string 17 | func (l *Language) SymbolCount() uint32 18 | func (l *Language) SymbolName(s Symbol) string 19 | func (l *Language) SymbolType(s Symbol) SymbolType 20 | type Node 21 | func Parse(content []byte, lang *Language) *NodeDEPRECATED 22 | func ParseCtx(ctx context.Context, content []byte, lang *Language) (*Node, error) 23 | func (n Node) Child(idx int) *Node 24 | func (n Node) ChildByFieldName(name string) *Node 25 | func (n Node) ChildCount() uint32 26 | func (n Node) Content(input []byte) string 27 | func (n Node) Edit(i EditInput) 28 | func (n Node) EndByte() uint32 29 | func (n Node) EndPoint() Point 30 | func (n Node) Equal(other *Node) bool 31 | func (n Node) FieldNameForChild(idx int) string 32 | func (n Node) HasChanges() bool 33 | func (n Node) HasError() bool 34 | func (n Node) IsError() bool 35 | func (n Node) IsExtra() bool 36 | func (n Node) IsMissing() bool 37 | func (n Node) IsNamed() bool 38 | func (n Node) IsNull() bool 39 | func (n Node) NamedChild(idx int) *Node 40 | func (n Node) NamedChildCount() uint32 41 | func (n Node) NamedDescendantForPointRange(start Point, end Point) *Node 42 | func (n Node) NextNamedSibling() *Node 43 | func (n Node) NextSibling() *Node 44 | func (n Node) Parent() *Node 45 | func (n Node) PrevNamedSibling() *Node 46 | func (n Node) PrevSibling() *Node 47 | func (n Node) StartByte() uint32 48 | func (n Node) StartPoint() Point 49 | func (n Node) String() string 50 | func (n Node) Symbol() Symbol 51 | func (n Node) Type() string 52 | type Parser 53 | func NewParser() *Parser 54 | func (p *Parser) Close() 55 | func (p *Parser) Debug() 56 | func (p *Parser) OperationLimit() int 57 | func (p *Parser) Parse(oldTree *Tree, content []byte) *TreeDEPRECATED 58 | func (p *Parser) ParseCtx(ctx context.Context, oldTree *Tree, content []byte) (*Tree, error) 59 | func (p *Parser) ParseInput(oldTree *Tree, input Input) *Tree 60 | func (p *Parser) ParseInputCtx(ctx context.Context, oldTree *Tree, input Input) (*Tree, error) 61 | func (p *Parser) Reset() 62 | func (p *Parser) SetIncludedRanges(ranges []Range) 63 | func (p *Parser) SetLanguage(lang *Language) 64 | func (p *Parser) SetOperationLimit(limit int) 65 | type Point 66 | type Query 67 | func NewQuery(pattern []byte, lang *Language) (*Query, error) 68 | func (q *Query) CaptureCount() uint32 69 | func (q *Query) CaptureNameForId(id uint32) string 70 | func (q *Query) Close() 71 | func (q *Query) PatternCount() uint32 72 | func (q *Query) PredicatesForPattern(patternIndex uint32) []QueryPredicateStep 73 | func (q *Query) StringCount() uint32 74 | func (q *Query) StringValueForId(id uint32) string 75 | type QueryCapture 76 | type QueryCursor 77 | func NewQueryCursor() *QueryCursor 78 | func (qc *QueryCursor) Close() 79 | func (qc *QueryCursor) Exec(q *Query, n *Node) 80 | func (qc *QueryCursor) FilterPredicates(m *QueryMatch, input []byte) *QueryMatch 81 | func (qc *QueryCursor) NextCapture() (*QueryMatch, uint32, bool) 82 | func (qc *QueryCursor) NextMatch() (*QueryMatch, bool) 83 | func (qc *QueryCursor) SetPointRange(startPoint Point, endPoint Point) 84 | type QueryError 85 | func (qe *QueryError) Error() string 86 | type QueryErrorType 87 | type QueryMatch 88 | type QueryPredicateStep 89 | type QueryPredicateStepType 90 | type Range 91 | type ReadFunc 92 | type Symbol 93 | type SymbolType 94 | func (t SymbolType) String() string 95 | type Tree 96 | func (t *Tree) Copy() *Tree 97 | func (t *Tree) Edit(i EditInput) 98 | func (t *Tree) RootNode() *Node 99 | type TreeCursor 100 | func NewTreeCursor(n *Node) *TreeCursor 101 | func (c *TreeCursor) Close() 102 | func (c *TreeCursor) CurrentFieldName() string 103 | func (c *TreeCursor) CurrentNode() *Node 104 | func (c *TreeCursor) GoToFirstChild() bool 105 | func (c *TreeCursor) GoToFirstChildForByte(b uint32) int64 106 | func (c *TreeCursor) GoToNextSibling() bool 107 | func (c *TreeCursor) GoToParent() bool 108 | func (c *TreeCursor) Reset(n *Node) 109 | 110 | -------------------------------------------------------------------------------- /contexts/tree-sitter-predicates-examples.md: -------------------------------------------------------------------------------- 1 | Predicates 2 | 3 | You can filter AST by using predicate S-expressions. 4 | 5 | Similar to Rust or WebAssembly bindings we support filtering on a few common predicates: 6 | 7 | eq?, not-eq? 8 | match?, not-match? 9 | Usage example: 10 | 11 | ```go 12 | func main() { 13 | // Javascript code 14 | sourceCode := []byte(` 15 | const camelCaseConst = 1; 16 | const SCREAMING_SNAKE_CASE_CONST = 2; 17 | const lower_snake_case_const = 3;`) 18 | // Query with predicates 19 | screamingSnakeCasePattern := `( 20 | (identifier) @constant 21 | (#match? @constant "^[A-Z][A-Z_]+") 22 | )` 23 | 24 | // Parse source code 25 | lang := javascript.GetLanguage() 26 | n, _ := sitter.ParseCtx(context.Background(), sourceCode, lang) 27 | // Execute the query 28 | q, _ := sitter.NewQuery([]byte(screamingSnakeCasePattern), lang) 29 | qc := sitter.NewQueryCursor() 30 | qc.Exec(q, n) 31 | // Iterate over query results 32 | for { 33 | m, ok := qc.NextMatch() 34 | if !ok { 35 | break 36 | } 37 | // Apply predicates filtering 38 | m = qc.FilterPredicates(m, sourceCode) 39 | for _, c := range m.Captures { 40 | fmt.Println(c.Node.Content(sourceCode)) 41 | } 42 | } 43 | } 44 | ``` 45 | 46 | // Output of this program: 47 | // SCREAMING_SNAKE_CASE_CONST -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-go-golems/oak 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/bmatcuk/doublestar/v4 v4.8.1 7 | github.com/go-go-golems/clay v0.1.39 8 | github.com/go-go-golems/glazed v0.5.48 9 | github.com/pkg/errors v0.9.1 10 | github.com/rs/zerolog v1.34.0 11 | github.com/smacker/go-tree-sitter v0.0.0-20231219031718-233c2f923ac7 12 | github.com/spf13/cobra v1.9.1 13 | github.com/spf13/viper v1.20.1 14 | gopkg.in/yaml.v3 v3.0.1 15 | ) 16 | 17 | require ( 18 | github.com/BurntSushi/toml v1.3.2 // indirect 19 | github.com/Masterminds/goutils v1.1.1 // indirect 20 | github.com/Masterminds/semver v1.5.0 // indirect 21 | github.com/Masterminds/sprig v2.22.0+incompatible // indirect 22 | github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect 23 | github.com/adrg/frontmatter v0.2.0 // indirect 24 | github.com/alecthomas/chroma/v2 v2.16.0 // indirect 25 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect 26 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 27 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 28 | github.com/aymerick/douceur v0.2.0 // indirect 29 | github.com/bahlo/generic-list-go v0.2.0 // indirect 30 | github.com/bits-and-blooms/bitset v1.22.0 // indirect 31 | github.com/blevesearch/bleve/v2 v2.5.0 // indirect 32 | github.com/blevesearch/bleve_index_api v1.2.7 // indirect 33 | github.com/blevesearch/geo v0.1.20 // indirect 34 | github.com/blevesearch/go-faiss v1.0.25 // indirect 35 | github.com/blevesearch/go-porterstemmer v1.0.3 // indirect 36 | github.com/blevesearch/gtreap v0.1.1 // indirect 37 | github.com/blevesearch/mmap-go v1.0.4 // indirect 38 | github.com/blevesearch/scorch_segment_api/v2 v2.3.9 // indirect 39 | github.com/blevesearch/segment v0.9.1 // indirect 40 | github.com/blevesearch/snowballstem v0.9.0 // indirect 41 | github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect 42 | github.com/blevesearch/vellum v1.1.0 // indirect 43 | github.com/blevesearch/zapx/v11 v11.4.1 // indirect 44 | github.com/blevesearch/zapx/v12 v12.4.1 // indirect 45 | github.com/blevesearch/zapx/v13 v13.4.1 // indirect 46 | github.com/blevesearch/zapx/v14 v14.4.1 // indirect 47 | github.com/blevesearch/zapx/v15 v15.4.1 // indirect 48 | github.com/blevesearch/zapx/v16 v16.2.2 // indirect 49 | github.com/buger/jsonparser v1.1.1 // indirect 50 | github.com/charmbracelet/colorprofile v0.3.0 // indirect 51 | github.com/charmbracelet/glamour v0.9.1 // indirect 52 | github.com/charmbracelet/lipgloss v1.1.0 // indirect 53 | github.com/charmbracelet/x/ansi v0.8.0 // indirect 54 | github.com/charmbracelet/x/cellbuf v0.0.13 // indirect 55 | github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 // indirect 56 | github.com/charmbracelet/x/term v0.2.1 // indirect 57 | github.com/dlclark/regexp2 v1.11.5 // indirect 58 | github.com/fsnotify/fsnotify v1.9.0 // indirect 59 | github.com/go-openapi/errors v0.22.0 // indirect 60 | github.com/go-openapi/strfmt v0.23.0 // indirect 61 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 62 | github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect 63 | github.com/golang/protobuf v1.5.4 // indirect 64 | github.com/golang/snappy v0.0.4 // indirect 65 | github.com/google/uuid v1.6.0 // indirect 66 | github.com/gorilla/css v1.0.1 // indirect 67 | github.com/huandu/xstrings v1.5.0 // indirect 68 | github.com/imdario/mergo v0.3.16 // indirect 69 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 70 | github.com/itchyny/gojq v0.12.12 // indirect 71 | github.com/itchyny/timefmt-go v0.1.5 // indirect 72 | github.com/jedib0t/go-pretty v4.3.0+incompatible // indirect 73 | github.com/json-iterator/go v1.1.12 // indirect 74 | github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54 // indirect 75 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 76 | github.com/mailru/easyjson v0.9.0 // indirect 77 | github.com/mattn/go-colorable v0.1.14 // indirect 78 | github.com/mattn/go-isatty v0.0.20 // indirect 79 | github.com/mattn/go-runewidth v0.0.16 // indirect 80 | github.com/microcosm-cc/bluemonday v1.0.27 // indirect 81 | github.com/mitchellh/copystructure v1.2.0 // indirect 82 | github.com/mitchellh/mapstructure v1.5.0 // indirect 83 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 84 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 85 | github.com/modern-go/reflect2 v1.0.2 // indirect 86 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 87 | github.com/mschoch/smat v0.2.0 // indirect 88 | github.com/muesli/reflow v0.3.0 // indirect 89 | github.com/muesli/termenv v0.16.0 // indirect 90 | github.com/oklog/ulid v1.3.1 // indirect 91 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 92 | github.com/richardlehane/mscfb v1.0.4 // indirect 93 | github.com/richardlehane/msoleps v1.0.4 // indirect 94 | github.com/rivo/uniseg v0.4.7 // indirect 95 | github.com/sagikazarmark/locafero v0.7.0 // indirect 96 | github.com/sourcegraph/conc v0.3.0 // indirect 97 | github.com/spf13/afero v1.12.0 // indirect 98 | github.com/spf13/cast v1.7.1 // indirect 99 | github.com/spf13/pflag v1.0.6 // indirect 100 | github.com/subosito/gotenv v1.6.0 // indirect 101 | github.com/tj/go-naturaldate v1.3.0 // indirect 102 | github.com/ugorji/go/codec v1.2.11 // indirect 103 | github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect 104 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect 105 | github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect 106 | github.com/xuri/excelize/v2 v2.9.0 // indirect 107 | github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect 108 | github.com/yuin/goldmark v1.7.8 // indirect 109 | github.com/yuin/goldmark-emoji v1.0.5 // indirect 110 | go.etcd.io/bbolt v1.4.0 // indirect 111 | go.mongodb.org/mongo-driver v1.14.0 // indirect 112 | go.uber.org/multierr v1.11.0 // indirect 113 | golang.org/x/crypto v0.37.0 // indirect 114 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect 115 | golang.org/x/net v0.39.0 // indirect 116 | golang.org/x/sync v0.14.0 // indirect 117 | golang.org/x/sys v0.32.0 // indirect 118 | golang.org/x/term v0.31.0 // indirect 119 | golang.org/x/text v0.24.0 // indirect 120 | google.golang.org/protobuf v1.36.1 // indirect 121 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 122 | gopkg.in/yaml.v2 v2.4.0 // indirect 123 | ) 124 | 125 | //replace github.com/smacker/go-tree-sitter => github.com/wesen/go-tree-sitter v0.0.0-20230423204225-a896a22ee48a 126 | -------------------------------------------------------------------------------- /lefthook.yml: -------------------------------------------------------------------------------- 1 | pre-commit: 2 | commands: 3 | lint: 4 | glob: '*.go' 5 | run: make lint 6 | test: 7 | glob: '*.go' 8 | run: make test 9 | parallel: true 10 | 11 | pre-push: 12 | commands: 13 | # release: 14 | # run: make goreleaser 15 | lint: 16 | run: make lint 17 | test: 18 | run: make test 19 | parallel: true 20 | -------------------------------------------------------------------------------- /pinocchio/oak/create-command.yaml: -------------------------------------------------------------------------------- 1 | name: create-command 2 | short: Generate a Oak query 3 | flags: 4 | - name: tree 5 | type: stringFromFile 6 | help: Example parsed tree 7 | required: false 8 | - name: query 9 | type: stringFromFile 10 | help: Example tree sitter queries 11 | required: false 12 | - name: types 13 | type: stringList 14 | help: List of types 15 | default: 16 | - int 17 | - string 18 | - date 19 | - stringList 20 | - intList 21 | - float 22 | - bool 23 | - floatList 24 | - name: instructions 25 | type: string 26 | help: Additional language specific instructions 27 | required: false 28 | - name: topic 29 | type: string 30 | help: Topic of the query 31 | required: false 32 | - name: instructions_file 33 | type: stringFromFile 34 | help: Additional language specific instructions 35 | required: false 36 | - name: topic_file 37 | type: stringFromFile 38 | help: Topic of the query 39 | required: false 40 | - name: example_name 41 | type: string 42 | help: Name of the example 43 | default: Get all struct and function declaration in a golang file. 44 | - name: example 45 | type: stringFromFile 46 | help: Example of a resulting command 47 | default: | 48 | name: definitions 49 | short: A simple example to extract go functions and structs 50 | 51 | flags: 52 | - name: verbose 53 | type: bool 54 | help: Output all results 55 | default: false 56 | - name: type 57 | type: string 58 | help: Only output structs types matching type 59 | 60 | language: go 61 | queries: 62 | - name: structDeclarations 63 | query: | 64 | ( 65 | (comment)* @comment . 66 | (type_declaration 67 | (type_spec 68 | name: (type_identifier) @structName 69 | type: (struct_type) @structBody)) 70 | {{ if .type }}(#eq? @structName "{{.type}}"){{end}} 71 | ) 72 | 73 | - name: functionDeclarations 74 | query: | 75 | {{ if not .type }} 76 | ((comment)* @comment . 77 | (function_declaration 78 | name: (identifier) @name 79 | parameters: (parameter_list)? @parameters 80 | result: (_)? @result 81 | body: (block))) 82 | {{end}} 83 | 84 | template: | 85 | {{ range $file, $results := .ResultsByFile -}} 86 | File: {{ $file }} 87 | {{ with $results -}} 88 | {{- range .structDeclarations.Matches }} 89 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 90 | type {{.structName.Text}} {{ .structBody.Text | indent 2 }}{{ end }} 91 | {{ range .functionDeclarations.Matches }} 92 | {{ if .comment }}{{ .comment.Text |indentBlock 2}}{{end -}} 93 | func {{ .name.Text }}{{ .parameters.Text }} {{ .result.Text }} {{ end }} 94 | {{ end -}} 95 | {{ end -}} 96 | 97 | {{ if .verbose -}} 98 | Results:{{ range $v := .Results }} 99 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 100 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 101 | {{end}}{{ end }} 102 | {{ end -}} 103 | 104 | prompt: | 105 | I want to generate templates for tree-sitter queries, stored in YAML. 106 | 107 | The queries array represents tree-sitter queries that are used to extract information from 108 | the parsed tree-sitter code. It is provided as go template that interpolate the flags passed to the command. 109 | 110 | The templates expose command line parameters that the user can use to control the query, 111 | and generate useful WHERE and GROUP BY statements. 112 | 113 | The `flags` stored in the YAML can be of different types: {{ .types | join ", " }}. These are then passed to the go 114 | template. 115 | 116 | Here is an example: 117 | 118 | NAME: {{ .example_name }} 119 | 120 | ```yaml 121 | {{ .example }} 122 | ``` 123 | 124 | {{ if .topic_file }}{{ .topic_file }} {{ end }} 125 | {{ if .topic }} {{ .topic }} {{ end }} 126 | {{ if .tree }}Generate a similar template with flags to query the abstract syntax tree described by: 127 | 128 | ```parse-tree 129 | {{ .tree }} 130 | ``` 131 | {{ end }} 132 | 133 | {{ if .query }}Generate a similar template with flags to run the following queries and interpolate their results: 134 | 135 | ```tree-sitter-query 136 | {{ .query }} 137 | ``` 138 | {{ end }} 139 | 140 | {{ if .instructions }} {{ .instructions }} {{ end }} 141 | {{ if .instructions_file }} {{ .instructions_file }} {{ end }} 142 | -------------------------------------------------------------------------------- /pkg/cmds/glazed.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/fs" 7 | "strings" 8 | 9 | "github.com/go-go-golems/oak/pkg" 10 | "gopkg.in/yaml.v3" 11 | 12 | "github.com/go-go-golems/glazed/pkg/cmds" 13 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 14 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 15 | "github.com/go-go-golems/glazed/pkg/cmds/layout" 16 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 17 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 18 | "github.com/go-go-golems/glazed/pkg/middlewares" 19 | "github.com/go-go-golems/glazed/pkg/settings" 20 | "github.com/go-go-golems/glazed/pkg/types" 21 | ) 22 | 23 | type OakGlazeCommand struct { 24 | *OakCommand 25 | } 26 | 27 | var _ cmds.GlazeCommand = (*OakGlazeCommand)(nil) 28 | 29 | type RunSettings struct { 30 | Sources []string `glazed.parameter:"sources"` 31 | } 32 | 33 | func (oc *OakGlazeCommand) RunIntoGlazeProcessor( 34 | ctx context.Context, 35 | parsedLayers *layers.ParsedLayers, 36 | gp middlewares.Processor, 37 | ) error { 38 | s := &RunSettings{} 39 | err := parsedLayers.InitializeStruct(layers.DefaultSlug, s) 40 | if err != nil { 41 | return err 42 | } 43 | ss := &OakSettings{} 44 | err = parsedLayers.InitializeStruct(OakSlug, ss) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | err = oc.RenderQueries(parsedLayers) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | if ss.PrintQueries { 55 | for _, q := range oc.Queries { 56 | v := types.NewRow( 57 | types.MRP("query", q.Query), 58 | types.MRP("name", q.Name), 59 | ) 60 | err := gp.AddRow(ctx, v) 61 | if err != nil { 62 | return err 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | glob_ := ss.Glob 70 | if ss.Recurse && len(glob_) == 0 { 71 | // use standard globs for the language of the command 72 | glob_, err = pkg.GetLanguageGlobs(oc.Language) 73 | if err != nil { 74 | return err 75 | } 76 | } 77 | sources_, err := collectSources(s.Sources, glob_) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | resultsByFile, err := oc.GetResultsByFile(ctx, sources_) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | for fileName, fileResults := range resultsByFile { 88 | for _, result := range fileResults { 89 | for _, match := range result.Matches { 90 | for _, capture := range match { 91 | row := types.NewRow( 92 | types.MRP("file", fileName), 93 | types.MRP("query", result.QueryName), 94 | types.MRP("capture", capture.Name), 95 | 96 | types.MRP("startRow", capture.StartPoint.Row), 97 | types.MRP("startColumn", capture.StartPoint.Column), 98 | types.MRP("endRow", capture.EndPoint.Row), 99 | types.MRP("endColumn", capture.EndPoint.Column), 100 | 101 | types.MRP("startByte", capture.StartByte), 102 | types.MRP("endByte", capture.EndByte), 103 | 104 | types.MRP("type", capture.Type), 105 | types.MRP("text", capture.Text), 106 | ) 107 | err = gp.AddRow(ctx, row) 108 | if err != nil { 109 | return err 110 | } 111 | } 112 | } 113 | } 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func NewOakGlazedCommand(d *cmds.CommandDescription, options ...OakCommandOption) *OakGlazeCommand { 120 | cmd := OakGlazeCommand{ 121 | OakCommand: &OakCommand{ 122 | CommandDescription: d, 123 | }, 124 | } 125 | for _, option := range options { 126 | option(cmd.OakCommand) 127 | } 128 | return &cmd 129 | } 130 | 131 | type OakGlazedCommandLoader struct{} 132 | 133 | func (o *OakGlazedCommandLoader) IsFileSupported(f fs.FS, fileName string) bool { 134 | return strings.HasSuffix(fileName, ".yaml") || strings.HasSuffix(fileName, ".yml") 135 | } 136 | 137 | var _ loaders.CommandLoader = (*OakGlazedCommandLoader)(nil) 138 | 139 | func (o *OakGlazedCommandLoader) LoadCommands( 140 | f fs.FS, entryName string, 141 | options []cmds.CommandDescriptionOption, 142 | aliasOptions []alias.Option, 143 | ) ([]cmds.Command, error) { 144 | r, err := f.Open(entryName) 145 | if err != nil { 146 | return nil, err 147 | } 148 | defer func(r fs.File) { 149 | _ = r.Close() 150 | }(r) 151 | 152 | return loaders.LoadCommandOrAliasFromReader( 153 | r, 154 | o.loadCommandFromReader, 155 | options, 156 | aliasOptions) 157 | } 158 | 159 | func (o *OakGlazedCommandLoader) loadCommandFromReader( 160 | s io.Reader, 161 | options []cmds.CommandDescriptionOption, 162 | _ []alias.Option, 163 | ) ([]cmds.Command, error) { 164 | ocd := &OakCommandDescription{} 165 | err := yaml.NewDecoder(s).Decode(ocd) 166 | if err != nil { 167 | return nil, err 168 | } 169 | 170 | oakLayer, err := NewOakParameterLayer() 171 | if err != nil { 172 | return nil, err 173 | } 174 | 175 | glazeLayer, err := settings.NewGlazedParameterLayers() 176 | if err != nil { 177 | return nil, err 178 | } 179 | layers := append(ocd.Layers, glazeLayer, oakLayer) 180 | 181 | options_ := []cmds.CommandDescriptionOption{ 182 | cmds.WithName(ocd.Name), 183 | cmds.WithShort(ocd.Short), 184 | cmds.WithLong(ocd.Long), 185 | cmds.WithFlags(ocd.Flags...), 186 | cmds.WithLayersList(layers...), 187 | cmds.WithArguments( 188 | parameters.NewParameterDefinition( 189 | "sources", 190 | parameters.ParameterTypeStringList, 191 | parameters.WithHelp("Files (or directories if recursing) to parse"), 192 | parameters.WithRequired(false), 193 | ), 194 | ), 195 | cmds.WithLayout(&layout.Layout{ 196 | Sections: ocd.Layout, 197 | }), 198 | } 199 | options_ = append(options_, options...) 200 | 201 | oakCommand := NewOakGlazedCommand( 202 | cmds.NewCommandDescription(ocd.Name, options_...), 203 | WithQueries(ocd.Queries...), 204 | WithTemplate(ocd.Template), 205 | WithLanguage(ocd.Language), 206 | ) 207 | 208 | return []cmds.Command{oakCommand}, nil 209 | } 210 | 211 | func (o *OakGlazedCommandLoader) LoadCommandAliasFromYAML( 212 | s io.Reader, 213 | options ...alias.Option, 214 | ) ([]*alias.CommandAlias, error) { 215 | return loaders.LoadCommandAliasFromYAML(s, options...) 216 | } 217 | -------------------------------------------------------------------------------- /pkg/cmds/layers/oak.yaml: -------------------------------------------------------------------------------- 1 | slug: oak 2 | name: Oak flags 3 | Description: | 4 | General purpose flags for oak 5 | flags: 6 | - name: recurse 7 | type: bool 8 | help: Recurse into subdirectories 9 | default: false 10 | - name: print-queries 11 | type: bool 12 | help: Print queries 13 | default: false 14 | - name: glob 15 | type: stringList 16 | help: Glob patterns to match files -------------------------------------------------------------------------------- /pkg/cmds/writer.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | _ "embed" 7 | "github.com/go-go-golems/glazed/pkg/cmds" 8 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 9 | "github.com/go-go-golems/glazed/pkg/helpers/templating" 10 | "github.com/go-go-golems/oak/pkg" 11 | tree_sitter "github.com/go-go-golems/oak/pkg/tree-sitter" 12 | "io" 13 | "strings" 14 | ) 15 | 16 | type OakWriterCommand struct { 17 | *OakCommand 18 | } 19 | 20 | var _ cmds.WriterCommand = (*OakWriterCommand)(nil) 21 | 22 | func NewOakWriterCommand(d *cmds.CommandDescription, options ...OakCommandOption) *OakWriterCommand { 23 | cmd := OakWriterCommand{ 24 | OakCommand: &OakCommand{ 25 | CommandDescription: d, 26 | }, 27 | } 28 | for _, option := range options { 29 | option(cmd.OakCommand) 30 | } 31 | return &cmd 32 | } 33 | 34 | func (oc *OakWriterCommand) RunIntoWriter( 35 | ctx context.Context, 36 | parsedLayers *layers.ParsedLayers, 37 | w io.Writer, 38 | ) error { 39 | s := &RunSettings{} 40 | err := parsedLayers.InitializeStruct(layers.DefaultSlug, s) 41 | if err != nil { 42 | return err 43 | } 44 | ss := &OakSettings{} 45 | err = parsedLayers.InitializeStruct(OakSlug, ss) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | err = oc.RenderQueries(parsedLayers) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | if ss.PrintQueries { 56 | err := oc.PrintQueries(w) 57 | if err != nil { 58 | return err 59 | } 60 | return nil 61 | } 62 | 63 | sources_ := s.Sources 64 | 65 | glob_ := ss.Glob 66 | 67 | if ss.Recurse && len(glob_) == 0 { 68 | // use standard globs for the language of the command 69 | glob_, err = pkg.GetLanguageGlobs(oc.Language) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | sources_, err = collectSources(sources_, glob_) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | resultsByFile, err := oc.GetResultsByFile(ctx, sources_) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | tmpl, err := templating.CreateTemplate("oak").Parse(oc.Template) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | allResults := tree_sitter.QueryResults{} 90 | 91 | for _, fileResults := range resultsByFile { 92 | for k, v := range fileResults { 93 | result, ok := allResults[k] 94 | if !ok { 95 | // store copy of v in allResults 96 | allResults[k] = v.Clone() 97 | continue 98 | } 99 | result.Matches = append(result.Matches, v.Matches...) 100 | } 101 | } 102 | 103 | data := parsedLayers.GetDataMap() 104 | data["ResultsByFile"] = resultsByFile 105 | data["Results"] = allResults 106 | 107 | var buf bytes.Buffer 108 | err = tmpl.Execute(&buf, data) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | s_ := buf.String() 114 | // trim left and right 115 | s_ = strings.TrimSpace(s_) + "\n" 116 | 117 | _, err = w.Write(([]byte)(s_)) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /pkg/doc/doc.go: -------------------------------------------------------------------------------- 1 | package doc 2 | 3 | import ( 4 | "embed" 5 | "github.com/go-go-golems/glazed/pkg/help" 6 | ) 7 | 8 | //go:embed * 9 | var docFS embed.FS 10 | 11 | func AddDocToHelpSystem(helpSystem *help.HelpSystem) error { 12 | return helpSystem.LoadSectionsFromFS(docFS, ".") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/lang.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pkg/errors" 6 | "github.com/smacker/go-tree-sitter" 7 | "github.com/smacker/go-tree-sitter/bash" 8 | "github.com/smacker/go-tree-sitter/c" 9 | "github.com/smacker/go-tree-sitter/cpp" 10 | "github.com/smacker/go-tree-sitter/csharp" 11 | "github.com/smacker/go-tree-sitter/css" 12 | "github.com/smacker/go-tree-sitter/cue" 13 | "github.com/smacker/go-tree-sitter/dockerfile" 14 | "github.com/smacker/go-tree-sitter/elixir" 15 | "github.com/smacker/go-tree-sitter/elm" 16 | "github.com/smacker/go-tree-sitter/golang" 17 | "github.com/smacker/go-tree-sitter/hcl" 18 | "github.com/smacker/go-tree-sitter/html" 19 | "github.com/smacker/go-tree-sitter/java" 20 | "github.com/smacker/go-tree-sitter/javascript" 21 | "github.com/smacker/go-tree-sitter/kotlin" 22 | //"github.com/smacker/go-tree-sitter/lua" 23 | "github.com/smacker/go-tree-sitter/ocaml" 24 | "github.com/smacker/go-tree-sitter/php" 25 | "github.com/smacker/go-tree-sitter/protobuf" 26 | "github.com/smacker/go-tree-sitter/python" 27 | "github.com/smacker/go-tree-sitter/ruby" 28 | "github.com/smacker/go-tree-sitter/rust" 29 | "github.com/smacker/go-tree-sitter/scala" 30 | "github.com/smacker/go-tree-sitter/svelte" 31 | "github.com/smacker/go-tree-sitter/toml" 32 | "github.com/smacker/go-tree-sitter/typescript/tsx" 33 | "github.com/smacker/go-tree-sitter/typescript/typescript" 34 | "github.com/smacker/go-tree-sitter/yaml" 35 | "path" 36 | ) 37 | 38 | func LanguageNameToSitterLanguage(name string) (*sitter.Language, error) { 39 | switch name { 40 | case "bash": 41 | return bash.GetLanguage(), nil 42 | case "c": 43 | return c.GetLanguage(), nil 44 | case "cpp": 45 | return cpp.GetLanguage(), nil 46 | case "csharp": 47 | return csharp.GetLanguage(), nil 48 | case "css": 49 | return css.GetLanguage(), nil 50 | case "cue": 51 | return cue.GetLanguage(), nil 52 | case "dockerfile": 53 | return dockerfile.GetLanguage(), nil 54 | case "elixir": 55 | return elixir.GetLanguage(), nil 56 | case "elm": 57 | return elm.GetLanguage(), nil 58 | case "golang": 59 | fallthrough 60 | case "go": 61 | return golang.GetLanguage(), nil 62 | case "hcl": 63 | return hcl.GetLanguage(), nil 64 | case "html": 65 | return html.GetLanguage(), nil 66 | case "java": 67 | return java.GetLanguage(), nil 68 | case "javascript": 69 | return javascript.GetLanguage(), nil 70 | case "kotlin": 71 | return kotlin.GetLanguage(), nil 72 | //case "lua": 73 | // return lua.GetLanguage(), nil 74 | case "ocaml": 75 | return ocaml.GetLanguage(), nil 76 | case "php": 77 | return php.GetLanguage(), nil 78 | case "protobuf": 79 | return protobuf.GetLanguage(), nil 80 | case "python": 81 | return python.GetLanguage(), nil 82 | case "ruby": 83 | return ruby.GetLanguage(), nil 84 | case "rust": 85 | return rust.GetLanguage(), nil 86 | case "scala": 87 | return scala.GetLanguage(), nil 88 | case "svelte": 89 | return svelte.GetLanguage(), nil 90 | case "toml": 91 | return toml.GetLanguage(), nil 92 | case "typescript": 93 | return typescript.GetLanguage(), nil 94 | case "tsx": 95 | return tsx.GetLanguage(), nil 96 | case "yaml": 97 | return yaml.GetLanguage(), nil 98 | default: 99 | return nil, errors.Errorf("unsupported language name: %s", name) 100 | } 101 | } 102 | 103 | var ( 104 | fileEndingToLanguageName = map[string][]string{ 105 | "*.sh": {"bash"}, 106 | "*.c": {"c"}, 107 | "*.cpp": {"cpp"}, 108 | "*.h": {"cpp"}, 109 | "*.hpp": {"cpp"}, 110 | "*.cs": {"csharp"}, 111 | "*.css": {"css"}, 112 | "*.cue": {"cue"}, 113 | "Dockerfile": {"dockerfile"}, 114 | "*.ex": {"elixir"}, 115 | "*.elm": {"elm"}, 116 | "*.go": {"go"}, 117 | "*.hcl": {"hcl"}, 118 | "*.tf": {"hcl"}, 119 | "*.html": {"html"}, 120 | "*.java": {"java"}, 121 | "*.js": {"javascript"}, 122 | "*.jsx": {"javascript"}, 123 | "*.kt": {"kotlin"}, 124 | //"*.lua": []string{"lua"}, 125 | "*.ml": {"ocaml"}, 126 | "*.mli": {"ocaml"}, 127 | "*.php": {"php"}, 128 | "*.proto": {"protobuf"}, 129 | "*.py": {"python"}, 130 | "*.rb": {"ruby"}, 131 | "*.rs": {"rust"}, 132 | "*.scala": {"scala"}, 133 | "*.svelte": {"svelte"}, 134 | "*.toml": {"toml"}, 135 | "*.ts": {"typescript", "tsx"}, 136 | "*.tsx": {"tsx"}, 137 | "*.yml": {"yaml"}, 138 | "*.yaml": {"yaml"}, 139 | } 140 | ) 141 | 142 | func GetLanguageGlobs(lang string) ([]string, error) { 143 | ret := []string{} 144 | 145 | for filename, lang_ := range fileEndingToLanguageName { 146 | for _, lang__ := range lang_ { 147 | if lang__ == lang { 148 | ret = append(ret, fmt.Sprintf("**/%s", filename)) 149 | } 150 | } 151 | } 152 | 153 | if len(ret) == 0 { 154 | return nil, errors.Errorf("unsupported language name: %s", lang) 155 | } else { 156 | return ret, nil 157 | } 158 | } 159 | 160 | func FileNameToSitterLanguage(filename string) (*sitter.Language, error) { 161 | baseName := path.Base(filename) 162 | for ending, name := range fileEndingToLanguageName { 163 | // use glob 164 | matched, err := path.Match(ending, baseName) 165 | if err != nil { 166 | return nil, err 167 | } 168 | if matched { 169 | return LanguageNameToSitterLanguage(name[0]) 170 | } 171 | } 172 | return nil, errors.Errorf("unsupported file name: %s", filename) 173 | } 174 | -------------------------------------------------------------------------------- /pkg/queries.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump.go: -------------------------------------------------------------------------------- 1 | package tree_sitter 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/go-go-golems/oak/pkg/tree-sitter/dump" 7 | sitter "github.com/smacker/go-tree-sitter" 8 | ) 9 | 10 | // DumpFormat represents a tree output format type 11 | type DumpFormat = dump.Format 12 | 13 | // DumpOptions contains settings for tree dumping 14 | type DumpOptions = dump.Options 15 | 16 | // Dumper is the interface that all tree dumpers must implement 17 | type Dumper = dump.Dumper 18 | 19 | // Format constants 20 | const ( 21 | FormatText DumpFormat = dump.FormatText 22 | FormatXML DumpFormat = dump.FormatXML 23 | FormatJSON DumpFormat = dump.FormatJSON 24 | FormatYAML DumpFormat = dump.FormatYAML 25 | ) 26 | 27 | // NewDumper creates a new tree dumper for the specified format 28 | func NewDumper(format DumpFormat) Dumper { 29 | return dump.NewDumper(format) 30 | } 31 | 32 | // DumpTree dumps a tree to the specified writer using the given format and options 33 | func DumpTree(tree *sitter.Tree, source []byte, w io.Writer, format DumpFormat, options DumpOptions) error { 34 | dumper := NewDumper(format) 35 | return dumper.Dump(tree, source, w, options) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump/dump.go: -------------------------------------------------------------------------------- 1 | package dump 2 | 3 | import ( 4 | "io" 5 | 6 | sitter "github.com/smacker/go-tree-sitter" 7 | ) 8 | 9 | // Format represents a tree output format type 10 | type Format string 11 | 12 | const ( 13 | // FormatText is the enhanced text output format 14 | FormatText Format = "text" 15 | // FormatXML is the XML output format 16 | FormatXML Format = "xml" 17 | // FormatJSON is the JSON output format 18 | FormatJSON Format = "json" 19 | // FormatYAML is the YAML output format 20 | FormatYAML Format = "yaml" 21 | ) 22 | 23 | // Options contains settings for tree dumping 24 | type Options struct { 25 | ShowBytes bool 26 | ShowContent bool 27 | ShowAttributes bool 28 | SkipWhitespace bool 29 | } 30 | 31 | // Dumper is the interface that all tree dumpers must implement 32 | type Dumper interface { 33 | Dump(tree *sitter.Tree, source []byte, w io.Writer, options Options) error 34 | } 35 | 36 | // NewDumper creates a new tree dumper for the specified format 37 | func NewDumper(format Format) Dumper { 38 | switch format { 39 | case FormatText: 40 | return &TextDumper{} 41 | case FormatXML: 42 | return &XMLDumper{} 43 | case FormatJSON: 44 | return &JSONDumper{} 45 | case FormatYAML: 46 | return &YAMLDumper{} 47 | default: 48 | return &TextDumper{} // Default to text format 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump/json.go: -------------------------------------------------------------------------------- 1 | package dump 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "regexp" 8 | 9 | "github.com/pkg/errors" 10 | sitter "github.com/smacker/go-tree-sitter" 11 | ) 12 | 13 | // JSONDumper implements the JSON format dumper 14 | type JSONDumper struct{} 15 | 16 | // NodeJSON represents a tree node in JSON format 17 | type NodeJSON struct { 18 | Type string `json:"type"` 19 | Position string `json:"pos"` // Format: "startLine,startCol-endLine,endCol" 20 | Bytes string `json:"bytes,omitempty"` // Optional 21 | IsNamed bool `json:"is_named,omitempty"` 22 | IsMissing bool `json:"is_missing,omitempty"` 23 | IsExtra bool `json:"is_extra,omitempty"` 24 | HasError bool `json:"has_error,omitempty"` 25 | Content string `json:"content,omitempty"` 26 | Fields map[string][]*NodeJSON `json:"fields,omitempty"` 27 | Children []*NodeJSON `json:"children,omitempty"` 28 | } 29 | 30 | // Dump outputs the tree in JSON format 31 | func (d *JSONDumper) Dump(tree *sitter.Tree, source []byte, w io.Writer, options Options) error { 32 | var buildJSON func(n *sitter.Node) *NodeJSON 33 | buildJSON = func(n *sitter.Node) *NodeJSON { 34 | if n.IsNull() { 35 | return nil 36 | } 37 | 38 | nodeType := n.Type() 39 | // Skip pure whitespace nodes 40 | if options.SkipWhitespace { 41 | if matched, _ := regexp.MatchString(`^\s+$`, nodeType); matched { 42 | return nil 43 | } 44 | } 45 | 46 | // Convert to 1-based line/column numbers for better readability 47 | startPoint := n.StartPoint() 48 | endPoint := n.EndPoint() 49 | startLine := startPoint.Row + 1 50 | startCol := startPoint.Column + 1 51 | endLine := endPoint.Row + 1 52 | endCol := endPoint.Column + 1 53 | 54 | // Format position as "startLine,startCol-endLine,endCol" 55 | position := fmt.Sprintf("%d,%d-%d,%d", startLine, startCol, endLine, endCol) 56 | 57 | // Create the JSON node 58 | node := &NodeJSON{ 59 | Type: nodeType, 60 | Position: position, 61 | } 62 | 63 | // Add byte range if requested 64 | if options.ShowBytes { 65 | node.Bytes = fmt.Sprintf("%d-%d", n.StartByte(), n.EndByte()) 66 | } 67 | 68 | // Only include these fields if true and if ShowAttributes is enabled 69 | if options.ShowAttributes { 70 | if n.IsNamed() { 71 | node.IsNamed = true 72 | } 73 | if n.IsMissing() { 74 | node.IsMissing = true 75 | } 76 | if n.IsExtra() { 77 | node.IsExtra = true 78 | } 79 | if n.HasError() { 80 | node.HasError = true 81 | } 82 | } 83 | 84 | // Add content if source is provided and ShowContent is enabled 85 | if source != nil && options.ShowContent { 86 | content := n.Content(source) 87 | // For very long content, truncate 88 | if len(content) > 60 { 89 | content = content[:57] + "..." 90 | } 91 | node.Content = content 92 | } 93 | 94 | // Process children 95 | fieldMap := make(map[string][]*NodeJSON) 96 | var plainChildren []*NodeJSON 97 | 98 | for i := 0; i < int(n.ChildCount()); i++ { 99 | fieldName := n.FieldNameForChild(i) 100 | child := buildJSON(n.Child(i)) 101 | 102 | if child == nil { 103 | continue 104 | } 105 | 106 | if fieldName != "" { 107 | if fieldMap[fieldName] == nil { 108 | fieldMap[fieldName] = []*NodeJSON{} 109 | } 110 | fieldMap[fieldName] = append(fieldMap[fieldName], child) 111 | } else { 112 | plainChildren = append(plainChildren, child) 113 | } 114 | } 115 | 116 | if len(fieldMap) > 0 { 117 | node.Fields = fieldMap 118 | } 119 | 120 | if len(plainChildren) > 0 { 121 | node.Children = plainChildren 122 | } 123 | 124 | return node 125 | } 126 | 127 | rootJSON := buildJSON(tree.RootNode()) 128 | if rootJSON == nil { 129 | return errors.New("failed to build JSON representation of tree") 130 | } 131 | 132 | enc := json.NewEncoder(w) 133 | enc.SetIndent("", " ") 134 | return enc.Encode(rootJSON) 135 | } 136 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump/text.go: -------------------------------------------------------------------------------- 1 | package dump 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "regexp" 7 | "strings" 8 | 9 | sitter "github.com/smacker/go-tree-sitter" 10 | ) 11 | 12 | // TextDumper implements the enhanced text format dumper 13 | type TextDumper struct{} 14 | 15 | // Dump outputs the tree in enhanced text format 16 | func (d *TextDumper) Dump(tree *sitter.Tree, source []byte, w io.Writer, options Options) error { 17 | var visitEnhanced func(n *sitter.Node, name string, depth int) error 18 | visitEnhanced = func(n *sitter.Node, name string, depth int) error { 19 | if n.IsNull() { 20 | return nil 21 | } 22 | 23 | nodeType := n.Type() 24 | // Skip whitespace nodes if configured 25 | if options.SkipWhitespace { 26 | if matched, _ := regexp.MatchString(`^\s+$`, nodeType); matched { 27 | return nil 28 | } 29 | } 30 | 31 | indent := strings.Repeat(" ", depth) 32 | prefix := "" 33 | if name != "" { 34 | prefix = name + ": " 35 | } 36 | 37 | // Convert to 1-based line/column numbers for better readability 38 | startPoint := n.StartPoint() 39 | endPoint := n.EndPoint() 40 | startLine := startPoint.Row + 1 41 | startCol := startPoint.Column + 1 42 | endLine := endPoint.Row + 1 43 | endCol := endPoint.Column + 1 44 | 45 | // Format position as "startLine,startCol-endLine,endCol" 46 | position := fmt.Sprintf("%d,%d-%d,%d", startLine, startCol, endLine, endCol) 47 | 48 | // Basic node info with position 49 | fmt.Fprintf(w, "%s%s%s [%s]", indent, prefix, nodeType, position) 50 | 51 | // Add byte offsets if requested 52 | if options.ShowBytes { 53 | fmt.Fprintf(w, " bytes:%d-%d", n.StartByte(), n.EndByte()) 54 | } 55 | 56 | // Additional attributes if requested 57 | if options.ShowAttributes { 58 | flags := []string{} 59 | if n.IsNamed() { 60 | flags = append(flags, "named") 61 | } 62 | if n.IsMissing() { 63 | flags = append(flags, "missing") 64 | } 65 | if n.IsExtra() { 66 | flags = append(flags, "extra") 67 | } 68 | if n.HasError() { 69 | flags = append(flags, "error") 70 | } 71 | 72 | if len(flags) > 0 { 73 | fmt.Fprintf(w, " [%s]", strings.Join(flags, ",")) 74 | } 75 | } 76 | 77 | // Content if source is provided and requested 78 | if source != nil && options.ShowContent { 79 | content := n.Content(source) 80 | // Escape newlines and other control characters for display 81 | content = strings.ReplaceAll(content, "\n", "\\n") 82 | content = strings.ReplaceAll(content, "\t", "\\t") 83 | content = strings.ReplaceAll(content, "\r", "\\r") 84 | 85 | // Truncate very long content 86 | if len(content) > 60 { 87 | content = content[:57] + "..." 88 | } 89 | 90 | fmt.Fprintf(w, " \"%s\"", content) 91 | } 92 | 93 | fmt.Fprintln(w) 94 | 95 | // Visit children 96 | for i := 0; i < int(n.ChildCount()); i++ { 97 | fieldName := n.FieldNameForChild(i) 98 | err := visitEnhanced(n.Child(i), fieldName, depth+1) 99 | if err != nil { 100 | return err 101 | } 102 | } 103 | 104 | return nil 105 | } 106 | 107 | return visitEnhanced(tree.RootNode(), "", 0) 108 | } 109 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump/xml.go: -------------------------------------------------------------------------------- 1 | package dump 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "regexp" 7 | "strings" 8 | 9 | sitter "github.com/smacker/go-tree-sitter" 10 | ) 11 | 12 | // XMLDumper implements the XML format dumper 13 | type XMLDumper struct{} 14 | 15 | // Dump outputs the tree in XML format 16 | func (d *XMLDumper) Dump(tree *sitter.Tree, source []byte, w io.Writer, options Options) error { 17 | // Write XML header 18 | fmt.Fprintf(w, "\n") 19 | fmt.Fprintf(w, "\n") 20 | 21 | var visitXML func(n *sitter.Node, depth int) error 22 | visitXML = func(n *sitter.Node, depth int) error { 23 | if n.IsNull() { 24 | return nil 25 | } 26 | 27 | indent := strings.Repeat(" ", depth) 28 | nodeType := n.Type() 29 | 30 | // Skip pure whitespace nodes 31 | if options.SkipWhitespace { 32 | if matched, _ := regexp.MatchString(`^\s+$`, nodeType); matched { 33 | return nil 34 | } 35 | } 36 | 37 | // Get node text content if source is provided 38 | var contentAttr string 39 | if source != nil && options.ShowContent { 40 | content := n.Content(source) 41 | // XML-escape the content 42 | content = strings.ReplaceAll(content, "&", "&") 43 | content = strings.ReplaceAll(content, "<", "<") 44 | content = strings.ReplaceAll(content, ">", ">") 45 | content = strings.ReplaceAll(content, "\"", """) 46 | contentAttr = fmt.Sprintf(" content=\"%s\"", content) 47 | } 48 | 49 | // Convert to 1-based line/column numbers for better readability 50 | startPoint := n.StartPoint() 51 | endPoint := n.EndPoint() 52 | startLine := startPoint.Row + 1 53 | startCol := startPoint.Column + 1 54 | endLine := endPoint.Row + 1 55 | endCol := endPoint.Column + 1 56 | 57 | // Format position as "startLine,startCol-endLine,endCol" 58 | posAttr := fmt.Sprintf(" pos=\"%d,%d-%d,%d\"", startLine, startCol, endLine, endCol) 59 | 60 | // Optionally include byte positions 61 | bytesAttr := "" 62 | if options.ShowBytes { 63 | bytesAttr = fmt.Sprintf(" bytes=\"%d-%d\"", n.StartByte(), n.EndByte()) 64 | } 65 | 66 | attributesStr := "" 67 | if options.ShowAttributes { 68 | attributesStr = fmt.Sprintf(" named=\"%t\" missing=\"%t\" extra=\"%t\" has_error=\"%t\"", 69 | n.IsNamed(), 70 | n.IsMissing(), 71 | n.IsExtra(), 72 | n.HasError()) 73 | } 74 | 75 | // Output opening tag with attributes 76 | fmt.Fprintf(w, "%s\n", 77 | indent, 78 | nodeType, 79 | posAttr, 80 | bytesAttr, 81 | attributesStr, 82 | contentAttr, 83 | ) 84 | 85 | // Visit children 86 | for i := 0; i < int(n.ChildCount()); i++ { 87 | fieldName := n.FieldNameForChild(i) 88 | child := n.Child(i) 89 | 90 | if fieldName != "" { 91 | fmt.Fprintf(w, "%s \n", indent, fieldName) 92 | err := visitXML(child, depth+2) 93 | if err != nil { 94 | return err 95 | } 96 | fmt.Fprintf(w, "%s \n", indent) 97 | } else { 98 | err := visitXML(child, depth+1) 99 | if err != nil { 100 | return err 101 | } 102 | } 103 | } 104 | 105 | // Output closing tag 106 | fmt.Fprintf(w, "%s\n", indent) 107 | return nil 108 | } 109 | 110 | err := visitXML(tree.RootNode(), 1) 111 | if err != nil { 112 | return err 113 | } 114 | 115 | fmt.Fprintf(w, "\n") 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /pkg/tree-sitter/dump/yaml.go: -------------------------------------------------------------------------------- 1 | package dump 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/rs/zerolog/log" 11 | sitter "github.com/smacker/go-tree-sitter" 12 | "gopkg.in/yaml.v3" 13 | ) 14 | 15 | // YAMLDumper implements the YAML format dumper 16 | type YAMLDumper struct{} 17 | 18 | // NodeYAML represents a tree node in YAML format 19 | type NodeYAML struct { 20 | Type string `yaml:"type"` 21 | Position string `yaml:"pos"` // Format: "startLine,startCol-endLine,endCol" 22 | Bytes string `yaml:"bytes,omitempty"` // Format: "startByte-endByte" (optional) 23 | Named bool `yaml:"named,omitempty"` 24 | Missing bool `yaml:"missing,omitempty"` 25 | Extra bool `yaml:"extra,omitempty"` 26 | HasError bool `yaml:"has_error,omitempty"` 27 | Content string `yaml:"content,omitempty"` 28 | Fields map[string][]*NodeYAML `yaml:"fields,omitempty"` 29 | Children []*NodeYAML `yaml:"children,omitempty"` 30 | } 31 | 32 | // Dump outputs the tree in YAML format 33 | func (d *YAMLDumper) Dump(tree *sitter.Tree, source []byte, w io.Writer, options Options) error { 34 | var buildYAML func(n *sitter.Node) *NodeYAML 35 | buildYAML = func(n *sitter.Node) *NodeYAML { 36 | if n.IsNull() { 37 | return nil 38 | } 39 | 40 | nodeType := n.Type() 41 | // Skip pure whitespace nodes 42 | if options.SkipWhitespace { 43 | if matched, _ := regexp.MatchString(`^\s+$`, nodeType); matched { 44 | return nil 45 | } 46 | } 47 | 48 | // Create the YAML node using compact line/column position format 49 | startPoint := n.StartPoint() 50 | endPoint := n.EndPoint() 51 | 52 | // Convert to 1-based line/column numbers for better readability 53 | startLine := startPoint.Row + 1 54 | startCol := startPoint.Column + 1 55 | endLine := endPoint.Row + 1 56 | endCol := endPoint.Column + 1 57 | 58 | // Format: "startLine,startCol-endLine,endCol" 59 | position := fmt.Sprintf("%d,%d-%d,%d", startLine, startCol, endLine, endCol) 60 | 61 | node := &NodeYAML{ 62 | Type: nodeType, 63 | Position: position, 64 | } 65 | 66 | // Add byte range if requested 67 | if options.ShowBytes { 68 | node.Bytes = fmt.Sprintf("%d-%d", n.StartByte(), n.EndByte()) 69 | } 70 | 71 | // Only include these fields if true and ShowAttributes is enabled 72 | if options.ShowAttributes { 73 | if n.IsNamed() { 74 | node.Named = true 75 | } 76 | if n.IsMissing() { 77 | node.Missing = true 78 | } 79 | if n.IsExtra() { 80 | node.Extra = true 81 | } 82 | if n.HasError() { 83 | node.HasError = true 84 | } 85 | } 86 | 87 | // Add content if source is provided and ShowContent is enabled 88 | if source != nil && options.ShowContent { 89 | content := n.Content(source) 90 | // Clean up content for YAML 91 | content = strings.ReplaceAll(content, "\n", "\\n") 92 | content = strings.ReplaceAll(content, "\t", "\\t") 93 | content = strings.ReplaceAll(content, "\r", "\\r") 94 | 95 | // Truncate very long content 96 | if len(content) > 60 { 97 | content = content[:57] + "..." 98 | } 99 | 100 | node.Content = content 101 | } 102 | 103 | // Process children 104 | fieldMap := make(map[string][]*NodeYAML) 105 | var plainChildren []*NodeYAML 106 | 107 | for i := 0; i < int(n.ChildCount()); i++ { 108 | fieldName := n.FieldNameForChild(i) 109 | child := buildYAML(n.Child(i)) 110 | 111 | if child == nil { 112 | continue 113 | } 114 | 115 | if fieldName != "" { 116 | if fieldMap[fieldName] == nil { 117 | fieldMap[fieldName] = []*NodeYAML{} 118 | } 119 | fieldMap[fieldName] = append(fieldMap[fieldName], child) 120 | } else { 121 | plainChildren = append(plainChildren, child) 122 | } 123 | } 124 | 125 | if len(fieldMap) > 0 { 126 | node.Fields = fieldMap 127 | } 128 | 129 | if len(plainChildren) > 0 { 130 | node.Children = plainChildren 131 | } 132 | 133 | return node 134 | } 135 | 136 | rootYAML := buildYAML(tree.RootNode()) 137 | if rootYAML == nil { 138 | return errors.New("failed to build YAML representation of tree") 139 | } 140 | 141 | enc := yaml.NewEncoder(w) 142 | defer func() { 143 | if err := enc.Close(); err != nil { 144 | log.Warn().Err(err).Msg("error closing yaml encoder") 145 | } 146 | }() 147 | return enc.Encode(rootYAML) 148 | } 149 | -------------------------------------------------------------------------------- /pkg/tree-sitter/tree-sitter.go: -------------------------------------------------------------------------------- 1 | package tree_sitter 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | sitter "github.com/smacker/go-tree-sitter" 6 | ) 7 | 8 | type Capture struct { 9 | // Name if the capture name from the query 10 | Name string 11 | // Text is the actual text that was captured 12 | Text string 13 | // Type is the Treesitter type of the captured node 14 | Type string 15 | 16 | // TODO(manuel, 2023-04-23): Add more information about the capture 17 | // for example: offset, line number, filename, query name, ... 18 | StartByte uint32 19 | EndByte uint32 20 | StartPoint sitter.Point 21 | EndPoint sitter.Point 22 | } 23 | 24 | type Match map[string]Capture 25 | 26 | type Result struct { 27 | QueryName string 28 | Matches []Match 29 | } 30 | 31 | func (r *Result) Clone() *Result { 32 | clone := &Result{ 33 | QueryName: r.QueryName, 34 | Matches: make([]Match, len(r.Matches)), 35 | } 36 | copy(clone.Matches, r.Matches) 37 | return clone 38 | } 39 | 40 | type SitterQuery struct { 41 | // Name of the resulting variable after parsing 42 | Name string `yaml:"name"` 43 | // Query contains the tree-sitter query that will be applied to the source code 44 | Query string `yaml:"query"` 45 | // Rendered keeps track if the Query was Rendered with RenderQueries. 46 | // This is an ugly way of doing things, but at least we'll signal at runtime 47 | // if the code tries to render a query multiple times. 48 | // See the NOTEs in RenderQueries. 49 | Rendered bool 50 | } 51 | 52 | type QueryResults map[string]*Result 53 | 54 | // ExecuteQueries runs the given queries on the given tree and returns the 55 | // results. Individual names are resolved using the sourceCode string, so as 56 | // to provide full identifier names when matched. 57 | // 58 | // TODO(manuel, 2023-06-19) We only need the language from oc here, right? 59 | func ExecuteQueries( 60 | lang *sitter.Language, 61 | tree *sitter.Node, 62 | queries []SitterQuery, 63 | sourceCode []byte, 64 | ) (QueryResults, error) { 65 | results := make(map[string]*Result) 66 | for _, query := range queries { 67 | matches := []Match{} 68 | 69 | // could parse queries up front and return an error if necessary 70 | q, err := sitter.NewQuery([]byte(query.Query), lang) 71 | if err != nil { 72 | switch err := err.(type) { 73 | case *sitter.QueryError: 74 | return nil, errors.Wrap(err, "error parsing query %s") 75 | } 76 | return nil, err 77 | } 78 | qc := sitter.NewQueryCursor() 79 | qc.Exec(q, tree) 80 | for { 81 | m, ok := qc.NextMatch() 82 | if !ok { 83 | break 84 | } 85 | if len(m.Captures) == 0 { 86 | continue 87 | } 88 | 89 | // for debugging purposes 90 | match := Match{} 91 | for _, c := range m.Captures { 92 | name := q.CaptureNameForId(c.Index) 93 | match[name] = Capture{ 94 | Name: name, 95 | Text: c.Node.Content(sourceCode), 96 | StartByte: c.Node.StartByte(), 97 | EndByte: c.Node.EndByte(), 98 | StartPoint: c.Node.StartPoint(), 99 | EndPoint: c.Node.EndPoint(), 100 | } 101 | } 102 | 103 | m = qc.FilterPredicates(m, sourceCode) 104 | 105 | if len(m.Captures) == 0 { 106 | continue 107 | } 108 | 109 | match = Match{} 110 | for _, c := range m.Captures { 111 | name := q.CaptureNameForId(c.Index) 112 | content := string(sourceCode[c.Node.StartByte():c.Node.EndByte()]) 113 | if m, ok := match[name]; ok { 114 | match[name] = Capture{ 115 | Name: name, 116 | Text: m.Text + "\n" + content, 117 | StartByte: m.StartByte, 118 | EndByte: c.Node.EndByte(), 119 | StartPoint: m.StartPoint, 120 | EndPoint: c.Node.EndPoint(), 121 | } 122 | continue 123 | } 124 | match[name] = Capture{ 125 | Name: name, 126 | Text: content, 127 | Type: c.Node.Type(), 128 | StartByte: c.Node.StartByte(), 129 | EndByte: c.Node.EndByte(), 130 | StartPoint: c.Node.StartPoint(), 131 | EndPoint: c.Node.EndPoint(), 132 | } 133 | } 134 | matches = append(matches, match) 135 | } 136 | 137 | results[query.Name] = &Result{ 138 | QueryName: query.Name, 139 | Matches: matches, 140 | } 141 | } 142 | 143 | return results, nil 144 | } 145 | -------------------------------------------------------------------------------- /prompto/oak/go-definitions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | oak go definitions $@ -------------------------------------------------------------------------------- /prompto/typescript/definitions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Script to call oak with typescript definitions/types and optional --with-body flag 4 | 5 | usage() { 6 | echo "Usage: $0 [--with-body] [file2] ..." 7 | echo 8 | echo "Options:" 9 | echo " --with-body Include body in the typescript definitions." 10 | echo " --with-comments Include comments in the typescript definitions." 11 | echo " --with-private Include private definitions" 12 | echo 13 | echo "Arguments:" 14 | echo " file1, file2, ... One or more filenames to process with oak." 15 | echo 16 | echo "Example:" 17 | echo " $0 --with-body file1.ts file2.ts" 18 | echo " $0 file1.ts file2.ts" 19 | } 20 | 21 | # Check if no arguments were provided 22 | if [ $# -eq 0 ]; then 23 | usage 24 | exit 1 25 | fi 26 | 27 | # Initialize an empty array to store filenames 28 | filenames=() 29 | 30 | # Initialize a variable to store the flag status 31 | with_body=false 32 | with_comments=false 33 | with_private=false 34 | 35 | for arg in "$@" 36 | do 37 | if [[ $arg == "--with-body" ]]; then 38 | with_body=true 39 | elif [[ $arg == "--with-comments" ]]; then 40 | with_comments=true 41 | elif [[ $arg == "--with-private" ]]; then 42 | with_private=true 43 | elif [[ $arg == "-h" ]] || [[ $arg == "--help" ]]; then 44 | usage 45 | exit 0 46 | else 47 | filenames+=("$arg") 48 | fi 49 | done 50 | 51 | # Check if filenames are provided 52 | if [ ${#filenames[@]} -eq 0 ]; then 53 | echo "Error: No filenames provided." 54 | usage 55 | exit 1 56 | fi 57 | 58 | echo "Types and definitions in ${filenames[@]}" 59 | 60 | oak typescript types "${filenames[@]}" 61 | 62 | echo 63 | echo 64 | 65 | # make a list of flags to pass typescript definitions based on with_body and with_comments 66 | flags=() 67 | if [ "$with_body" = true ]; then 68 | flags+=("--with-body") 69 | fi 70 | if [ "$with_comments" = true ]; then 71 | flags+=("--with-comments") 72 | fi 73 | if [ "$with_private" = true ]; then 74 | flags+=("--with-private") 75 | fi 76 | 77 | echo oak typescript definitions "${flags[@]}" "${filenames[@]}" 78 | -------------------------------------------------------------------------------- /queries/create-description.yaml: -------------------------------------------------------------------------------- 1 | language: javascript 2 | 3 | queries: 4 | - name: shoulds 5 | query: | 6 | (call_expression 7 | function: (member_expression 8 | object: (member_expression 9 | property: (property_identifier) @object_property) 10 | ) 11 | arguments: (_) @arguments) @call 12 | 13 | (#eq? @object_property "should") 14 | 15 | template: | 16 | Write a concise unit test description that can be used as first argument to 17 | `describe()` or `it()`, and that matches the following should statements: 18 | {{ range .shoulds.Matches }} 19 | - {{ .call.Text }}{{ end }} 20 | 21 | Output a single sentence starting with a verb. -------------------------------------------------------------------------------- /queries/describe.txt: -------------------------------------------------------------------------------- 1 | ((call_expression 2 | function: (identifier) @function_identifier 3 | arguments: (arguments . (_) @first_argument)) 4 | 5 | (#eq? @function_identifier "describe")) 6 | -------------------------------------------------------------------------------- /queries/descriptions.txt: -------------------------------------------------------------------------------- 1 | (call_expression 2 | function: (identifier) @function_identifier 3 | arguments: (arguments (_) @first_argument)) 4 | 5 | (or 6 | (eq? @function_identifier "describe") 7 | (eq? @function_identifier "it")) 8 | -------------------------------------------------------------------------------- /queries/example1.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: functionDeclarations 5 | query: | 6 | (function_declaration 7 | name: (identifier) @name 8 | parameters: (parameter_list) @parameters 9 | body: (block)) 10 | - name: importStatements 11 | query: | 12 | (import_declaration 13 | (import_spec_list [ 14 | (import_spec 15 | (package_identifier) @name 16 | path: (interpreted_string_literal) @path) 17 | (import_spec 18 | path: (interpreted_string_literal) @path) 19 | ])) 20 | 21 | template: | 22 | Function Declarations: 23 | {{ range .functionDeclarations.Matches }} 24 | - {{ .name.Text }}{{ .parameters.Text }}{{ end }} 25 | 26 | Import Statements: 27 | {{ range .importStatements.Matches }} 28 | - {{ if .name }}name: {{ .name.Text }}, {{end -}} path: {{ .path.Text }}{{ end }} 29 | 30 | Results:{{ range $v := .Results }} 31 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 32 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 33 | {{end}}{{ end }} -------------------------------------------------------------------------------- /queries/examples/go/returns.yaml: -------------------------------------------------------------------------------- 1 | name: returns 2 | short: A simple example to extract return statements in go functions 3 | 4 | flags: 5 | - name: verbose 6 | type: bool 7 | help: Output all results 8 | default: false 9 | 10 | language: go 11 | queries: 12 | - name: nakedReturns 13 | query: | 14 | ( 15 | (function_declaration 16 | name: (identifier) @functionName 17 | result: [ 18 | (parameter_list 19 | (parameter_declaration 20 | !name 21 | type: (type_identifier)? @resultType)) 22 | (type_identifier) @resultType 23 | (pointer_type) @resultType 24 | ])) 25 | - name: namedReturns 26 | query: | 27 | ( 28 | (function_declaration 29 | name: (identifier) @functionName 30 | result: (parameter_list 31 | (parameter_declaration 32 | name: (_) @resultName 33 | type: [ 34 | (type_identifier) @resultType 35 | (pointer_type) @resultType 36 | ])))) 37 | 38 | template: | 39 | {{ range $file, $results := .ResultsByFile -}} 40 | File: {{ $file }} 41 | 42 | {{ with $results -}} 43 | Naked Returns: 44 | {{- range .nakedReturns.Matches }} 45 | {{ .functionName.Text }} returns {{ .resultType.Text }} 46 | {{- end }} 47 | 48 | Named Returns: 49 | {{- range .namedReturns.Matches }} 50 | {{ .functionName.Text }} returns {{ .resultName.Text }} {{ .resultType.Text }} 51 | {{- end }} 52 | {{ end -}} 53 | {{ end -}} 54 | 55 | {{ if .verbose -}} 56 | Results:{{ range $v := .Results }} 57 | {{ $v.QueryName }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 58 | {{ $captureName }}: {{ $captureValue.Text }}{{ end }} 59 | {{end}}{{ end }} 60 | {{end -}} 61 | -------------------------------------------------------------------------------- /queries/go-func-declaration.txt: -------------------------------------------------------------------------------- 1 | (function_declaration 2 | name: (identifier) @name 3 | parameters: (parameter_list) @parameters 4 | body: (block)) 5 | -------------------------------------------------------------------------------- /queries/go-spec-fixed.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: binaryExpressionsWithNumberLiterals 5 | query: | 6 | (binary_expression 7 | (int_literal) @left 8 | (int_literal) @right) 9 | 10 | - name: binaryExpressionsWithStringLiteral 11 | query: | 12 | (binary_expression 13 | (interpreted_string_literal) @string) 14 | 15 | - name: assignmentExpressionsWithMemberExpression 16 | query: | 17 | (assignment_statement 18 | left: (expression_list 19 | (selector_expression 20 | operand: (identifier))) @member) 21 | 22 | - name: typeDeclarationsWithoutTypeParameters 23 | query: | 24 | (type_declaration 25 | (type_spec 26 | name: (type_identifier) @class_name 27 | !type_parameters)) 28 | 29 | - name: assignmentOfFunctionToIdentifier 30 | query: | 31 | (assignment_statement 32 | left: (identifier) @the_function_name 33 | right: (function_literal)) 34 | 35 | - name: methodDefinitionsInStruct 36 | query: | 37 | (type_declaration 38 | name: (type_identifier) @the_struct_name 39 | body: (struct_type 40 | (field_declaration 41 | name: (field_identifier) @the_method_name))) 42 | 43 | - name: sequenceOfComments 44 | query: | 45 | (comment)+ @comments 46 | 47 | - name: typeDeclarationsWithDecorators 48 | query: | 49 | (type_declaration 50 | (decorator)* @the_decorator 51 | name: (type_identifier) @the_name) 52 | 53 | - name: functionCallsWithStringArgument 54 | query: | 55 | (call_expression 56 | function: (identifier) @the_function 57 | arguments: (argument_list (interpreted_string_literal)? @the_string_arg)) 58 | 59 | - name: commentFollowedByFunctionDeclaration 60 | query: | 61 | ( 62 | (comment) @comment 63 | (function_declaration) @function 64 | ) 65 | 66 | - name: commaSeparatedSeriesOfNumbers 67 | query: | 68 | ( 69 | (int_literal) @number 70 | ("," (int_literal))* @numbers 71 | ) 72 | 73 | - name: callToVariableOrObjectProperty 74 | query: | 75 | (call_expression 76 | function: [ 77 | (identifier) @function 78 | (selector_expression 79 | field: (field_identifier) @method) 80 | ]) 81 | 82 | - name: keywordTokens 83 | query: | 84 | [ 85 | "break" 86 | "delete" 87 | "else" 88 | "for" 89 | "func" 90 | "if" 91 | "return" 92 | "try" 93 | "while" 94 | ] @keyword 95 | 96 | - name: anyNodeInsideCall 97 | query: | 98 | (call_expression (_) @call_inner) 99 | 100 | - name: firstIdentifierInArray 101 | query: | 102 | (array_type . (type_identifier) @the_element) 103 | 104 | - name: lastExpressionInBlock 105 | query: | 106 | (block (_) @last_expression .) 107 | 108 | - name: consecutiveIdentifiersInDottedName 109 | query: | 110 | (selector_expression 111 | operand: (identifier) @prev_id 112 | field: (field_identifier) @next_id) 113 | 114 | - name: identifierInScreamingSnakeCase 115 | query: | 116 | ( 117 | (identifier) @constant 118 | (#match? @constant "^[A-Z][A-Z_]+") 119 | ) 120 | 121 | - name: keyValuePairsWithSameName 122 | query: | 123 | ( 124 | (key_value_pair 125 | key: (field_identifier) @key_name 126 | value: (identifier) @value_name) 127 | (#eq? @key_name @value_name) 128 | ) 129 | 130 | template: | 131 | Results:{{ range $v := .Results }} 132 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 133 | {{ $captureName }}: {{ $captureValue }}{{ end }} 134 | {{end}}{{ end }} 135 | -------------------------------------------------------------------------------- /queries/go-spec-fixed2.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: binaryExpressionsWithNumberLiterals 5 | query: | 6 | (binary_expression 7 | left: (int_literal) @left 8 | right: (int_literal) @right) 9 | 10 | - name: binaryExpressionsWithStringLiteral 11 | query: | 12 | (binary_expression 13 | left: (interpreted_string_literal) @string 14 | _ 15 | right: (interpreted_string_literal) @string) 16 | 17 | - name: assignmentExpressionsWithMemberExpression 18 | query: | 19 | (assignment_statement 20 | left: (expression_list 21 | (selector_expression 22 | operand: (identifier) @member))) 23 | 24 | - name: typeDeclarationsWithoutTypeParameters 25 | query: | 26 | (type_declaration 27 | (type_spec 28 | name: (type_identifier) @class_name 29 | !type_parameters)) 30 | 31 | - name: assignmentOfFunctionToIdentifier 32 | query: | 33 | (assignment_statement 34 | left: (expression_list 35 | (identifier) @the_identifier) 36 | right: (expression_list 37 | (call_expression))) 38 | 39 | - name: methodDefinitionsInStruct 40 | query: | 41 | (type_declaration 42 | (type_spec 43 | name: (type_identifier) @the_struct_name 44 | type: (struct_type 45 | (field_declaration_list 46 | (field_declaration 47 | name: (field_identifier) @the_method_name))))) 48 | 49 | - name: sequenceOfComments 50 | query: | 51 | (comment)+ @comments 52 | 53 | - name: typeDeclarationsWithDecorators 54 | query: | 55 | (type_declaration 56 | (comment)* @the_decorator 57 | (type_spec 58 | name: (type_identifier) @the_name)) 59 | 60 | - name: functionCallsWithStringArgument 61 | query: | 62 | (call_expression 63 | function: (identifier) @the_function 64 | arguments: (argument_list (interpreted_string_literal)? @the_string_arg)) 65 | 66 | - name: commentFollowedByFunctionDeclaration 67 | query: | 68 | ( 69 | (comment) @comment 70 | (function_declaration) @function 71 | ) 72 | 73 | - name: commaSeparatedSeriesOfNumbers 74 | query: | 75 | ( 76 | (int_literal) @number 77 | ("," (int_literal))* @numbers 78 | ) 79 | 80 | - name: callToVariableOrObjectProperty 81 | query: | 82 | (call_expression 83 | function: [ 84 | (identifier) @function 85 | (selector_expression 86 | operand: (identifier) 87 | field: (field_identifier) @method) 88 | ]) 89 | 90 | - name: keywordTokens 91 | query: | 92 | [ 93 | "break" 94 | "else" 95 | "for" 96 | "func" 97 | "if" 98 | "return" 99 | ] @keyword 100 | 101 | - name: anyNodeInsideCall 102 | query: | 103 | (call_expression (_) @call_inner) 104 | 105 | - name: firstIdentifierInArray 106 | query: | 107 | (slice_type . (type_identifier) @the_element) 108 | 109 | - name: lastExpressionInBlock 110 | query: | 111 | (block (_) @last_expression .) 112 | 113 | - name: consecutiveIdentifiersInDottedName 114 | query: | 115 | (selector_expression 116 | operand: (identifier) @prev_id 117 | field: (field_identifier) @next_id) 118 | 119 | - name: identifierInScreamingSnakeCase 120 | query: | 121 | ( 122 | (identifier) @constant 123 | (#match? @constant "^[A-Z][A-Z_]+") 124 | ) 125 | 126 | - name: keyValuePairsWithSameName 127 | query: | 128 | ( 129 | (keyed_element 130 | (literal_element) @key_name 131 | (literal_element) @value_name) 132 | (#eq? @key_name @value_name) 133 | ) 134 | 135 | template: | 136 | Results:{{ range $v := .Results }} 137 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 138 | {{ $captureName }}({{ $captureValue.Type }}): {{ abbrev 50 $captureValue.Text }}{{ end }} 139 | {{end}}{{ end }} 140 | -------------------------------------------------------------------------------- /queries/go-spec.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: binaryExpressionsWithNumberLiterals 5 | query: | 6 | (binary_expression 7 | (int_literal) @left 8 | (int_literal) @right) 9 | 10 | - name: binaryExpressionsWithStringLiteral 11 | query: | 12 | (binary_expression 13 | (interpreted_string_literal) @string) 14 | 15 | - name: assignmentExpressionsWithMemberExpression 16 | query: | 17 | (assignment_statement 18 | left: (selector_expression 19 | operand: (call_expression)) @member) 20 | 21 | - name: classDeclarationsWithoutTypeParameters 22 | query: | 23 | (type_declaration 24 | name: (type_identifier) @class_name 25 | !type_parameters) 26 | 27 | - name: assignmentOfFunctionToIdentifier 28 | query: | 29 | (assignment_statement 30 | left: (identifier) @the_function_name 31 | right: (function_literal)) 32 | 33 | - name: methodDefinitionsInClass 34 | query: | 35 | (type_declaration 36 | name: (type_identifier) @the_class_name 37 | body: (struct_type 38 | (field_declaration 39 | name: (field_identifier) @the_method_name))) 40 | 41 | - name: sequenceOfComments 42 | query: | 43 | (comment)+ @comments 44 | 45 | - name: classDeclarationsWithDecorators 46 | query: | 47 | (type_declaration 48 | (decorator)* @the_decorator 49 | name: (type_identifier) @the_name) 50 | 51 | - name: functionCallsWithStringArgument 52 | query: | 53 | (call_expression 54 | function: (identifier) @the_function 55 | arguments: (argument_list (interpreted_string_literal)? @the_string_arg)) 56 | 57 | - name: commentFollowedByFunctionDeclaration 58 | query: | 59 | ( 60 | (comment) @comment 61 | (function_declaration) @function 62 | ) 63 | 64 | - name: commaSeparatedSeriesOfNumbers 65 | query: | 66 | ( 67 | (int_literal) @number 68 | ("," (int_literal))* @numbers 69 | ) 70 | 71 | - name: callToVariableOrObjectProperty 72 | query: | 73 | (call_expression 74 | function: [ 75 | (identifier) @function 76 | (selector_expression 77 | field: (field_identifier) @method) 78 | ]) 79 | 80 | - name: keywordTokens 81 | query: | 82 | [ 83 | "break" 84 | "delete" 85 | "else" 86 | "for" 87 | "func" 88 | "if" 89 | "return" 90 | "try" 91 | "while" 92 | ] @keyword 93 | 94 | - name: anyNodeInsideCall 95 | query: | 96 | (call_expression (_) @call_inner) 97 | 98 | - name: firstIdentifierInArray 99 | query: | 100 | (array_type . (type_identifier) @the_element) 101 | 102 | - name: lastExpressionInBlock 103 | query: | 104 | (block (_) @last_expression .) 105 | 106 | - name: consecutiveIdentifiersInDottedName 107 | query: | 108 | (selector_expression 109 | operand: (identifier) @prev_id 110 | field: (field_identifier) @next_id) 111 | 112 | - name: identifierInScreamingSnakeCase 113 | query: | 114 | ( 115 | (identifier) @constant 116 | (#match? @constant "^[A-Z][A-Z_]+") 117 | ) 118 | 119 | - name: keyValuePairsWithSameName 120 | query: | 121 | ( 122 | (key_value_pair 123 | key: (field_identifier) @key_name 124 | value: (identifier) @value_name) 125 | (#eq? @key_name @value_name) 126 | ) 127 | 128 | template: | 129 | Results:{{ range $v := .Results }} 130 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 131 | {{ $captureName }}: {{ $captureValue }}{{ end }} 132 | {{end}}{{ end }} 133 | -------------------------------------------------------------------------------- /queries/it.txt: -------------------------------------------------------------------------------- 1 | ((call_expression 2 | function: (identifier) @function_identifier 3 | arguments: (arguments . (_) @first_argument)) 4 | 5 | (#eq? @function_identifier "it")) 6 | -------------------------------------------------------------------------------- /queries/match-should.txt: -------------------------------------------------------------------------------- 1 | (call_expression 2 | function: (member_expression 3 | object: (member_expression 4 | property: (property_identifier) @object_property) 5 | property: (property_identifier) @equal_property) 6 | arguments: (_) @arguments) @call 7 | 8 | (eq? @object_property "should") 9 | (eq? @equal_property "equal") 10 | -------------------------------------------------------------------------------- /queries/match-should2.txt: -------------------------------------------------------------------------------- 1 | (call_expression 2 | function: (member_expression 3 | object: (member_expression 4 | property: (property_identifier) @object_property) 5 | property: (property_identifier) @equal_property) 6 | arguments: (_) @arguments) @call 7 | 8 | (#eq? @object_property "should") 9 | (#eq? @equal_property "should") 10 | -------------------------------------------------------------------------------- /queries/match-should3.txt: -------------------------------------------------------------------------------- 1 | ((call_expression 2 | function: (member_expression 3 | object: (member_expression 4 | property: (property_identifier) @object_property) 5 | property: (property_identifier) @equal_property 6 | ) 7 | arguments: (_) @arguments) @call 8 | ) 9 | 10 | -------------------------------------------------------------------------------- /queries/should.yaml: -------------------------------------------------------------------------------- 1 | language: javascript 2 | 3 | queries: 4 | - name: shoulds 5 | query: | 6 | (call_expression 7 | function: (member_expression 8 | object: (member_expression 9 | property: (property_identifier) @object_property) 10 | ) 11 | arguments: (_) @arguments) @call 12 | 13 | (#eq? @object_property "should") 14 | - name: describe 15 | query: | 16 | (call_expression 17 | function: (identifier) @function_identifier 18 | arguments: (arguments . (_) @first_argument) 19 | (#eq? @function_identifier "describe")) 20 | - name: it 21 | query: | 22 | (call_expression 23 | function: (identifier) @function_identifier 24 | arguments: (arguments . (_) @first_argument) 25 | (#eq? @function_identifier "it")) 26 | 27 | template: | 28 | Write a concise unit test description that can be used as first argument to 29 | `describe()` or `it()`, and that matches the following should statements: 30 | 31 | matches the following should statements: 32 | {{ range .shoulds.Matches }} 33 | - {{ .call.Text }}{{ end }} 34 | 35 | Output a single sentence starting with a verb. 36 | 37 | Then, check if the description matches the following descriptions: 38 | {{ range .describe.Matches }} 39 | - function `{{ .function_identifier.Text }}`: {{ .first_argument.Text }}{{ end }} 40 | {{- range .it.Matches }} 41 | - function `{{ .function_identifier.Text }}`: {{ .first_argument.Text }}{{ end }} 42 | 43 | Answer with YES or NO. 44 | -------------------------------------------------------------------------------- /queries/spec.txt: -------------------------------------------------------------------------------- 1 | (binary_expression 2 | (number_literal) @left 3 | (number_literal) @right) 4 | 5 | (binary_expression 6 | (string_literal) @string) 7 | 8 | (assignment_expression 9 | left: (member_expression 10 | object: (call_expression)) @member) 11 | 12 | (class_declaration 13 | name: (identifier) @class_name 14 | !type_parameters) 15 | 16 | (assignment_expression 17 | left: (identifier) @the_function_name 18 | right: (function)) 19 | 20 | (class_declaration 21 | name: (identifier) @the_class_name 22 | body: (class_body 23 | (method_definition 24 | name: (property_identifier) @the_method_name))) 25 | 26 | (comment)+ @comments 27 | 28 | (class_declaration 29 | (decorator)* @the_decorator 30 | name: (identifier) @the_name) 31 | 32 | (call_expression 33 | function: (identifier) @the_function 34 | arguments: (arguments (string)? @the_string_arg)) 35 | 36 | ( 37 | (comment) @comment 38 | (function_declaration) @function 39 | ) 40 | 41 | ( 42 | (number) @number 43 | ("," (number))* @numbers 44 | ) 45 | 46 | (call_expression 47 | function: [ 48 | (identifier) @function 49 | (member_expression 50 | property: (property_identifier) @method) 51 | ]) 52 | 53 | [ 54 | "break" 55 | "delete" 56 | "else" 57 | "for" 58 | "function" 59 | "if" 60 | "return" 61 | "try" 62 | "while" 63 | ] @keyword 64 | 65 | (call (_) @call_inner) 66 | 67 | (array . (identifier) @the_element) 68 | 69 | (block (_) @last_expression .) 70 | 71 | (dotted_name 72 | (identifier) @prev_id 73 | . 74 | (identifier) @next_id) 75 | 76 | ( 77 | (identifier) @constant 78 | (#match? @constant "^[A-Z][A-Z_]+") 79 | ) 80 | 81 | ( 82 | (pair 83 | key: (property_identifier) @key_name 84 | value: (identifier) @value_name) 85 | (#eq? @key_name @value_name) 86 | ) 87 | 88 | template: | 89 | Results:{{ range $v := .Results }} 90 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 91 | {{ $captureName }}: {{ $captureValue }}{{ end }} 92 | {{end}}{{ end }} 93 | 94 | -------------------------------------------------------------------------------- /queries/spec.yaml: -------------------------------------------------------------------------------- 1 | language: example 2 | 3 | queries: 4 | - name: binaryExpressionsWithNumberLiterals 5 | query: | 6 | (binary_expression 7 | (number_literal) @left 8 | (number_literal) @right) 9 | 10 | - name: binaryExpressionsWithStringLiteral 11 | query: | 12 | (binary_expression 13 | (string_literal) @string) 14 | 15 | - name: assignmentExpressionsWithMemberExpression 16 | query: | 17 | (assignment_statement 18 | left: (member_expression 19 | object: (call_expression)) @member) 20 | 21 | - name: classDeclarationsWithoutTypeParameters 22 | query: | 23 | (class_declaration 24 | name: (identifier) @class_name 25 | !type_parameters) 26 | 27 | - name: assignmentOfFunctionToIdentifier 28 | query: | 29 | (assignment_statement 30 | left: (identifier) @the_function_name 31 | right: (function)) 32 | 33 | - name: methodDefinitionsInClass 34 | query: | 35 | (class_declaration 36 | name: (identifier) @the_class_name 37 | body: (class_body 38 | (method_definition 39 | name: (property_identifier) @the_method_name))) 40 | 41 | - name: sequenceOfComments 42 | query: | 43 | (comment)+ @comments 44 | 45 | - name: classDeclarationsWithDecorators 46 | query: | 47 | (class_declaration 48 | (decorator)* @the_decorator 49 | name: (identifier) @the_name) 50 | 51 | - name: functionCallsWithStringArgument 52 | query: | 53 | (call_expression 54 | function: (identifier) @the_function 55 | arguments: (arguments (string)? @the_string_arg)) 56 | 57 | - name: commentFollowedByFunctionDeclaration 58 | query: | 59 | ( 60 | (comment) @comment 61 | (function_declaration) @function 62 | ) 63 | 64 | - name: commaSeparatedSeriesOfNumbers 65 | query: | 66 | ( 67 | (number) @number 68 | ("," (number))* @numbers 69 | ) 70 | 71 | - name: callToVariableOrObjectProperty 72 | query: | 73 | (call_expression 74 | function: [ 75 | (identifier) @function 76 | (member_expression 77 | property: (property_identifier) @method) 78 | ]) 79 | 80 | - name: keywordTokens 81 | query: | 82 | [ 83 | "break" 84 | "delete" 85 | "else" 86 | "for" 87 | "function" 88 | "if" 89 | "return" 90 | "try" 91 | "while" 92 | ] @keyword 93 | 94 | - name: anyNodeInsideCall 95 | query: | 96 | (call (_) @call_inner) 97 | 98 | - name: firstIdentifierInArray 99 | query: | 100 | (array . (identifier) @the_element) 101 | 102 | - name: lastExpressionInBlock 103 | query: | 104 | (block (_) @last_expression .) 105 | 106 | - name: consecutiveIdentifiersInDottedName 107 | query: | 108 | (dotted_name 109 | (identifier) @prev_id 110 | . 111 | (identifier) @next_id) 112 | 113 | - name: identifierInScreamingSnakeCase 114 | query: | 115 | ( 116 | (identifier) @constant 117 | (#match? @constant "^[A-Z][A-Z_]+") 118 | ) 119 | 120 | - name: keyValuePairsWithSameName 121 | query: | 122 | ( 123 | (pair 124 | key: (property_identifier) @key_name 125 | value: (identifier) @value_name) 126 | (#eq? @key_name @value_name) 127 | ) 128 | 129 | 130 | template: | 131 | Results:{{ range $v := .Results }} 132 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 133 | {{ $captureName }}: {{ $captureValue }}{{ end }} 134 | {{end}}{{ end }} 135 | -------------------------------------------------------------------------------- /queries/spec2.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: fields 5 | query: | 6 | (field_declaration 7 | name: (field_identifier) @fieldName 8 | type: (type_identifier) @fieldType) 9 | 10 | - name: negatedFields 11 | query: | 12 | (struct_type 13 | !field_declaration_list) 14 | 15 | - name: anonymousNodes 16 | query: | 17 | (binary_expression 18 | operator: "!=" 19 | right: (null)) 20 | 21 | - name: capturingNodes 22 | query: | 23 | (assignment_expression 24 | left: (identifier) @variableName 25 | right: (function)) 26 | 27 | - name: quantificationOperators 28 | query: | 29 | (call_expression 30 | function: (identifier) @functionName 31 | arguments: (arguments (string)? @stringArg)) 32 | 33 | - name: groupingSiblingNodes 34 | query: | 35 | ( 36 | (comment) 37 | (function_declaration) 38 | ) 39 | 40 | - name: alternations 41 | query: | 42 | (call_expression 43 | function: [ 44 | (identifier) @function 45 | (member_expression 46 | property: (property_identifier) @method) 47 | ]) 48 | 49 | - name: wildcardNode 50 | query: | 51 | (call (_) @callInner) 52 | 53 | - name: anchors 54 | query: | 55 | (array . (identifier) @firstElement) 56 | 57 | - name: predicates 58 | query: | 59 | ( 60 | (identifier) @constant 61 | (#match? @constant "^[A-Z][A-Z_]+") 62 | ) 63 | 64 | template: | 65 | Results:{{ range $v := .Results }} 66 | {{ $v.Name }}: {{ range $match := $v.Matches }}{{ range $captureName, $captureValue := $match }} 67 | {{ $captureName }}: {{ $captureValue }}{{ end }} 68 | {{end}}{{ end }} 69 | -------------------------------------------------------------------------------- /queries/test-constant.txt: -------------------------------------------------------------------------------- 1 | ( 2 | (identifier) @constant 3 | (#match? @constant "^[A-Z][A-Z_]+") 4 | ) 5 | -------------------------------------------------------------------------------- /queries/test.yaml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | queries: 4 | - name: testPredicate 5 | query: | 6 | (binary_expression 7 | left: (_) @left 8 | right: (_) @right 9 | (#eq? @right 1)) 10 | 11 | template: | 12 | {{ range .testPredicate.Matches }} 13 | - {{ .left.Text }} - {{.right.Text}}{{ end }} 14 | 15 | -------------------------------------------------------------------------------- /test-inputs/constants.js: -------------------------------------------------------------------------------- 1 | const camelCaseConst = 1; 2 | const SCREAMING_SNAKE_CASE_CONST = 2; 3 | const lower_snake_case_const = 3; 4 | -------------------------------------------------------------------------------- /test-inputs/ecs.hcl: -------------------------------------------------------------------------------- 1 | resource "aws_ecs_cluster" "go_webserver_test_cluster" { 2 | name = "go_webserver_test_cluster" 3 | } 4 | 5 | resource "aws_ecs_task_definition" "go_webserver_test_task_definition" { 6 | family = "go_webserver_test_task_family" 7 | container_definitions = jsonencode([ 8 | { 9 | "name" : "go_webserver_test_container", 10 | "image" : "${aws_ecr_repository.go_webserver_test_repository.repository_url}:latest", 11 | "portMappings" : [ 12 | { 13 | "containerPort" : 80, 14 | "hostPort" : 80, 15 | "protocol" : "tcp" 16 | } 17 | ], 18 | "logConfiguration" : { 19 | "logDriver" : "awslogs", 20 | "options" : { 21 | "awslogs-group" : "${aws_cloudwatch_log_group.go_webserver_test_log_group.name}", 22 | "awslogs-region" : "${data.aws_region.current.name}", 23 | "awslogs-stream-prefix" : "go_webserver_test_container" 24 | } 25 | }, 26 | healthCheck = { 27 | command = ["CMD-SHELL", "curl -f http://localhost:80/health || exit 1"] 28 | interval = 30 29 | timeout = 5 30 | startPeriod = 60 31 | retries = 3 32 | } 33 | } 34 | ]) 35 | requires_compatibilities = ["FARGATE"] 36 | cpu = "256" 37 | memory = "512" 38 | network_mode = "awsvpc" 39 | execution_role_arn = aws_iam_role.ecs_execution_role.arn 40 | } 41 | 42 | resource "aws_ecs_service" "go_webserver_test_service" { 43 | name = "go_webserver_test_service" 44 | cluster = aws_ecs_cluster.go_webserver_test_cluster.arn 45 | task_definition = aws_ecs_task_definition.go_webserver_test_task_definition.arn 46 | desired_count = 2 47 | launch_type = "FARGATE" 48 | 49 | # depends_on = [ 50 | # aws_alb_listener.go_webserver_test_listener, 51 | # ] 52 | 53 | deployment_minimum_healthy_percent = 80 54 | deployment_maximum_percent = 150 55 | 56 | network_configuration { 57 | security_groups = [ 58 | aws_security_group.ecs_security_group.id] # Setting the security group 59 | 60 | subnets = [ 61 | aws_subnet.private-subnet-1.id, 62 | aws_subnet.private-subnet-2.id, 63 | ] 64 | 65 | } 66 | load_balancer { 67 | target_group_arn = aws_alb_target_group.go_webserver_test_target_group.arn 68 | container_name = "go_webserver_test_container" 69 | container_port = 80 70 | } 71 | } 72 | 73 | resource "aws_iam_role" "ecs_execution_role" { 74 | name = "ecs_execution_role" 75 | 76 | assume_role_policy = jsonencode({ 77 | Version = "2012-10-17" 78 | Statement = [ 79 | { 80 | Action = "sts:AssumeRole" 81 | Effect = "Allow" 82 | Principal = { 83 | Service = "ecs-tasks.amazonaws.com" 84 | } 85 | Sid = "" 86 | } 87 | ] 88 | }) 89 | } 90 | 91 | resource "aws_security_group" "ecs_security_group" { 92 | name = "ecs_security_group" 93 | description = "Allow all inbound traffic" 94 | vpc_id = aws_vpc.go_webserver_test_vpc.id 95 | 96 | ingress { 97 | from_port = 0 98 | to_port = 0 99 | protocol = "-1" 100 | security_groups = [ 101 | aws_security_group.go_webserver_test_sg.id 102 | ] 103 | } 104 | 105 | egress { 106 | from_port = 0 107 | to_port = 0 108 | protocol = "-1" 109 | cidr_blocks = ["0.0.0.0/0"] 110 | } 111 | } 112 | 113 | resource "aws_iam_role_policy_attachment" "ecs_execution_role_policy_attachment" { 114 | policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" 115 | role = aws_iam_role.ecs_execution_role.name 116 | } 117 | -------------------------------------------------------------------------------- /test-inputs/go-example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Simple is an exported function 4 | func Simple(a int, b string) string { 5 | return b 6 | } 7 | 8 | // complexFunction is an unexported function with multiple parameters 9 | func complexFunction(items []string, callback func(string) bool) []string { 10 | var result []string 11 | for _, item := range items { 12 | if callback(item) { 13 | result = append(result, item) 14 | } 15 | } 16 | return result 17 | } 18 | 19 | // ExampleStruct is a simple struct with methods 20 | type ExampleStruct struct { 21 | Name string 22 | Age int 23 | } 24 | 25 | // PublicMethod is an exported method 26 | func (e *ExampleStruct) PublicMethod(param string) error { 27 | return nil 28 | } 29 | 30 | // privateMethod is an unexported method 31 | func (e ExampleStruct) privateMethod() { 32 | // Do something 33 | } 34 | -------------------------------------------------------------------------------- /test-inputs/php/all.php: -------------------------------------------------------------------------------- 1 | var1 = $var1; 20 | } 21 | 22 | /** 23 | * An example function. 24 | * 25 | * @return int 26 | */ 27 | public function exampleFunction(): int 28 | { 29 | return ExampleUtil::exampleFunction($this->var1); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test-inputs/php/finalClass.php: -------------------------------------------------------------------------------- 1 | 13 |

{title}

14 | 15 |

This is a card with some content.

16 | 18 | ); 19 | } 20 | 21 | type CardProps = { 22 | title: string; 23 | children: React.ReactNode; 24 | }; 25 | 26 | // Arrow function component that accepts children 27 | export const Card = ({ title, children }: CardProps) => { 28 | const [expanded, setExpanded] = useState(false); 29 | 30 | return ( 31 |
32 |

setExpanded(!expanded)}>{title}

33 | {expanded &&
{children}
} 34 |
35 | ); 36 | }; 37 | 38 | // Unexported arrow function component 39 | const InternalComponent = ({ message }: { message: string }) => { 40 | return
{message}
; 41 | }; 42 | 43 | // Component that uses children prop implicitly via props 44 | function Container(props: { className?: string }) { 45 | return ( 46 |
47 | {props.children} 48 |
49 | ); 50 | } 51 | 52 | export default Container; -------------------------------------------------------------------------------- /test-inputs/typescript/const-fn.ts: -------------------------------------------------------------------------------- 1 | const foobar = () => { console.log('foobar'); }; 2 | -------------------------------------------------------------------------------- /test-inputs/typescript/enum.ts: -------------------------------------------------------------------------------- 1 | enum FooEnum { 2 | CLAIM = 'claim', 3 | STORE_CREDIT = 'store_credit', 4 | GIFT_CARD = 'gift_card', 5 | } 6 | -------------------------------------------------------------------------------- /test-inputs/typescript/interface.ts: -------------------------------------------------------------------------------- 1 | interface Props { 2 | filter: string 3 | label?: string 4 | } 5 | -------------------------------------------------------------------------------- /test-inputs/typescript/types.ts: -------------------------------------------------------------------------------- 1 | type Props = { 2 | filter: string 3 | label?: string 4 | } 5 | --------------------------------------------------------------------------------