├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── dependency-scanning.yml │ ├── lint.yml │ ├── push.yml │ ├── release.yml │ └── secret-scanning.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yaml ├── .idea ├── misc.xml ├── modules.xml ├── sqliton.iml └── workspace.xml ├── .vscode └── launch.json ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── changelog.md ├── cmd └── sqleton │ ├── cmds │ ├── codegen.go │ ├── db.go │ ├── flags │ │ └── select.yaml │ ├── mcp │ │ └── mcp.go │ ├── query.go │ ├── run.go │ ├── select.go │ ├── serve.go │ ├── static │ │ └── favicon.ico │ ├── test-config-explicit-renderer.yaml │ ├── test-config-no-renderer.yaml │ ├── test-config-no-static.yaml │ ├── test-config-simplest.yaml │ └── test-config.yaml │ ├── doc │ ├── applications │ │ └── 01-mysql-get-distinct-connected-users.md │ ├── examples │ │ ├── 01-help-example.md │ │ ├── 02-list-dbt.md │ │ ├── mysql │ │ │ └── 01-mysql-ps.md │ │ ├── run-select-query │ │ │ ├── 01-run-show-process-list.md │ │ │ ├── 02-query-cli.md │ │ │ ├── 03-run-stdin.md │ │ │ ├── 04-select-table.md │ │ │ └── 05-create-select-query.md │ │ └── wp │ │ │ ├── 01-wp-ls-posts.md │ │ │ ├── 02-wp-ls-posts-shrubs.md │ │ │ └── 03-wp-ls-posts-select.md │ ├── topics │ │ ├── 01-sqleton.md │ │ ├── 02-database-sources.md │ │ ├── 03-subqueries.md │ │ ├── 04-aliases.md │ │ ├── 05-print-settings.md │ │ └── 06-query-commands.md │ └── tutorials │ │ └── 01-placeholder-tutorial.md │ ├── main.go │ └── queries │ ├── examples │ ├── 01-get-posts.yaml │ ├── 02-get-posts-limit.yaml │ └── 03-get-posts-by-type.yaml │ ├── mysql │ ├── 01-show-full-processlist.yaml │ ├── 02-show-tables.yaml │ ├── index.yaml │ ├── ps.yaml │ ├── schema.yaml │ ├── schema │ │ └── short.yaml │ ├── tables.yaml │ └── users.yaml │ ├── pg │ ├── connections.yaml │ ├── kill-connections.yaml │ └── locks.yaml │ ├── sqlite │ ├── hishtory │ │ └── ls.yaml │ └── tables.yaml │ └── wp │ ├── ls-posts.yaml │ ├── post-content.yaml │ ├── posts-counts.yaml │ └── wc │ ├── categories.yaml │ └── tax-rates.yaml ├── doc ├── logo.png └── vhs │ └── demo.tape ├── docker-compose.override.yml ├── docker-compose.yml ├── examples ├── config.yml └── show-processlist.sql ├── go.mod ├── go.sum ├── lefthook.yml ├── misc └── parca.yaml ├── pinocchio └── sqleton │ └── create-command.yaml └── pkg ├── cmds ├── factory.go ├── loaders.go ├── sql.go └── sql_test.go ├── codegen └── codegen.go └── flags ├── helpers.yaml └── settings.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | open-pull-requests-limit: 10 8 | labels: 9 | - "dependencies" 10 | - "security" 11 | ignore: 12 | - dependency-name: "*" 13 | update-types: ["version-update:semver-patch"] 14 | 15 | - package-ecosystem: "github-actions" 16 | directory: "/" 17 | schedule: 18 | interval: "weekly" 19 | open-pull-requests-limit: 10 20 | labels: 21 | - "dependencies" 22 | - "security" -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL Analysis" 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | schedule: 9 | - cron: '0 0 * * 0' # Run weekly on Sunday at midnight 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-latest 15 | permissions: 16 | security-events: write 17 | 18 | steps: 19 | - name: Checkout repository 20 | uses: actions/checkout@v4 21 | 22 | - name: Initialize CodeQL 23 | uses: github/codeql-action/init@v3 24 | with: 25 | languages: go 26 | 27 | - name: Perform CodeQL Analysis 28 | uses: github/codeql-action/analyze@v3 -------------------------------------------------------------------------------- /.github/workflows/dependency-scanning.yml: -------------------------------------------------------------------------------- 1 | name: Dependency Scanning 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | schedule: 9 | - cron: '0 0 * * 0' # Run weekly on Sunday at midnight 10 | 11 | jobs: 12 | dependency-review: 13 | name: Dependency Review 14 | runs-on: ubuntu-latest 15 | if: github.event_name == 'pull_request' 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v4 19 | 20 | - name: Dependency Review 21 | uses: actions/dependency-review-action@v4 22 | with: 23 | fail-on-severity: high 24 | 25 | govulncheck: 26 | name: Go Vulnerability Check 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout code 30 | uses: actions/checkout@v4 31 | 32 | - name: Set up Go 33 | uses: actions/setup-go@v5 34 | with: 35 | go-version: '1.24' 36 | 37 | - name: Install govulncheck 38 | run: go install golang.org/x/vuln/cmd/govulncheck@latest 39 | 40 | - name: Run govulncheck 41 | run: govulncheck ./... 42 | 43 | nancy: 44 | name: Nancy Vulnerability Scan 45 | runs-on: ubuntu-latest 46 | steps: 47 | - name: Checkout code 48 | uses: actions/checkout@v4 49 | 50 | - name: Set up Go 51 | uses: actions/setup-go@v5 52 | with: 53 | go-version: '1.24' 54 | 55 | - name: Install Nancy 56 | run: go install github.com/sonatype-nexus-community/nancy@latest 57 | 58 | - name: Run Nancy 59 | run: go list -json -deps ./... | nancy sleuth 60 | 61 | gosec: 62 | name: GoSec Security Scan 63 | runs-on: ubuntu-latest 64 | steps: 65 | - name: Checkout code 66 | uses: actions/checkout@v4 67 | 68 | - name: Set up Go 69 | uses: actions/setup-go@v5 70 | with: 71 | go-version: '1.24' 72 | 73 | - name: Run Gosec Security Scanner 74 | uses: securego/gosec@master 75 | with: 76 | args: -exclude=G101,G304,G301,G306,G204 -exclude-dir=.history ./... -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | # this is copied over from goreleaser/goreleaser/.github/workflows/lint.yml 2 | name: golangci-lint 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | branches: 8 | - main 9 | pull_request: 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | golangci: 15 | permissions: 16 | contents: read 17 | pull-requests: read 18 | name: lint 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | with: 23 | fetch-depth: 0 24 | - run: git fetch --force --tags 25 | - uses: actions/setup-go@v5 26 | with: 27 | go-version: '>=1.19.5' 28 | cache: true 29 | - name: golangci-lint 30 | uses: golangci/golangci-lint-action@v7 31 | with: 32 | version: v2.0.2 33 | args: --timeout=5m -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: golang-pipeline 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | - run: git fetch --force --tags 17 | - uses: actions/setup-go@v5 18 | with: 19 | go-version: '>=1.19.5' 20 | cache: true 21 | - 22 | name: Run unit tests 23 | run: go test ./... 24 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | # run only against tags 4 | tags: 5 | - '*' 6 | 7 | permissions: 8 | contents: write 9 | # packages: write 10 | # issues: write 11 | 12 | 13 | jobs: 14 | goreleaser: 15 | runs-on: ubuntu-latest 16 | env: 17 | DOCKER_CLI_EXPERIMENTAL: "enabled" 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 0 22 | 23 | - name: Set up QEMU 24 | uses: docker/setup-qemu-action@v3 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | 29 | - name: Docker Login 30 | uses: docker/login-action@v3 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.repository_owner }} 34 | password: ${{ secrets.RELEASE_ACTION_PAT }} 35 | 36 | - run: git fetch --force --tags 37 | - uses: actions/setup-go@v5 38 | with: 39 | go-version: '>=1.19.5' 40 | cache: true 41 | 42 | - name: Import GPG key 43 | id: import_gpg 44 | uses: crazy-max/ghaction-import-gpg@v6 45 | with: 46 | gpg_private_key: ${{ secrets.GO_GO_GOLEMS_SIGN_KEY }} 47 | passphrase: ${{ secrets.GO_GO_GOLEMS_SIGN_PASSPHRASE }} 48 | fingerprint: "6EBE1DF0BDF48A1BBA381B5B79983EF218C6ED7E" 49 | 50 | - uses: goreleaser/goreleaser-action@v6 51 | with: 52 | distribution: goreleaser 53 | version: latest 54 | args: release --clean 55 | env: 56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | COSIGN_PWD: ${{ secrets.COSIGN_PWD }} 58 | TAP_GITHUB_TOKEN: ${{ secrets.RELEASE_ACTION_PAT }} 59 | GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} 60 | FURY_TOKEN: ${{ secrets.FURY_TOKEN }} 61 | -------------------------------------------------------------------------------- /.github/workflows/secret-scanning.yml: -------------------------------------------------------------------------------- 1 | name: Secret Scanning 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | trufflehog: 11 | name: TruffleHog Secret Scan 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: TruffleHog OSS 20 | uses: trufflesecurity/trufflehog@main 21 | with: 22 | path: ./ 23 | base: ${{ github.event.repository.default_branch }} 24 | head: HEAD -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.env 2 | go.work 3 | dist/ 4 | ./sqleton 5 | /.envrc 6 | .history 7 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | linters: 4 | default: none 5 | enable: 6 | # defaults 7 | - errcheck 8 | - govet 9 | - ineffassign 10 | - staticcheck 11 | - unused 12 | # stuff I'm adding 13 | - exhaustive 14 | # - gochecknoglobals 15 | # - gochecknoinits 16 | - nonamedreturns 17 | - predeclared 18 | exclusions: 19 | rules: 20 | - linters: 21 | - staticcheck 22 | text: 'SA1019: cli.CreateProcessorLegacy' 23 | 24 | formatters: 25 | enable: 26 | - gofmt -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: sqleton 3 | 4 | before: 5 | hooks: 6 | # You may remove this if you don't use go modules. 7 | - go mod tidy 8 | # you may remove this if you don't need go generate 9 | - go generate ./... 10 | builds: 11 | - env: 12 | - CGO_ENABLED=0 13 | main: ./cmd/sqleton 14 | binary: sqleton 15 | id: sqleton-binaries 16 | goos: 17 | - linux 18 | # I am not able to test windows at the time 19 | # - windows 20 | - darwin 21 | goarch: 22 | - amd64 23 | - arm64 24 | 25 | # Add this section to include raw binaries 26 | archives: 27 | - id: raw-binaries 28 | builds: 29 | - sqleton-binaries 30 | format: "binary" 31 | 32 | checksum: 33 | name_template: 'checksums.txt' 34 | snapshot: 35 | name_template: "{{ incpatch .Version }}-next" 36 | changelog: 37 | sort: asc 38 | filters: 39 | exclude: 40 | - '^docs:' 41 | - '^test:' 42 | brews: 43 | - name: sqleton 44 | description: "Sqleton is a tool for querying databases" 45 | homepage: "https://github.com/go-go-golems/sqleton" 46 | repository: 47 | owner: go-go-golems 48 | name: homebrew-go-go-go 49 | token: "{{ .Env.TAP_GITHUB_TOKEN }}" 50 | 51 | dockers: 52 | - image_templates: 53 | - ghcr.io/go-go-golems/{{.ProjectName}}:{{ .Version }}-amd64 54 | dockerfile: Dockerfile 55 | use: buildx 56 | build_flag_templates: 57 | - "--pull" 58 | - --platform=linux/amd64 59 | - --label=org.opencontainers.image.title={{ .ProjectName }} 60 | - --label=org.opencontainers.image.description={{ .ProjectName }} 61 | - --label=org.opencontainers.image.url=https://github.com/go-go-golems/{{ .ProjectName }} 62 | - --label=org.opencontainers.image.source=https://github.com/go-go-golems/{{ .ProjectName }} 63 | - --label=org.opencontainers.image.version={{ .Version }} 64 | - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} 65 | - --label=org.opencontainers.image.revision={{ .FullCommit }} 66 | - --label=org.opencontainers.image.licenses=MIT 67 | - image_templates: 68 | - ghcr.io/go-go-golems/{{.ProjectName}}:{{ .Version }}-arm64v8 69 | dockerfile: Dockerfile 70 | use: buildx 71 | build_flag_templates: 72 | - "--pull" 73 | - --platform=linux/arm64/v8 74 | - --label=org.opencontainers.image.title={{ .ProjectName }} 75 | - --label=org.opencontainers.image.description={{ .ProjectName }} 76 | - --label=org.opencontainers.image.url=https://github.com/go-go-golems/{{ .ProjectName }} 77 | - --label=org.opencontainers.image.source=https://github.com/go-go-golems/{{ .ProjectName }} 78 | - --label=org.opencontainers.image.version={{ .Version }} 79 | - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} 80 | - --label=org.opencontainers.image.revision={{ .FullCommit }} 81 | - --label=org.opencontainers.image.licenses=MIT 82 | docker_manifests: 83 | - name_template: ghcr.io/go-go-golems/{{ .ProjectName }}:{{ .Version }} 84 | image_templates: 85 | - ghcr.io/go-go-golems/{{ .ProjectName }}:{{ .Version }}-amd64 86 | - ghcr.io/go-go-golems/{{ .ProjectName }}:{{ .Version }}-arm64v8 87 | - name_template: ghcr.io/go-go-golems/{{ .ProjectName }}:latest 88 | image_templates: 89 | - ghcr.io/go-go-golems/{{ .ProjectName }}:{{ .Version }}-amd64 90 | - ghcr.io/go-go-golems/{{ .ProjectName }}:{{ .Version }}-arm64v8 91 | nfpms: 92 | - 93 | id: packages 94 | 95 | vendor: GO GO GOLEMS 96 | homepage: https://github.com/go-go-golems/ 97 | maintainer: Manuel Odendahl 98 | 99 | description: |- 100 | Sqleton is a tool to query databases. 101 | 102 | license: MIT 103 | 104 | # Formats to be generated. 105 | formats: 106 | # - apk 107 | - deb 108 | - rpm 109 | 110 | # Version Release. 111 | release: "1" 112 | 113 | # Section. 114 | section: default 115 | 116 | # Priority. 117 | priority: extra 118 | 119 | # Custom configuration applied only to the Deb packager. 120 | deb: 121 | # Lintian overrides 122 | lintian_overrides: 123 | - statically-linked-binary 124 | - changelog-file-missing-in-native-package 125 | 126 | publishers: 127 | - name: fury.io 128 | # by specifying `packages` id here goreleaser will only use this publisher 129 | # with artifacts identified by this id 130 | ids: 131 | - packages 132 | dir: "{{ dir .ArtifactPath }}" 133 | cmd: curl -F package=@{{ .ArtifactName }} https://{{ .Env.FURY_TOKEN }}@push.fury.io/go-go-golems/ 134 | 135 | # modelines, feel free to remove those if you don't want/use them: 136 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 137 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/sqliton.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 1657028532328 6 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Completion ZSH", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}/cmd/sqleton", 13 | "args": ["completion", "zsh"] 14 | }, 15 | { 16 | "name": "Launch PostgreSQL Connection", 17 | "type": "go", 18 | "request": "launch", 19 | "mode": "auto", 20 | "program": "${workspaceFolder}/cmd/sqleton", 21 | "args": ["pg", "connections"], 22 | "env": { 23 | "SQLETON_DEBUG": "true" 24 | }, 25 | "envFile": "${workspaceFolder}/.envrc" 26 | }, 27 | { 28 | "name": "Launch MCP Tools List", 29 | "type": "go", 30 | "request": "launch", 31 | "mode": "auto", 32 | "program": "${workspaceFolder}/cmd/sqleton", 33 | "args": ["mcp", "tools", "list", "--print-parsed-parameters"], 34 | "env": { 35 | "SQLETON_DEBUG": "true" 36 | }, 37 | "envFile": "${workspaceFolder}/.envrc" 38 | } 39 | ] 40 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | # install curl, netstat, ping, vim 3 | RUN apt-get update && apt-get install -y \ 4 | ca-certificates curl net-tools iputils-ping vim \ 5 | && rm -rf /var/lib/apt/lists/* 6 | ENTRYPOINT ["/sqleton"] 7 | EXPOSE 8080 8 | COPY sqleton /sqleton -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Manuel Odendahl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: gifs test build lint lintmax docker-lint gosec govulncheck goreleaser tag-major tag-minor tag-patch release bump-glazed install codeql-local 2 | 3 | VERSION ?= $(shell svu) 4 | COMMIT ?= $(shell git rev-parse --short HEAD) 5 | DIRTY ?= $(shell git diff --quiet || echo "dirty") 6 | LDFLAGS=-ldflags "-X main.version=$(VERSION)-$(COMMIT)-$(DIRTY)" 7 | 8 | all: test build 9 | 10 | TAPES=$(shell ls doc/vhs/*tape) 11 | gifs: $(TAPES) 12 | for i in $(TAPES); do vhs < $$i; done 13 | 14 | docker-lint: 15 | docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v2.0.2 golangci-lint run -v 16 | 17 | ghcr-login: 18 | op read "$(CR_PAT)" | docker login ghcr.io -u wesen --password-stdin 19 | 20 | lint: 21 | golangci-lint run -v 22 | 23 | lintmax: 24 | golangci-lint run -v --max-same-issues=100 25 | 26 | gosec: 27 | go install github.com/securego/gosec/v2/cmd/gosec@latest 28 | # Adjust exclusions as needed 29 | gosec -exclude=G101,G304,G301,G306,G204 -exclude-dir=.history ./... 30 | 31 | govulncheck: 32 | go install golang.org/x/vuln/cmd/govulncheck@latest 33 | govulncheck ./... 34 | 35 | test: 36 | go test ./... 37 | 38 | build: 39 | go generate ./... 40 | go build $(LDFLAGS) ./... 41 | 42 | sqleton: 43 | go build $(LDFLAGS) -o sqleton ./cmd/sqleton 44 | 45 | build-docker: sqleton 46 | # GOOS=linux GOARCH=amd64 go build -o sqleton ./cmd/sqleton 47 | # docker buildx build -t go-go-golems/sqleton:amd64 . --platform=linux/amd64 48 | GOOS=linux GOARCH=arm64 go build $(LDFLAGS) -o sqleton ./cmd/sqleton 49 | docker buildx build -t go-go-golems/sqleton:arm64v8 . --platform=linux/arm64/v8 50 | 51 | up: 52 | docker compose up 53 | 54 | bash: 55 | docker compose exec sqleton bash 56 | 57 | goreleaser: 58 | goreleaser release --skip=sign --snapshot --clean 59 | 60 | tag-major: 61 | git tag $(shell svu major) 62 | 63 | tag-minor: 64 | git tag $(shell svu minor) 65 | 66 | tag-patch: 67 | git tag $(shell svu patch) 68 | 69 | release: 70 | git push --tags 71 | GOPROXY=proxy.golang.org go list -m github.com/go-go-golems/sqleton@$(shell svu current) 72 | 73 | bump-glazed: 74 | go get github.com/go-go-golems/glazed@latest 75 | go get github.com/go-go-golems/clay@latest 76 | go get github.com/go-go-golems/parka@latest 77 | go mod tidy 78 | 79 | SQLETON_BINARY=$(shell which sqleton) 80 | install: 81 | go build $(LDFLAGS) -o ./dist/sqleton ./cmd/sqleton && \ 82 | cp ./dist/sqleton $(SQLETON_BINARY) 83 | 84 | # Path to CodeQL CLI - adjust based on installation location 85 | CODEQL_PATH ?= $(shell which codeql) 86 | # Path to CodeQL queries - adjust based on where you cloned the repository 87 | CODEQL_QUERIES ?= $(HOME)/codeql-go/ql/src/go 88 | 89 | # Create CodeQL database and run analysis 90 | codeql-local: 91 | @if [ -z "$(CODEQL_PATH)" ]; then echo "CodeQL CLI not found. Install from https://github.com/github/codeql-cli-binaries/releases"; exit 1; fi 92 | @if [ ! -d "$(CODEQL_QUERIES)" ]; then echo "CodeQL queries not found. Clone from https://github.com/github/codeql-go"; exit 1; fi 93 | $(CODEQL_PATH) database create --language=go --source-root=. ./codeql-db 94 | $(CODEQL_PATH) database analyze ./codeql-db $(CODEQL_QUERIES)/Security --format=sarif-latest --output=codeql-results.sarif 95 | @echo "Results saved to codeql-results.sarif" 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ☠️ sqleton ☠️ - a tool to quickly execute SQL commands 2 | 3 | [![golangci-lint](https://github.com/wesen/sqleton/actions/workflows/lint.yml/badge.svg)](https://github.com/wesen/sqleton/actions/workflows/lint.yml) 4 | [![golang-pipeline](https://github.com/wesen/sqleton/actions/workflows/push.yml/badge.svg)](https://github.com/wesen/sqleton/actions/workflows/push.yml) 5 | 6 | ![sqleton logo](doc/logo.png) 7 | 8 | sqleton is a tool to easily run SQL commands. 9 | 10 | ## ☠️ Main features ☠️ 11 | 12 | - easily make a self-contained CLI to interact with your application 13 | - extend the CLI application with your own repository of commands 14 | - create aliases for existing commands to save flag presets 15 | - rich data format output thanks to [glazed](https://github.com/go-go-golems/glazed) 16 | - rich help system courtesy of [glazed](https://github.com/go-go-golems/glazed) 17 | - easily add documentation to your CLI by editing markdown files 18 | - support for connection information from: 19 | - environment variables 20 | - configuration file 21 | - command line flags 22 | - DBT profiles 23 | - easy API for developers to supply custom commands 24 | 25 | ## Demo 26 | 27 | ![Demo of sqleton](https://i.imgur.com/agO8aYr.gif) 28 | 29 | ## Overview 30 | 31 | sqleton makes it easy to specify SQL commands along with parameters in a YAML file and run 32 | the result as a nice CLI application. 33 | 34 | These files look like the following: 35 | 36 | ```yaml 37 | name: ls-posts [types...] 38 | short: "Show all WordPress posts" 39 | long: | 40 | Show all WordPress posts and their ID, allowing you to filter by type, status, 41 | date and title. 42 | flags: 43 | - name: limit 44 | shortFlag: l 45 | type: int 46 | default: 10 47 | help: Limit the number of posts 48 | - name: status 49 | type: stringList 50 | help: Select posts by status 51 | required: false 52 | - name: order_by 53 | type: string 54 | default: post_date DESC 55 | help: Order by column 56 | - name: types 57 | type: stringList 58 | default: 59 | - post 60 | - page 61 | help: Select posts by type 62 | required: false 63 | - name: from 64 | type: date 65 | help: Select posts from date 66 | required: false 67 | - name: to 68 | type: date 69 | help: Select posts to date 70 | required: false 71 | - name: title_like 72 | type: string 73 | help: Select posts by title 74 | required: false 75 | query: | 76 | SELECT wp.ID, wp.post_title, wp.post_type, wp.post_status, wp.post_date FROM wp_posts wp 77 | WHERE post_type IN ({{ .types | sqlStringIn }}) 78 | {{ if .status -}} 79 | AND post_status IN ({{ .status | sqlStringIn }}) 80 | {{- end -}} 81 | {{ if .from -}} 82 | AND post_date >= {{ .from | sqlDate }} 83 | {{- end -}} 84 | {{ if .to -}} 85 | AND post_date <= {{ .to | sqlDate }} 86 | {{- end -}} 87 | {{ if .title_like -}} 88 | AND post_title LIKE {{ .title_like | sqlLike }} 89 | {{- end -}} 90 | ORDER BY {{ .order_by }} 91 | LIMIT {{ .limit }} 92 | ``` 93 | 94 | Furthermore, these commands can be included in the binary itself, which makes 95 | distributing a rich set of queries very easy. The same concept is used to provide 96 | a rich help system, which are just bundled markdown files. Instead of having to edit 97 | the source code to provide advanced documentation for flags, commands and general topics, 98 | all you need to do is add a markdown file in the doc/ directory! 99 | 100 | ``` 101 | ❯ sqleton queries --fields name,source 102 | +--------------------------+--------------------------------------------------------+ 103 | | name | source | 104 | +--------------------------+--------------------------------------------------------+ 105 | | ls-posts | embed:queries/examples/01-get-posts.yaml | 106 | | ls-posts-limit | embed:queries/examples/02-get-posts-limit.yaml | 107 | | ls-posts-type [types...] | embed:queries/examples/03-get-posts-by-type.yaml | 108 | | full-ps | embed:queries/mysql/01-show-full-processlist.yaml | 109 | | ls-tables | embed:queries/mysql/02-show-tables.yaml | 110 | | ls-posts [types...] | embed:queries/wp/ls-posts.yaml | 111 | | count | file:/Users/manuel/.sqleton/queries/misc/count.yaml | 112 | | ls-orders | file:/Users/manuel/.sqleton/queries/ttc/01-orders.yaml | 113 | +--------------------------+--------------------------------------------------------+ 114 | ``` 115 | 116 | It makes heavy use of my [glazed](https://github.com/go-go-golems/glazed) library, 117 | and in many ways is a test-driver for its development presets -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | ## Docker Registry Authentication Fix 2 | Documented proper PAT setup for GitHub Container Registry authentication and enabled required permissions. 3 | 4 | - Uncommented packages write permission in release workflow 5 | - Added documentation for setting up RELEASE_ACTION_PAT with proper scopes 6 | 7 | # AWS Config Debug Logging 8 | 9 | Added debug logging for AWS configuration to aid in troubleshooting AWS connectivity issues. The logging includes non-sensitive information like region and retry mode settings. 10 | 11 | - Added debug logging after AWS config loading in SSM evaluator 12 | 13 | # Enhanced AWS Debug Information 14 | 15 | Added additional AWS debugging information to help with authentication and identity issues: 16 | 17 | - Added truncated access key ID logging (first 4 chars only) 18 | - Added credential provider source logging 19 | - Added STS caller identity information (account, ARN, user ID) 20 | 21 | # Enhanced SSM Error Logging 22 | 23 | Added AWS identity information to SSM parameter retrieval error logs to help diagnose permission issues: 24 | 25 | - Added account and ARN information when SSM parameter retrieval fails 26 | - Integrated STS identity checks into error context 27 | 28 | # Fix SSM Region Resolution 29 | 30 | Fixed SSM parameter access by explicitly setting AWS region: 31 | 32 | - Added explicit us-east-1 region configuration to match SSM parameter location 33 | - Addresses ResolveEndpointV2 errors when region is not set in environment 34 | 35 | # Improve Region Configuration 36 | 37 | Enhanced AWS region configuration to be more flexible: 38 | 39 | - Added support for AWS_REGION and AWS_DEFAULT_REGION environment variables 40 | - Added debug logging for region environment variables 41 | - Fallback to us-east-1 only if no region is set in environment -------------------------------------------------------------------------------- /cmd/sqleton/cmds/codegen.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "fmt" 5 | "github.com/go-go-golems/glazed/pkg/cmds" 6 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 7 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 8 | cmds2 "github.com/go-go-golems/sqleton/pkg/cmds" 9 | "github.com/go-go-golems/sqleton/pkg/codegen" 10 | "github.com/pkg/errors" 11 | "github.com/spf13/cobra" 12 | "os" 13 | "path" 14 | "strings" 15 | ) 16 | 17 | func NewCodegenCommand() *cobra.Command { 18 | ret := &cobra.Command{ 19 | Use: "codegen [file...]", 20 | Short: "A program to convert Sqleton YAML commands into Go code", 21 | Args: cobra.MinimumNArgs(1), 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | packageName := cmd.Flag("package-name").Value.String() 24 | outputDir := cmd.Flag("output-dir").Value.String() 25 | 26 | s := &codegen.SqlCommandCodeGenerator{ 27 | PackageName: packageName, 28 | } 29 | for _, fileName := range args { 30 | loader := &cmds2.SqlCommandLoader{ 31 | DBConnectionFactory: nil, 32 | } 33 | 34 | fs_, fileName, err := loaders.FileNameToFsFilePath(fileName) 35 | if err != nil { 36 | return err 37 | } 38 | cmds_, err := loader.LoadCommands(fs_, fileName, []cmds.CommandDescriptionOption{}, []alias.Option{}) 39 | if err != nil { 40 | return err 41 | } 42 | if len(cmds_) != 1 { 43 | return errors.Errorf("expected exactly one command, got %d", len(cmds_)) 44 | } 45 | cmd := cmds_[0].(*cmds2.SqlCommand) 46 | 47 | f, err := s.GenerateCommandCode(cmd) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | s := f.GoString() 53 | // store in path.go after removing .yaml 54 | p, _ := strings.CutSuffix(path.Base(fileName), ".yaml") 55 | p = p + ".go" 56 | p = path.Join(outputDir, p) 57 | 58 | fmt.Printf("Converting %s to %s\n", fileName, p) 59 | _ = os.WriteFile(p, []byte(s), 0644) 60 | } 61 | 62 | return nil 63 | }, 64 | } 65 | 66 | ret.PersistentFlags().StringP("output-dir", "o", ".", "Output directory for generated code") 67 | ret.PersistentFlags().StringP("package-name", "p", "main", "Package name for generated code") 68 | return ret 69 | } 70 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/db.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | sql2 "github.com/go-go-golems/clay/pkg/sql" 7 | "github.com/go-go-golems/glazed/pkg/cli" 8 | "github.com/go-go-golems/glazed/pkg/cmds" 9 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 10 | "github.com/go-go-golems/glazed/pkg/middlewares/row" 11 | "github.com/go-go-golems/glazed/pkg/types" 12 | "github.com/jmoiron/sqlx" 13 | "github.com/spf13/cobra" 14 | "github.com/spf13/viper" 15 | "os" 16 | 17 | _ "github.com/go-sql-driver/mysql" // MySQL driver for database/sql 18 | ) 19 | 20 | // From chatGPT: 21 | // To run SQL commands against a PostgreSQL or SQLite database, you can use a similar 22 | // approach, but you will need to use the appropriate driver for the database. 23 | // For example, to use PostgreSQL, you can use the github.com/lib/pq driver, and to use SQLite, 24 | // you can use the github.com/mattn/go-sqlite3 25 | 26 | var DbCmd = &cobra.Command{ 27 | Use: "db", 28 | Short: "Manage databases", 29 | } 30 | 31 | func createConfigFromCobra(cmd *cobra.Command) *sql2.DatabaseConfig { 32 | connectionLayer, err := sql2.NewSqlConnectionParameterLayer() 33 | cobra.CheckErr(err) 34 | 35 | dbtLayer, err := sql2.NewDbtParameterLayer() 36 | cobra.CheckErr(err) 37 | 38 | description := cmds.NewCommandDescription( 39 | cmd.Name(), 40 | cmds.WithLayersList(connectionLayer, dbtLayer), 41 | ) 42 | 43 | parser, err := cli.NewCobraParserFromLayers( 44 | description.Layers, 45 | cli.WithCobraMiddlewaresFunc(sql2.GetCobraCommandSqletonMiddlewares), 46 | ) 47 | cobra.CheckErr(err) 48 | 49 | parsedLayers, err := parser.Parse(cmd, nil) 50 | cobra.CheckErr(err) 51 | 52 | sqlParsedLayer := parsedLayers.GetOrCreate(connectionLayer) 53 | dbtParsedLayer := parsedLayers.GetOrCreate(dbtLayer) 54 | config, err := sql2.NewConfigFromParsedLayers(dbtParsedLayer, sqlParsedLayer) 55 | cobra.CheckErr(err) 56 | 57 | return config 58 | } 59 | 60 | var dbTestConnectionCmd = &cobra.Command{ 61 | Use: "test", 62 | Short: "Test the connection to a database", 63 | Run: func(cmd *cobra.Command, args []string) { 64 | config := createConfigFromCobra(cmd) 65 | 66 | fmt.Printf("Testing connection to %s\n", config.ToString()) 67 | db, err := config.Connect() 68 | cobra.CheckErr(err) 69 | 70 | cobra.CheckErr(err) 71 | defer func(db *sqlx.DB) { 72 | _ = db.Close() 73 | }(db) 74 | 75 | err = db.Ping() 76 | cobra.CheckErr(err) 77 | 78 | fmt.Println("Connection successful") 79 | }, 80 | } 81 | 82 | // dbTestConnectionCmdWithPrefix is a test command to use 83 | // configuration flags and settings with a prefix, which can be used to 84 | // mix sqleton commands with say, escuse-me commands 85 | var dbTestConnectionCmdWithPrefix = &cobra.Command{ 86 | Use: "test-prefix", 87 | Short: "Test the connection to a database, but all sqleton flags have the test- prefix", 88 | Run: func(cmd *cobra.Command, args []string) { 89 | config := createConfigFromCobra(cmd) 90 | fmt.Printf("Testing connection to %s\n", config.ToString()) 91 | db, err := config.Connect() 92 | cobra.CheckErr(err) 93 | 94 | cobra.CheckErr(err) 95 | defer func(db *sqlx.DB) { 96 | _ = db.Close() 97 | }(db) 98 | 99 | err = db.Ping() 100 | cobra.CheckErr(err) 101 | 102 | fmt.Println("Connection successful") 103 | }, 104 | } 105 | 106 | // dbTestConnectionCmdWithPrefix is a test command to use 107 | // configuration flags and settings with a prefix, which can be used to 108 | // mix sqleton commands with say, escuse-me commands 109 | var dbPrintEvidenceSettingsCmd = &cobra.Command{ 110 | Use: "print-evidence-settings", 111 | Short: "Output the settings to connect to a database for evidence.dev", 112 | Run: func(cmd *cobra.Command, args []string) { 113 | config := createConfigFromCobra(cmd) 114 | source, err := config.GetSource() 115 | cobra.CheckErr(err) 116 | 117 | gitRepo, _ := cmd.Flags().GetString("git-repo") 118 | 119 | type EvidenceCredentials struct { 120 | Host string `json:"host"` 121 | Database string `json:"database"` 122 | User string `json:"user"` 123 | Password string `json:"password"` 124 | Port string `json:"port"` 125 | } 126 | 127 | type EvidenceSettings struct { 128 | GitRepo string `json:"gitRepo,omitempty"` 129 | Database string `json:"database"` 130 | Credentials EvidenceCredentials `json:"credentials"` 131 | } 132 | 133 | credentials := EvidenceCredentials{ 134 | Host: source.Hostname, 135 | Database: source.Database, 136 | User: source.Username, 137 | Password: source.Password, 138 | Port: fmt.Sprintf("%d", source.Port), 139 | } 140 | 141 | settings := EvidenceSettings{ 142 | GitRepo: gitRepo, 143 | Database: source.Type, 144 | Credentials: credentials, 145 | } 146 | 147 | encoder := json.NewEncoder(os.Stdout) 148 | encoder.SetIndent("", " ") 149 | err = encoder.Encode(settings) 150 | cobra.CheckErr(err) 151 | }, 152 | } 153 | 154 | var dbPrintEnvCmd = &cobra.Command{ 155 | Use: "print-env", 156 | Short: "Output the settings to connect to a database as environment variables", 157 | Run: func(cmd *cobra.Command, args []string) { 158 | config := createConfigFromCobra(cmd) 159 | source, err := config.GetSource() 160 | cobra.CheckErr(err) 161 | 162 | isEnvRc, _ := cmd.Flags().GetBool("envrc") 163 | envPrefix, _ := cmd.Flags().GetString("env-prefix") 164 | 165 | prefix := "" 166 | if isEnvRc { 167 | prefix = "export " 168 | } 169 | prefix = prefix + envPrefix 170 | fmt.Printf("%s%s=%s\n", prefix, "TYPE", source.Type) 171 | fmt.Printf("%s%s=%s\n", prefix, "HOST", source.Hostname) 172 | fmt.Printf("%s%s=%s\n", prefix, "PORT", fmt.Sprintf("%d", source.Port)) 173 | fmt.Printf("%s%s=%s\n", prefix, "DATABASE", source.Database) 174 | fmt.Printf("%s%s=%s\n", prefix, "USER", source.Username) 175 | fmt.Printf("%s%s=%s\n", prefix, "PASSWORD", source.Password) 176 | fmt.Printf("%s%s=%s\n", prefix, "SCHEMA", source.Schema) 177 | if config.UseDbtProfiles { 178 | fmt.Printf("%s%s=1\n", prefix, "USE_DBT_PROFILES") 179 | } else { 180 | fmt.Printf("%s%s=\n", prefix, "USE_DBT_PROFILES") 181 | } 182 | fmt.Printf("%s%s=%s\n", prefix, "DBT_PROFILES_PATH", config.DbtProfilesPath) 183 | fmt.Printf("%s%s=%s\n", prefix, "DBT_PROFILE", config.DbtProfile) 184 | }, 185 | } 186 | 187 | var dbPrintSettingsCmd = &cobra.Command{ 188 | Use: "print-settings", 189 | Short: "Output the settings to connect to a database using glazed", 190 | Run: func(cmd *cobra.Command, args []string) { 191 | config := createConfigFromCobra(cmd) 192 | source, err := config.GetSource() 193 | cobra.CheckErr(err) 194 | 195 | gp, _, err := cli.CreateGlazedProcessorFromCobra(cmd) 196 | if err != nil { 197 | _, _ = fmt.Fprintf(os.Stderr, "Could not create glaze procersors: %v\n", err) 198 | os.Exit(1) 199 | } 200 | 201 | individualRows, _ := cmd.Flags().GetBool("individual-rows") 202 | useSqletonEnvNames, _ := cmd.Flags().GetBool("use-env-names") 203 | withEnvPrefix, _ := cmd.Flags().GetString("with-env-prefix") 204 | 205 | ctx := cmd.Context() 206 | 207 | host := "host" 208 | port := "port" 209 | database := "database" 210 | user := "user" 211 | password := "password" 212 | type_ := "type" 213 | schema := "schema" 214 | dbtProfile := "dbtProfile" 215 | useDbtProfiles := "useDbtProfiles" 216 | dbtProfilesPath := "dbtProfilesPath" 217 | 218 | if useSqletonEnvNames { 219 | host = "SQLETON_HOST" 220 | port = "SQLETON_PORT" 221 | database = "SQLETON_DATABASE" 222 | user = "SQLETON_USER" 223 | password = "SQLETON_PASSWORD" 224 | type_ = "SQLETON_TYPE" 225 | schema = "SQLETON_SCHEMA" 226 | dbtProfile = "SQLETON_DBT_PROFILE" 227 | useDbtProfiles = "SQLETON_USE_DBT_PROFILES" 228 | dbtProfilesPath = "SQLETON_DBT_PROFILES_PATH" 229 | } else if withEnvPrefix != "" { 230 | host = fmt.Sprintf("%sHOST", withEnvPrefix) 231 | port = fmt.Sprintf("%sPORT", withEnvPrefix) 232 | database = fmt.Sprintf("%sDATABASE", withEnvPrefix) 233 | user = fmt.Sprintf("%sUSER", withEnvPrefix) 234 | password = fmt.Sprintf("%sPASSWORD", withEnvPrefix) 235 | type_ = fmt.Sprintf("%sTYPE", withEnvPrefix) 236 | schema = fmt.Sprintf("%sSCHEMA", withEnvPrefix) 237 | dbtProfile = fmt.Sprintf("%sDBT_PROFILE", withEnvPrefix) 238 | useDbtProfiles = fmt.Sprintf("%sUSE_DBT_PROFILES", withEnvPrefix) 239 | dbtProfilesPath = fmt.Sprintf("%sDBT_PROFILES_PATH", withEnvPrefix) 240 | } 241 | 242 | addRow := func(name string, value interface{}) { 243 | _ = gp.AddRow(ctx, types.NewRow( 244 | types.MRP("name", name), 245 | types.MRP("value", value), 246 | )) 247 | } 248 | if individualRows { 249 | addRow(host, source.Hostname) 250 | addRow(port, source.Port) 251 | addRow(database, source.Database) 252 | addRow(user, source.Username) 253 | addRow(password, source.Password) 254 | addRow(type_, source.Type) 255 | addRow(schema, source.Schema) 256 | addRow(dbtProfile, config.DbtProfile) 257 | addRow(useDbtProfiles, config.UseDbtProfiles) 258 | addRow(dbtProfilesPath, config.DbtProfilesPath) 259 | } else { 260 | _ = gp.AddRow(ctx, types.NewRow( 261 | types.MRP(host, source.Hostname), 262 | types.MRP(port, source.Port), 263 | types.MRP(database, source.Database), 264 | types.MRP(user, source.Username), 265 | types.MRP(password, source.Password), 266 | types.MRP(type_, source.Type), 267 | types.MRP(schema, source.Schema), 268 | types.MRP(dbtProfile, config.DbtProfile), 269 | types.MRP(useDbtProfiles, config.UseDbtProfiles), 270 | types.MRP(dbtProfilesPath, config.DbtProfilesPath), 271 | )) 272 | } 273 | 274 | err = gp.Close(ctx) 275 | if err != nil { 276 | _, _ = fmt.Fprintf(os.Stderr, "Error rendering output: %s\n", err) 277 | os.Exit(1) 278 | } 279 | cobra.CheckErr(err) 280 | }, 281 | } 282 | 283 | var dbLsCmd = &cobra.Command{ 284 | Use: "ls", 285 | Short: "List databases from profiles", 286 | Run: func(cmd *cobra.Command, args []string) { 287 | ctx := cmd.Context() 288 | 289 | useDbtProfiles := viper.GetBool("use-dbt-profiles") 290 | 291 | if !useDbtProfiles { 292 | cmd.PrintErrln("Not using dbt profiles") 293 | return 294 | } 295 | 296 | dbtProfilesPath := viper.GetString("dbt-profiles-path") 297 | 298 | sources, err := sql2.ParseDbtProfiles(dbtProfilesPath) 299 | cobra.CheckErr(err) 300 | 301 | gp, _, err := cli.CreateGlazedProcessorFromCobra(cmd) 302 | if err != nil { 303 | _, _ = fmt.Fprintf(os.Stderr, "Could not create glaze procersors: %v\n", err) 304 | os.Exit(1) 305 | } 306 | 307 | // don't output the password 308 | gp.AddRowMiddleware(row.NewFieldsFilterMiddleware( 309 | row.WithFields([]string{"name", "type", "hostname", "port", "database", "schema"}), 310 | row.WithFilters([]string{"password"}), 311 | )) 312 | gp.AddRowMiddleware(row.NewReorderColumnOrderMiddleware([]string{"name", "type", "hostname", "port", "database", "schema"})) 313 | 314 | for _, source := range sources { 315 | row := types.NewRowFromStruct(source, true) 316 | err := gp.AddRow(ctx, row) 317 | cobra.CheckErr(err) 318 | } 319 | 320 | err = gp.Close(ctx) 321 | if err != nil { 322 | _, _ = fmt.Fprintf(os.Stderr, "Error rendering output: %s\n", err) 323 | os.Exit(1) 324 | } 325 | cobra.CheckErr(err) 326 | }, 327 | } 328 | 329 | func init() { 330 | err := cli.AddGlazedProcessorFlagsToCobraCommand(dbLsCmd) 331 | cobra.CheckErr(err) 332 | DbCmd.AddCommand(dbLsCmd) 333 | 334 | connectionLayer, err := sql2.NewSqlConnectionParameterLayer() 335 | cobra.CheckErr(err) 336 | dbtParameterLayer, err := sql2.NewDbtParameterLayer() 337 | cobra.CheckErr(err) 338 | 339 | err = connectionLayer.AddLayerToCobraCommand(dbTestConnectionCmd) 340 | cobra.CheckErr(err) 341 | DbCmd.AddCommand(dbTestConnectionCmd) 342 | 343 | err = dbtParameterLayer.AddLayerToCobraCommand(dbTestConnectionCmd) 344 | cobra.CheckErr(err) 345 | 346 | err = connectionLayer.AddLayerToCobraCommand(dbPrintEvidenceSettingsCmd) 347 | cobra.CheckErr(err) 348 | dbPrintEvidenceSettingsCmd.Flags().String("git-repo", "", "Git repo to use for evidence.dev") 349 | DbCmd.AddCommand(dbPrintEvidenceSettingsCmd) 350 | 351 | err = connectionLayer.AddLayerToCobraCommand(dbPrintEnvCmd) 352 | cobra.CheckErr(err) 353 | dbPrintEnvCmd.Flags().Bool("envrc", false, "Output as an .envrc file") 354 | dbPrintEnvCmd.Flags().String("env-prefix", "SQLETON_", "Prefix for environment variables") 355 | DbCmd.AddCommand(dbPrintEnvCmd) 356 | 357 | err = connectionLayer.AddLayerToCobraCommand(dbPrintSettingsCmd) 358 | cobra.CheckErr(err) 359 | err = dbtParameterLayer.AddLayerToCobraCommand(dbPrintSettingsCmd) 360 | cobra.CheckErr(err) 361 | 362 | dbPrintSettingsCmd.Flags().Bool("individual-rows", false, "Output as individual rows") 363 | dbPrintSettingsCmd.Flags().String("with-env-prefix", "", "Output as environment variables with a prefix") 364 | dbPrintSettingsCmd.Flags().Bool("use-env-names", false, "Output as SQLETON_ environment variables with a prefix") 365 | err = cli.AddGlazedProcessorFlagsToCobraCommand(dbPrintSettingsCmd) 366 | cobra.CheckErr(err) 367 | DbCmd.AddCommand(dbPrintSettingsCmd) 368 | 369 | connectionLayer, err = sql2.NewSqlConnectionParameterLayer( 370 | layers.WithPrefix("test-"), 371 | ) 372 | cobra.CheckErr(err) 373 | err = connectionLayer.AddLayerToCobraCommand(dbTestConnectionCmdWithPrefix) 374 | cobra.CheckErr(err) 375 | DbCmd.AddCommand(dbTestConnectionCmdWithPrefix) 376 | } 377 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/flags/select.yaml: -------------------------------------------------------------------------------- 1 | slug: select 2 | name: Select flags 3 | Description: | 4 | These are the flags used to select data from a database. 5 | flags: 6 | - name: where 7 | type: stringList 8 | help: Where clause 9 | default: [] 10 | - name: order-by 11 | type: string 12 | help: Order by clause 13 | default: "" 14 | - name: limit 15 | type: int 16 | help: Limit clause (0 for no limit) 17 | default: 50 18 | - name: offset 19 | type: int 20 | help: Offset clause 21 | default: 0 22 | - name: count 23 | type: bool 24 | help: Count rows 25 | default: false 26 | - name: columns 27 | type: stringList 28 | help: Columns to select 29 | default: [] 30 | - name: create-query 31 | type: string 32 | help: Output the query as yaml to use as a sqleton command 33 | default: "" 34 | - name: distinct 35 | type: bool 36 | help: Select distinct rows 37 | default: false 38 | - name: table 39 | type: string 40 | help: Table to select from 41 | required: true -------------------------------------------------------------------------------- /cmd/sqleton/cmds/mcp/mcp.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "os" 10 | 11 | "github.com/go-go-golems/clay/pkg/repositories" 12 | "github.com/go-go-golems/clay/pkg/repositories/mcp" 13 | "github.com/go-go-golems/clay/pkg/sql" 14 | "github.com/go-go-golems/glazed/pkg/cli" 15 | "github.com/go-go-golems/glazed/pkg/cmds" 16 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 17 | cmd_middlewares "github.com/go-go-golems/glazed/pkg/cmds/middlewares" 18 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 19 | "github.com/go-go-golems/glazed/pkg/cmds/runner" 20 | "github.com/go-go-golems/glazed/pkg/middlewares" 21 | "github.com/go-go-golems/glazed/pkg/settings" 22 | "github.com/go-go-golems/glazed/pkg/types" 23 | "github.com/go-go-golems/sqleton/pkg/flags" 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | type McpCommands struct { 28 | repositories []*repositories.Repository 29 | } 30 | 31 | func NewMcpCommands(repositories []*repositories.Repository) *McpCommands { 32 | return &McpCommands{ 33 | repositories: repositories, 34 | } 35 | } 36 | 37 | var McpCmd = &cobra.Command{ 38 | Use: "mcp", 39 | Short: "MCP (Machine Control Protocol) related commands", 40 | } 41 | 42 | type ListToolsCommand struct { 43 | *cmds.CommandDescription 44 | repositories []*repositories.Repository 45 | } 46 | 47 | func NewListToolsCommand(repositories []*repositories.Repository) (*ListToolsCommand, error) { 48 | glazedLayer, err := settings.NewGlazedParameterLayers() 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | return &ListToolsCommand{ 54 | CommandDescription: cmds.NewCommandDescription( 55 | "list", 56 | cmds.WithShort("List all available tools"), 57 | cmds.WithFlags( 58 | parameters.NewParameterDefinition( 59 | "repository", 60 | parameters.ParameterTypeString, 61 | parameters.WithHelp("Filter tools by repository name"), 62 | parameters.WithDefault(""), 63 | ), 64 | ), 65 | cmds.WithLayersList(glazedLayer), 66 | ), 67 | repositories: repositories, 68 | }, nil 69 | } 70 | 71 | func (c *ListToolsCommand) RunIntoGlazeProcessor( 72 | ctx context.Context, 73 | parsedLayers *layers.ParsedLayers, 74 | gp middlewares.Processor, 75 | ) error { 76 | s := &struct { 77 | Repository string `glazed.parameter:"repository"` 78 | }{} 79 | if err := parsedLayers.InitializeStruct(layers.DefaultSlug, s); err != nil { 80 | return err 81 | } 82 | 83 | allTools := []mcp.Tool{} 84 | for _, repo := range c.repositories { 85 | tools, _, err := repo.ListTools(ctx, s.Repository) 86 | if err != nil { 87 | return fmt.Errorf("error listing tools from repository: %w", err) 88 | } 89 | allTools = append(allTools, tools...) 90 | } 91 | 92 | for _, tool := range allTools { 93 | var prettySchema bytes.Buffer 94 | err := json.Indent(&prettySchema, tool.InputSchema, "", " ") 95 | if err != nil { 96 | return fmt.Errorf("error formatting input schema: %w", err) 97 | } 98 | 99 | var inputSchema_ interface{} 100 | 101 | output, _ := parsedLayers.GetParameter(settings.GlazedSlug, "output") 102 | if output.Value == "json" { 103 | inputSchema_ = tool.InputSchema 104 | } else { 105 | inputSchema_ = prettySchema.String() 106 | } 107 | 108 | row := map[string]interface{}{ 109 | "name": tool.Name, 110 | "description": tool.Description, 111 | "inputSchema": inputSchema_, 112 | } 113 | row_ := types.NewRowFromMap(row) 114 | if err := gp.AddRow(ctx, row_); err != nil { 115 | return err 116 | } 117 | } 118 | 119 | return nil 120 | } 121 | 122 | // createCommandMiddlewares creates the common middleware chain used by MCP commands 123 | func createCommandMiddlewares( 124 | parsedLayers *layers.ParsedLayers, 125 | cmd *cobra.Command, 126 | args []string, 127 | outputOverride cmd_middlewares.Middleware, 128 | ) ([]cmd_middlewares.Middleware, error) { 129 | // Start with cobra-specific middlewares 130 | middlewares_ := []cmd_middlewares.Middleware{ 131 | cmd_middlewares.ParseFromCobraCommand(cmd, 132 | parameters.WithParseStepSource("cobra"), 133 | ), 134 | cmd_middlewares.GatherArguments(args, 135 | parameters.WithParseStepSource("arguments"), 136 | ), 137 | } 138 | 139 | sqletonMiddlewares, err := sql.GetSqletonMiddlewares(parsedLayers) 140 | if err != nil { 141 | return nil, err 142 | } 143 | middlewares_ = append(middlewares_, sqletonMiddlewares...) 144 | 145 | // Add output override if provided 146 | if outputOverride != nil { 147 | middlewares_ = append(middlewares_, outputOverride) 148 | } 149 | 150 | return middlewares_, nil 151 | } 152 | 153 | func (mc *McpCommands) CreateToolsCmd() *cobra.Command { 154 | toolsCmd := &cobra.Command{ 155 | Use: "tools", 156 | Short: "Tool related commands", 157 | } 158 | 159 | listCmd, err := NewListToolsCommand(mc.repositories) 160 | if err != nil { 161 | panic(err) 162 | } 163 | 164 | // Create middleware to override output format to YAML 165 | outputOverride := cmd_middlewares.UpdateFromMap( 166 | map[string]map[string]interface{}{ 167 | "glazed": { 168 | "output": "json", 169 | }, 170 | }, 171 | parameters.WithParseStepSource("output-override"), 172 | ) 173 | 174 | // Build cobra command with custom middlewares 175 | cobraCmd, err := cli.BuildCobraCommandFromCommand(listCmd, 176 | cli.WithCobraMiddlewaresFunc(func( 177 | parsedLayers *layers.ParsedLayers, 178 | cmd *cobra.Command, 179 | args []string, 180 | ) ([]cmd_middlewares.Middleware, error) { 181 | return createCommandMiddlewares(parsedLayers, cmd, args, outputOverride) 182 | }), 183 | cli.WithCobraShortHelpLayers( 184 | layers.DefaultSlug, 185 | sql.DbtSlug, 186 | sql.SqlConnectionSlug, 187 | flags.SqlHelpersSlug, 188 | ), 189 | cli.WithProfileSettingsLayer(), 190 | ) 191 | if err != nil { 192 | panic(err) 193 | } 194 | 195 | toolsCmd.AddCommand(cobraCmd) 196 | 197 | runCmd := mc.CreateRunCmd() 198 | toolsCmd.AddCommand(runCmd) 199 | 200 | schemaCmd, err := NewSchemaCommand(mc.repositories) 201 | if err != nil { 202 | panic(err) 203 | } 204 | 205 | cobraSchemaCmd, err := cli.BuildCobraCommandFromCommand(schemaCmd, 206 | cli.WithCobraMiddlewaresFunc(func( 207 | parsedLayers *layers.ParsedLayers, 208 | cmd *cobra.Command, 209 | args []string, 210 | ) ([]cmd_middlewares.Middleware, error) { 211 | return createCommandMiddlewares(parsedLayers, cmd, args, nil) 212 | }), 213 | cli.WithCobraShortHelpLayers( 214 | layers.DefaultSlug, 215 | sql.DbtSlug, 216 | sql.SqlConnectionSlug, 217 | flags.SqlHelpersSlug, 218 | ), 219 | cli.WithProfileSettingsLayer(), 220 | ) 221 | if err != nil { 222 | panic(err) 223 | } 224 | 225 | toolsCmd.AddCommand(cobraSchemaCmd) 226 | 227 | return toolsCmd 228 | } 229 | 230 | // RunCommandSettings holds the parameters for the run command 231 | type RunCommandSettings struct { 232 | Name string `glazed.parameter:"name"` 233 | Args string `glazed.parameter:"args"` 234 | ArgsFromFile map[string]interface{} `glazed.parameter:"args-from-file"` 235 | } 236 | 237 | type RunCommand struct { 238 | *cmds.CommandDescription 239 | repositories []*repositories.Repository 240 | } 241 | 242 | func NewRunCommand(repositories []*repositories.Repository) (*RunCommand, error) { 243 | return &RunCommand{ 244 | CommandDescription: cmds.NewCommandDescription( 245 | "run", 246 | cmds.WithShort("Run a tool by name"), 247 | cmds.WithArguments( 248 | parameters.NewParameterDefinition( 249 | "name", 250 | parameters.ParameterTypeString, 251 | parameters.WithHelp("Name of the tool to run"), 252 | parameters.WithRequired(true), 253 | ), 254 | ), 255 | cmds.WithFlags( 256 | parameters.NewParameterDefinition( 257 | "args", 258 | parameters.ParameterTypeString, 259 | parameters.WithHelp("Arguments as JSON string"), 260 | parameters.WithDefault("{}"), 261 | ), 262 | parameters.NewParameterDefinition( 263 | "args-from-file", 264 | parameters.ParameterTypeObjectFromFile, 265 | parameters.WithHelp("Load arguments from JSON/YAML file"), 266 | ), 267 | ), 268 | ), 269 | repositories: repositories, 270 | }, nil 271 | } 272 | 273 | func (c *RunCommand) Run(ctx context.Context, parsedLayers *layers.ParsedLayers) error { 274 | // Parse settings 275 | s := &RunCommandSettings{} 276 | if err := parsedLayers.InitializeStruct(layers.DefaultSlug, s); err != nil { 277 | return err 278 | } 279 | 280 | // Find tool in repositories 281 | var foundCmd cmds.Command 282 | for _, repo := range c.repositories { 283 | cmd, ok := repo.GetCommand(s.Name) 284 | if ok { 285 | foundCmd = cmd 286 | break 287 | } 288 | } 289 | if foundCmd == nil { 290 | return fmt.Errorf("command %s not found", s.Name) 291 | } 292 | 293 | // Parse args string into map 294 | var argsMap map[string]interface{} 295 | if err := json.Unmarshal([]byte(s.Args), &argsMap); err != nil { 296 | return fmt.Errorf("failed to parse args JSON: %w", err) 297 | } 298 | 299 | // Merge with args from file if provided 300 | if s.ArgsFromFile != nil { 301 | for k, v := range s.ArgsFromFile { 302 | argsMap[k] = v 303 | } 304 | } 305 | 306 | sqletonMiddlewares, err := sql.GetSqletonMiddlewares(parsedLayers) 307 | if err != nil { 308 | return fmt.Errorf("failed to get sqleton middlewares: %w", err) 309 | } 310 | 311 | // Parse parameters using runner 312 | parsedToolLayers, err := runner.ParseCommandParameters( 313 | foundCmd, 314 | runner.WithValuesForLayers(map[string]map[string]interface{}{ 315 | layers.DefaultSlug: argsMap, 316 | }), 317 | runner.WithAdditionalMiddlewares(sqletonMiddlewares...), 318 | ) 319 | if err != nil { 320 | return fmt.Errorf("failed to parse tool parameters: %w", err) 321 | } 322 | 323 | // Run the command using the runner 324 | err = runner.RunCommand( 325 | ctx, 326 | foundCmd, 327 | parsedToolLayers, 328 | runner.WithWriter(os.Stdout), // For WriterCommand 329 | ) 330 | if err != nil { 331 | return fmt.Errorf("failed to run tool: %w", err) 332 | } 333 | 334 | return nil 335 | } 336 | 337 | func (mc *McpCommands) CreateRunCmd() *cobra.Command { 338 | runCmd, err := NewRunCommand(mc.repositories) 339 | if err != nil { 340 | panic(err) 341 | } 342 | 343 | // Build cobra command with custom middlewares 344 | cobraCmd, err := cli.BuildCobraCommandFromCommand(runCmd, 345 | cli.WithCobraMiddlewaresFunc(func( 346 | parsedLayers *layers.ParsedLayers, 347 | cmd *cobra.Command, 348 | args []string, 349 | ) ([]cmd_middlewares.Middleware, error) { 350 | return createCommandMiddlewares(parsedLayers, cmd, args, nil) 351 | }), 352 | cli.WithCobraShortHelpLayers( 353 | layers.DefaultSlug, 354 | sql.DbtSlug, 355 | sql.SqlConnectionSlug, 356 | flags.SqlHelpersSlug, 357 | ), 358 | cli.WithProfileSettingsLayer(), 359 | ) 360 | if err != nil { 361 | panic(err) 362 | } 363 | 364 | return cobraCmd 365 | } 366 | 367 | type SchemaCommand struct { 368 | *cmds.CommandDescription 369 | repositories []*repositories.Repository 370 | } 371 | 372 | func NewSchemaCommand(repositories []*repositories.Repository) (*SchemaCommand, error) { 373 | return &SchemaCommand{ 374 | CommandDescription: cmds.NewCommandDescription( 375 | "schema", 376 | cmds.WithShort("Get JSON schema for a tool"), 377 | cmds.WithArguments( 378 | parameters.NewParameterDefinition( 379 | "name", 380 | parameters.ParameterTypeString, 381 | parameters.WithHelp("Name of the tool to get schema for"), 382 | ), 383 | ), 384 | ), 385 | repositories: repositories, 386 | }, nil 387 | } 388 | 389 | func (c *SchemaCommand) RunIntoWriter( 390 | ctx context.Context, 391 | parsedLayers *layers.ParsedLayers, 392 | w io.Writer, 393 | ) error { 394 | s := &struct { 395 | Name string `glazed.parameter:"name"` 396 | }{} 397 | if err := parsedLayers.InitializeStruct(layers.DefaultSlug, s); err != nil { 398 | return err 399 | } 400 | 401 | // Find tool in repositories 402 | var foundCmd cmds.Command 403 | for _, repo := range c.repositories { 404 | cmd, ok := repo.GetCommand(s.Name) 405 | if ok { 406 | foundCmd = cmd 407 | break 408 | } 409 | } 410 | if foundCmd == nil { 411 | return fmt.Errorf("command %s not found", s.Name) 412 | } 413 | 414 | // Get JSON schema from command description 415 | schema, err := foundCmd.Description().ToJsonSchema() 416 | if err != nil { 417 | return fmt.Errorf("failed to get schema: %w", err) 418 | } 419 | 420 | // Pretty print the schema 421 | encoder := json.NewEncoder(w) 422 | encoder.SetIndent("", " ") 423 | if err := encoder.Encode(schema); err != nil { 424 | return fmt.Errorf("failed to encode schema: %w", err) 425 | } 426 | 427 | return nil 428 | } 429 | 430 | func (mc *McpCommands) AddToRootCommand(rootCmd *cobra.Command) { 431 | toolsCmd := mc.CreateToolsCmd() 432 | McpCmd.AddCommand(toolsCmd) 433 | rootCmd.AddCommand(McpCmd) 434 | } 435 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/query.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | "github.com/go-go-golems/clay/pkg/sql" 6 | "github.com/go-go-golems/glazed/pkg/cmds" 7 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 8 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 9 | "github.com/go-go-golems/glazed/pkg/middlewares" 10 | "github.com/go-go-golems/glazed/pkg/settings" 11 | "github.com/jmoiron/sqlx" 12 | ) 13 | 14 | type QueryCommand struct { 15 | dbConnectionFactory sql.DBConnectionFactory 16 | *cmds.CommandDescription 17 | } 18 | 19 | var _ cmds.GlazeCommand = (*QueryCommand)(nil) 20 | 21 | func NewQueryCommand( 22 | dbConnectionFactory sql.DBConnectionFactory, 23 | options ...cmds.CommandDescriptionOption, 24 | ) (*QueryCommand, error) { 25 | glazeParameterLayer, err := settings.NewGlazedParameterLayers() 26 | if err != nil { 27 | return nil, err 28 | } 29 | options_ := append([]cmds.CommandDescriptionOption{ 30 | cmds.WithShort("Run a SQL query passed as a CLI argument"), 31 | cmds.WithArguments(parameters.NewParameterDefinition( 32 | "query", 33 | parameters.ParameterTypeString, 34 | parameters.WithHelp("The SQL query to run"), 35 | parameters.WithRequired(true), 36 | ), 37 | ), 38 | cmds.WithLayersList(glazeParameterLayer), 39 | }, options...) 40 | 41 | return &QueryCommand{ 42 | dbConnectionFactory: dbConnectionFactory, 43 | CommandDescription: cmds.NewCommandDescription("query", options_...), 44 | }, nil 45 | } 46 | 47 | type QuerySettings struct { 48 | Query string `glazed.parameter:"query"` 49 | } 50 | 51 | func (q *QueryCommand) RunIntoGlazeProcessor( 52 | ctx context.Context, 53 | parsedLayers *layers.ParsedLayers, 54 | gp middlewares.Processor, 55 | ) error { 56 | d := parsedLayers.GetDefaultParameterLayer() 57 | s := &QuerySettings{} 58 | err := d.InitializeStruct(s) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | db, err := q.dbConnectionFactory(parsedLayers) 64 | if err != nil { 65 | return err 66 | } 67 | defer func(db *sqlx.DB) { 68 | _ = db.Close() 69 | }(db) 70 | 71 | err = db.PingContext(ctx) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | err = sql.RunNamedQueryIntoGlaze(ctx, db, s.Query, map[string]interface{}{}, gp) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/run.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | "github.com/go-go-golems/clay/pkg/sql" 6 | "github.com/go-go-golems/glazed/pkg/cmds" 7 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 8 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 9 | "github.com/go-go-golems/glazed/pkg/middlewares" 10 | cli "github.com/go-go-golems/glazed/pkg/settings" 11 | "github.com/go-go-golems/sqleton/pkg/flags" 12 | "github.com/jmoiron/sqlx" 13 | "github.com/pkg/errors" 14 | 15 | "github.com/spf13/cobra" 16 | "io" 17 | "os" 18 | ) 19 | 20 | type RunCommand struct { 21 | *cmds.CommandDescription 22 | dbConnectionFactory sql.DBConnectionFactory 23 | } 24 | 25 | var _ cmds.GlazeCommand = (*RunCommand)(nil) 26 | 27 | type RunSettings struct { 28 | InputFiles []string `glazed.parameter:"input-files"` 29 | } 30 | 31 | func (c *RunCommand) RunIntoGlazeProcessor( 32 | ctx context.Context, 33 | parsedLayers *layers.ParsedLayers, 34 | gp middlewares.Processor) error { 35 | 36 | s := &RunSettings{} 37 | err := parsedLayers.InitializeStruct(layers.DefaultSlug, s) 38 | if err != nil { 39 | return err 40 | } 41 | ss := &flags.SqlHelpersSettings{} 42 | err = parsedLayers.InitializeStruct(flags.SqlHelpersSlug, ss) 43 | if err != nil { 44 | return errors.Wrap(err, "could not initialize sql-helpers settings") 45 | } 46 | 47 | db, err := c.dbConnectionFactory(parsedLayers) 48 | if err != nil { 49 | return errors.Wrap(err, "could not open database") 50 | } 51 | defer func(db *sqlx.DB) { 52 | _ = db.Close() 53 | }(db) 54 | 55 | err = db.PingContext(ctx) 56 | if err != nil { 57 | return errors.Wrapf(err, "Could not ping database") 58 | } 59 | 60 | for _, arg := range s.InputFiles { 61 | query := "" 62 | 63 | if arg == "-" { 64 | inBytes, err := io.ReadAll(os.Stdin) 65 | cobra.CheckErr(err) 66 | query = string(inBytes) 67 | } else { 68 | // read file 69 | queryBytes, err := os.ReadFile(arg) 70 | cobra.CheckErr(err) 71 | 72 | query = string(queryBytes) 73 | } 74 | 75 | if ss.Explain { 76 | query = "EXPLAIN " + query 77 | } 78 | 79 | // TODO(2022-12-20, manuel): collect named parameters here, maybe through prerun? 80 | // See: https://github.com/wesen/sqleton/issues/40 81 | err = sql.RunNamedQueryIntoGlaze(ctx, db, query, map[string]interface{}{}, gp) 82 | cobra.CheckErr(err) 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func NewRunCommand( 89 | dbConnectionFactory sql.DBConnectionFactory, 90 | options ...cmds.CommandDescriptionOption, 91 | ) (*RunCommand, error) { 92 | glazedParameterLayer, err := cli.NewGlazedParameterLayers() 93 | if err != nil { 94 | return nil, errors.Wrap(err, "could not create Glazed parameter layer") 95 | } 96 | sqlHelpersParameterLayer, err := flags.NewSqlHelpersParameterLayer() 97 | if err != nil { 98 | return nil, errors.Wrap(err, "could not create SQL helpers parameter layer") 99 | } 100 | 101 | options_ := append([]cmds.CommandDescriptionOption{ 102 | cmds.WithShort("Run a SQL query from sql files"), 103 | cmds.WithArguments( 104 | parameters.NewParameterDefinition( 105 | "input-files", 106 | parameters.ParameterTypeStringList, 107 | parameters.WithRequired(true), 108 | ), 109 | ), 110 | cmds.WithLayersList( 111 | glazedParameterLayer, 112 | sqlHelpersParameterLayer, 113 | ), 114 | }, options...) 115 | 116 | return &RunCommand{ 117 | dbConnectionFactory: dbConnectionFactory, 118 | CommandDescription: cmds.NewCommandDescription( 119 | "run", 120 | options_..., 121 | ), 122 | }, nil 123 | } 124 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/select.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | _ "embed" 6 | "fmt" 7 | sql2 "github.com/go-go-golems/clay/pkg/sql" 8 | "github.com/go-go-golems/glazed/pkg/cmds" 9 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 10 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 11 | "github.com/go-go-golems/glazed/pkg/helpers/cast" 12 | "github.com/go-go-golems/glazed/pkg/middlewares" 13 | "github.com/go-go-golems/glazed/pkg/settings" 14 | cmds2 "github.com/go-go-golems/sqleton/pkg/cmds" 15 | "github.com/go-go-golems/sqleton/pkg/flags" 16 | "github.com/huandu/go-sqlbuilder" 17 | "github.com/jmoiron/sqlx" 18 | "github.com/pkg/errors" 19 | "gopkg.in/yaml.v3" 20 | "strings" 21 | ) 22 | 23 | //go:embed "flags/select.yaml" 24 | var selectFlagsYaml []byte 25 | 26 | const SelectSlug = "select" 27 | 28 | func NewSelectParameterLayer() (*layers.ParameterLayerImpl, error) { 29 | ret := &layers.ParameterLayerImpl{} 30 | err := ret.LoadFromYAML(selectFlagsYaml) 31 | if err != nil { 32 | return nil, errors.Wrap(err, "Failed to initialize select parameter layer") 33 | } 34 | return ret, nil 35 | } 36 | 37 | type SelectCommand struct { 38 | *cmds.CommandDescription 39 | dbConnectionFactory sql2.DBConnectionFactory 40 | } 41 | 42 | type SelectCommandSettings struct { 43 | Columns []string `glazed.parameter:"columns"` 44 | Limit int `glazed.parameter:"limit"` 45 | Offset int `glazed.parameter:"offset"` 46 | Count bool `glazed.parameter:"count"` 47 | Where []string `glazed.parameter:"where"` 48 | OrderBy string `glazed.parameter:"order-by"` 49 | Distinct bool `glazed.parameter:"distinct"` 50 | Table string `glazed.parameter:"table"` 51 | CreateQuery string `glazed.parameter:"create-query"` 52 | } 53 | 54 | var _ cmds.GlazeCommand = (*SelectCommand)(nil) 55 | 56 | func (sc *SelectCommand) RunIntoGlazeProcessor( 57 | ctx context.Context, 58 | parsedLayers *layers.ParsedLayers, 59 | gp middlewares.Processor, 60 | ) error { 61 | s := &SelectCommandSettings{} 62 | err := parsedLayers.InitializeStruct(SelectSlug, s) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | ss := &flags.SqlHelpersSettings{} 68 | err = parsedLayers.InitializeStruct(flags.SqlHelpersSlug, ss) 69 | if err != nil { 70 | return errors.Wrap(err, "could not initialize sql-helpers settings") 71 | } 72 | 73 | sb := sqlbuilder.NewSelectBuilder() 74 | sb = sb.From(s.Table) 75 | 76 | if s.Count { 77 | countColumns := strings.Join(s.Columns, ", ") 78 | if s.Distinct { 79 | countColumns = "DISTINCT " + countColumns 80 | } 81 | s.Columns = []string{sb.As(fmt.Sprintf("COUNT(%s)", countColumns), "count")} 82 | } else { 83 | if len(s.Columns) == 0 { 84 | s.Columns = []string{"*"} 85 | } 86 | } 87 | sb = sb.Select(s.Columns...) 88 | if s.Distinct && !s.Count { 89 | sb = sb.Distinct() 90 | } 91 | 92 | for _, where := range s.Where { 93 | sb = sb.Where(where) 94 | } 95 | 96 | if s.Limit > 0 && !s.Count { 97 | sb = sb.Limit(s.Limit) 98 | } 99 | if s.Offset > 0 { 100 | sb = sb.Offset(s.Offset) 101 | } 102 | if s.OrderBy != "" { 103 | sb = sb.OrderBy(s.OrderBy) 104 | } 105 | 106 | if s.CreateQuery != "" { 107 | short := fmt.Sprintf("Select"+" columns from %s", s.Table) 108 | if s.Count { 109 | short = fmt.Sprintf("Count all rows from %s", s.Table) 110 | } 111 | if len(s.Where) > 0 { 112 | short = fmt.Sprintf("Select"+" from %s where %s", s.Table, strings.Join(s.Where, " AND ")) 113 | } 114 | 115 | flags := []*parameters.ParameterDefinition{} 116 | if len(s.Where) == 0 { 117 | flags = append(flags, ¶meters.ParameterDefinition{ 118 | Name: "where", 119 | Type: parameters.ParameterTypeStringList, 120 | }) 121 | } 122 | if !s.Count { 123 | flags = append(flags, ¶meters.ParameterDefinition{ 124 | Name: "limit", 125 | Type: parameters.ParameterTypeInteger, 126 | Help: fmt.Sprintf("Limit the number of rows (default: %d), set to 0 to disable", s.Limit), 127 | Default: cast.InterfaceAddr(s.Limit), 128 | }) 129 | flags = append(flags, ¶meters.ParameterDefinition{ 130 | Name: "offset", 131 | Type: parameters.ParameterTypeInteger, 132 | Help: fmt.Sprintf("Offset the number of rows (default: %d)", s.Offset), 133 | Default: cast.InterfaceAddr(s.Offset), 134 | }) 135 | flags = append(flags, ¶meters.ParameterDefinition{ 136 | Name: "distinct", 137 | Type: parameters.ParameterTypeBool, 138 | Help: fmt.Sprintf("Whether to select distinct rows (default: %t)", s.Distinct), 139 | Default: cast.InterfaceAddr(s.Distinct), 140 | }) 141 | 142 | orderByHelp := "Order by" 143 | var orderDefault interface{} 144 | if s.OrderBy != "" { 145 | orderByHelp = fmt.Sprintf("Order by (default: %s)", s.OrderBy) 146 | orderDefault = s.OrderBy 147 | } 148 | flags = append(flags, ¶meters.ParameterDefinition{ 149 | Name: "order_by", 150 | Type: parameters.ParameterTypeString, 151 | Help: orderByHelp, 152 | Default: cast.InterfaceAddr(orderDefault), 153 | }) 154 | } 155 | 156 | sb := &strings.Builder{} 157 | _, _ = fmt.Fprintf(sb, "SELECT ") 158 | if !s.Count { 159 | _, _ = fmt.Fprintf(sb, "{{ if .distinct }}DISTINCT{{ end }} ") 160 | } 161 | _, _ = fmt.Fprintf(sb, "%s FROM %s", strings.Join(s.Columns, ", "), s.Table) 162 | if len(s.Where) > 0 { 163 | _, _ = fmt.Fprintf(sb, " WHERE %s", strings.Join(s.Where, " AND ")) 164 | } else { 165 | _, _ = fmt.Fprintf(sb, "\nWHERE 1=1\n{{ range .where }} AND {{.}} {{ end }}") 166 | } 167 | 168 | _, _ = fmt.Fprintf(sb, "\n{{ if .order_by }} ORDER BY {{ .order_by }}{{ end }}") 169 | _, _ = fmt.Fprintf(sb, "\n{{ if .limit }} LIMIT {{ .limit }}{{ end }}") 170 | _, _ = fmt.Fprintf(sb, "\nOFFSET {{ .offset }}") 171 | 172 | query := sb.String() 173 | sqlCommand, err := cmds2.NewSqlCommand( 174 | cmds.NewCommandDescription(s.CreateQuery, 175 | cmds.WithShort(short), cmds.WithFlags(flags...)), 176 | cmds2.WithDbConnectionFactory(sql2.OpenDatabaseFromDefaultSqlConnectionLayer), 177 | cmds2.WithQuery(query), 178 | ) 179 | if err != nil { 180 | return err 181 | } 182 | 183 | // marshal to yaml 184 | yamlBytes, err := yaml.Marshal(sqlCommand) 185 | if err != nil { 186 | return err 187 | } 188 | 189 | fmt.Println(string(yamlBytes)) 190 | return nil 191 | } 192 | 193 | query, queryArgs := sb.Build() 194 | 195 | if ss.PrintQuery { 196 | fmt.Println(query) 197 | if len(queryArgs) > 0 { 198 | fmt.Println("Args:") 199 | fmt.Println(queryArgs) 200 | } 201 | return nil 202 | } 203 | 204 | db, err := sc.dbConnectionFactory(parsedLayers) 205 | if err != nil { 206 | return err 207 | } 208 | defer func(db *sqlx.DB) { 209 | _ = db.Close() 210 | }(db) 211 | 212 | err = db.PingContext(ctx) 213 | if err != nil { 214 | return err 215 | } 216 | 217 | err = sql2.RunQueryIntoGlaze(ctx, db, query, queryArgs, gp) 218 | if err != nil { 219 | return err 220 | } 221 | return nil 222 | } 223 | 224 | func NewSelectCommand( 225 | dbConnectionFactory sql2.DBConnectionFactory, 226 | options ...cmds.CommandDescriptionOption, 227 | ) (*SelectCommand, error) { 228 | glazedParameterLayer, err := settings.NewGlazedParameterLayers() 229 | if err != nil { 230 | return nil, errors.Wrap(err, "could not create Glazed parameter layer") 231 | } 232 | sqlHelpersParameterLayer, err := flags.NewSqlHelpersParameterLayer() 233 | if err != nil { 234 | return nil, errors.Wrap(err, "could not create SQL helpers parameter layer") 235 | } 236 | selectParameterLayer, err := NewSelectParameterLayer() 237 | if err != nil { 238 | return nil, errors.Wrap(err, "could not create select parameter layer") 239 | } 240 | 241 | options_ := append([]cmds.CommandDescriptionOption{ 242 | cmds.WithShort("Select" + " all columns from a table"), 243 | cmds.WithLayersList( 244 | selectParameterLayer, 245 | glazedParameterLayer, 246 | sqlHelpersParameterLayer, 247 | ), 248 | }, options...) 249 | 250 | return &SelectCommand{ 251 | dbConnectionFactory: dbConnectionFactory, 252 | CommandDescription: cmds.NewCommandDescription( 253 | "select", 254 | options_..., 255 | ), 256 | }, nil 257 | } 258 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-go-golems/sqleton/7c4bc3869206a2c125ce96faf01909ebcff16b59/cmd/sqleton/cmds/static/favicon.ico -------------------------------------------------------------------------------- /cmd/sqleton/cmds/test-config-explicit-renderer.yaml: -------------------------------------------------------------------------------- 1 | routes: 2 | # landing page 3 | - path: / 4 | template: 5 | templateFile: ~/code/ttc/ttc/sql/sqleton/index.analytics.md 6 | # content pages 7 | - path: / 8 | templateDirectory: 9 | localDirectory: ~/code/ttc/ttc/sql/sqleton 10 | # static content 11 | - path: /static 12 | static: 13 | localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/static 14 | # - path: /dist 15 | # static: 16 | # localPath: ~/code/wesen/corporate-headquarters/parka/pkg/server/web/dist 17 | # commands 18 | - path: / 19 | commandDirectory: 20 | repositories: 21 | - ~/.sqleton/repositories 22 | - ~/code/ttc/ttc/sql/sqleton 23 | # maybe we could allow multiple directories here 24 | # we also need to find a syntax to add embedded or default template lookup options 25 | templateDirectory: ~/code/wesen/corporate-headquarters/parka/pkg/render/datatables/templates 26 | 27 | # need to pass template path here 28 | # localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/templates 29 | defaults: 30 | renderer: 31 | templateDirectory: ~/code/wesen/corporate-headquarters/parka/pkg/server/web/src/templates 32 | markdownBaseTemplateName: base.tmpl.html 33 | 34 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/test-config-no-renderer.yaml: -------------------------------------------------------------------------------- 1 | routes: 2 | # landing page 3 | - path: / 4 | template: 5 | templateFile: ~/code/ttc/ttc/sql/sqleton/index.analytics.md 6 | # content pages 7 | - path: / 8 | templateDirectory: 9 | localDirectory: ~/code/ttc/ttc/sql/sqleton 10 | # static content 11 | - path: /static 12 | static: 13 | localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/static 14 | # commands 15 | - path: / 16 | commandDirectory: 17 | repositories: 18 | - ~/.sqleton/repositories 19 | - ~/code/ttc/ttc/sql/sqleton 20 | # maybe we could allow multiple directories here 21 | # we also need to find a syntax to add embedded or default template lookup options 22 | templateDirectory: ~/code/wesen/corporate-headquarters/parka/pkg/render/datatables/templates 23 | 24 | # need to pass template path here 25 | # localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/templates 26 | defaults: 27 | renderer: 28 | useDefaultParkaRenderer: false 29 | 30 | 31 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/test-config-no-static.yaml: -------------------------------------------------------------------------------- 1 | routes: 2 | # landing page 3 | - path: / 4 | template: 5 | templateFile: ~/code/ttc/ttc/sql/sqleton/index.analytics.md 6 | # content pages 7 | - path: / 8 | templateDirectory: 9 | localDirectory: ~/code/ttc/ttc/sql/sqleton 10 | # static content 11 | - path: /static 12 | static: 13 | localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/static 14 | # commands 15 | - path: / 16 | commandDirectory: 17 | repositories: 18 | - ~/.sqleton/repositories 19 | - ~/code/ttc/ttc/sql/sqleton 20 | # maybe we could allow multiple directories here 21 | # we also need to find a syntax to add embedded or default template lookup options 22 | templateDirectory: ~/code/wesen/corporate-headquarters/parka/pkg/render/datatables/templates 23 | 24 | # need to pass template path here 25 | # localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/templates 26 | defaults: 27 | useParkaStaticFiles: false 28 | 29 | 30 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/test-config-simplest.yaml: -------------------------------------------------------------------------------- 1 | routes: 2 | # landing page 3 | - path: / 4 | template: 5 | templateFile: ~/code/ttc/ttc/sql/sqleton/index.analytics.md 6 | # content pages 7 | - path: / 8 | templateDirectory: 9 | localDirectory: ~/code/ttc/ttc/sql/sqleton 10 | # commands 11 | - path: / 12 | commandDirectory: 13 | includeDefaultRepositories: true 14 | -------------------------------------------------------------------------------- /cmd/sqleton/cmds/test-config.yaml: -------------------------------------------------------------------------------- 1 | routes: 2 | # landing page 3 | - path: / 4 | template: 5 | templateFile: ~/code/ttc/ttc/sql/sqleton/index.analytics.md 6 | # content pages 7 | - path: / 8 | templateDirectory: 9 | localDirectory: ~/code/ttc/ttc/sql/sqleton 10 | # static content 11 | - path: /static 12 | static: 13 | localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/static 14 | # commands 15 | - path: / 16 | commandDirectory: 17 | includeDefaultRepositories: true 18 | repositories: 19 | - ~/code/ttc/ttc/sql/sqleton 20 | 21 | templateLookup: 22 | directories: 23 | - ~/code/wesen/corporate-headquarters/parka/pkg/glazed/handlers/datatables/templates 24 | indexTemplateName: index.tmpl.html 25 | defaults: 26 | flags: 27 | limit: 1337 28 | layers: 29 | glazed: 30 | filter: 31 | - id 32 | overrides: 33 | layers: 34 | dbt: 35 | dbt-profile: ttc.analytics 36 | glazed: 37 | # these don't work yet because of the lack of row level middleware (which would apply the filtering) 38 | filter: 39 | - quantity_sold 40 | - sales_usd 41 | additionalData: 42 | foobar: baz 43 | - path: /analytics 44 | commandDirectory: 45 | includeDefaultRepositories: false 46 | repositories: 47 | - ~/code/ttc/ttc/sql/sqleton 48 | - path: /foobar 49 | commandDirectory: 50 | repositories: 51 | - ~/.sqleton/repositories 52 | - ~/code/ttc/ttc/sql/sqleton 53 | overrides: 54 | layers: 55 | dbt: 56 | dbt-profile: ttc.foobar 57 | - path: /prod 58 | commandDirectory: 59 | repositories: 60 | - ~/.sqleton/repositories 61 | - ~/code/ttc/ttc/sql/sqleton 62 | overrides: 63 | layers: 64 | dbt: 65 | dbt-profile: ttc.prod 66 | 67 | # need to pass template path here 68 | # localPath: ~/code/wesen/corporate-headquarters/sqleton/cmd/sqleton/cmd/templates 69 | 70 | 71 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/applications/01-mysql-get-distinct-connected-users.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Get the list of currently connected MySQL users 3 | Slug: mysql-distinct-connected-users 4 | Short: Show the list of connected users 5 | Topics: 6 | - sqleton 7 | - mysql 8 | Commands: 9 | - select 10 | Flags: 11 | - distinct 12 | - columns 13 | IsTemplate: false 14 | IsTopLevel: true 15 | ShowPerDefault: true 16 | SectionType: Application 17 | --- 18 | 19 | You can get the list of currently connected users on a MySQL database with: 20 | 21 | ``` 22 | ❯ sqleton select --table information_schema.processlist --distinct --columns USER 23 | +-------------------+ 24 | | USER | 25 | +-------------------+ 26 | | ttc_analytics_dev | 27 | +-------------------+ 28 | ``` -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/01-help-example.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show the list of all toplevel topics 3 | Slug: help-example-1 4 | Short: | 5 | ``` 6 | sqleton help --list 7 | ``` 8 | Topics: 9 | - help-system 10 | Commands: 11 | - help 12 | Flags: 13 | - list 14 | IsTemplate: false 15 | IsTopLevel: false 16 | ShowPerDefault: true 17 | SectionType: Example 18 | --- 19 | You can ask the help system to list all toplevel topics (not just the default ones) in 20 | a concise list. 21 | 22 | --- 23 | 24 | ``` 25 | ❯ sqleton help --list 26 | 27 | sqleton - sqleton runs SQL queries out of template files 28 | 29 | For more help, run: sqleton help sqleton 30 | 31 | ## General topics 32 | 33 | Run sqleton help to view a topic's page. 34 | 35 | • help-system - Help System 36 | 37 | ## Examples 38 | 39 | Run sqleton help to view an example in full. 40 | 41 | • help-example-1 - Show the list of all toplevel topics 42 | • ls-dbt-profiles - Show the list of all dbt profiles 43 | ``` 44 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/02-list-dbt.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show the list of all dbt profiles 3 | Slug: ls-dbt-profiles 4 | Short: | 5 | ``` 6 | sqleton db ls --use-dbt-profiles 7 | ``` 8 | Topics: 9 | - dbt 10 | - config 11 | - database-sources 12 | Commands: 13 | - ls 14 | - db 15 | Flags: 16 | - use-dbt-profiles 17 | IsTemplate: false 18 | IsTopLevel: false 19 | ShowPerDefault: true 20 | SectionType: Example 21 | --- 22 | You can ask sqleton to list all dbt profiles it is able to find. 23 | 24 | Don't forget to enable `--use-dbt-profiles`. Use `--dbt-profiles-path` to use another file. 25 | 26 | --- 27 | 28 | ``` 29 | ❯ sqleton db ls --use-dbt-profiles --fields name,type,hostname,database 30 | +---------------------+-------+-----------+-------------------+ 31 | | name | type | hostname | database | 32 | +---------------------+-------+-----------+-------------------+ 33 | | localhost.localhost | mysql | localhost | ttc_analytics | 34 | | ttc.prod | mysql | localhost | ttc_analytics | 35 | | prod.prod | mysql | localhost | ttc_analytics | 36 | | dev.dev | mysql | localhost | ttc_dev_analytics | 37 | +---------------------+-------+-----------+-------------------+ 38 | ``` 39 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/mysql/01-mysql-ps.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show the process list of a mysql server 3 | Slug: mysql-ps 4 | Short: | 5 | ``` 6 | sqleton mysql ps 7 | ``` 8 | Topics: 9 | - mysql 10 | Commands: 11 | - mysql 12 | - ps 13 | IsTemplate: false 14 | IsTopLevel: false 15 | ShowPerDefault: true 16 | SectionType: Example 17 | --- 18 | You can use sqleton to run `show processlist` on a mysql server, 19 | and use the full set of glazed flags. 20 | 21 | This command bundles the actual query directly inside sqleton, instead of 22 | loading it from a file as shown in `03-run-show-process-list` 23 | 24 | ``` 25 | ❯ sqleton mysql ps --fields User,Host,Command,Info 26 | +-----------------+------------------+---------+------------------+ 27 | | User | Host | Command | Info | 28 | +-----------------+------------------+---------+------------------+ 29 | | event_scheduler | localhost | Daemon | | 30 | | ttc | 172.20.0.7:41054 | Sleep | | 31 | | ttc | 172.20.0.7:41058 | Sleep | | 32 | | root | 172.20.0.1:61900 | Query | SHOW PROCESSLIST | 33 | +-----------------+------------------+---------+------------------+ 34 | ``` 35 | 36 | ``` 37 | ❯ sqleton mysql ps --select Id 38 | 4 39 | 29 40 | 30 41 | 549 42 | ``` 43 | 44 | ``` 45 | ❯ sqleton mysql ps --select-template "{{.Id}} -- {{.User}}" 46 | 4 -- event_scheduler 47 | 4084 -- root 48 | ``` 49 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/run-select-query/01-run-show-process-list.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show the process list of a mysql server 3 | Slug: mysql-show-processlist 4 | Short: | 5 | ``` 6 | sqleton run examples/show-processlist.sql 7 | ``` 8 | Topics: 9 | - mysql 10 | Commands: 11 | - run 12 | IsTemplate: false 13 | IsTopLevel: true 14 | ShowPerDefault: true 15 | SectionType: Example 16 | --- 17 | You can use sqleton to run `show processlist` on a mysql server, 18 | and use the full set of glazed flags. 19 | 20 | ``` 21 | ❯ sqleton run examples/show-processlist.sql --fields User,Host,Command,Info 22 | +-----------------+------------------+---------+------------------+ 23 | | User | Host | Command | Info | 24 | +-----------------+------------------+---------+------------------+ 25 | | event_scheduler | localhost | Daemon | | 26 | | ttc | 172.20.0.7:41054 | Sleep | | 27 | | ttc | 172.20.0.7:41058 | Sleep | | 28 | | root | 172.20.0.1:61900 | Query | SHOW PROCESSLIST | 29 | +-----------------+------------------+---------+------------------+ 30 | ``` 31 | 32 | ``` 33 | ❯ sqleton run examples/show-processlist.sql --select Id 34 | 4 35 | 29 36 | 30 37 | 549 38 | ``` 39 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/run-select-query/02-query-cli.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Run a SQL query from the CLI 3 | Slug: query-file 4 | Short: | 5 | ``` 6 | sqleton query "SHOW PROCESSLIST" 7 | ``` 8 | Topics: 9 | - mysql 10 | Commands: 11 | - query 12 | IsTemplate: false 13 | IsTopLevel: true 14 | ShowPerDefault: true 15 | SectionType: Example 16 | --- 17 | You can use sqleton to run a query straight from the CLI. 18 | and use the full set of glazed flags. 19 | 20 | ``` 21 | ❯ sqleton run examples/show-processlist.sql --fields User,Host,Command,Info 22 | +-----------------+------------------+---------+------------------+ 23 | | User | Host | Command | Info | 24 | +-----------------+------------------+---------+------------------+ 25 | | event_scheduler | localhost | Daemon | | 26 | | ttc | 172.20.0.7:41054 | Sleep | | 27 | | ttc | 172.20.0.7:41058 | Sleep | | 28 | | root | 172.20.0.1:61900 | Query | SHOW PROCESSLIST | 29 | +-----------------+------------------+---------+------------------+ 30 | ``` 31 | 32 | ``` 33 | ❯ sqleton run examples/show-processlist.sql --select Id 34 | 4 35 | 29 36 | 30 37 | 549 38 | ``` 39 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/run-select-query/03-run-stdin.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Run a SQL query passed through stdin 3 | Slug: run-stdin 4 | Short: | 5 | ``` 6 | echo "SHOW PROCESSLIST" | sqleton run --fields Id,User,Host - 7 | ``` 8 | Topics: 9 | - mysql 10 | Commands: 11 | - run 12 | IsTemplate: false 13 | IsTopLevel: true 14 | ShowPerDefault: false 15 | SectionType: Example 16 | --- 17 | You can pass queries into `sqleton run` by passing "-" as filename. 18 | 19 | ``` 20 | ❯ echo "SHOW PROCESSLIST" | sqleton run --fields Id,User,Host - 21 | +----------+-------------------+-------------------+ 22 | | Id | User | Host | 23 | +----------+-------------------+-------------------+ 24 | | 39636346 | ttc_analytics_dev | 172.31.20.6:40120 | 25 | +----------+-------------------+-------------------+ 26 | ``` 27 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/run-select-query/04-select-table.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Quickly select a table 3 | Slug: select-table 4 | Short: | 5 | ``` 6 | sqleton select --table orders \ 7 | --columns order_number,status,totals \ 8 | --limit 10 --order-by "order_number DESC" 9 | ``` 10 | Topics: 11 | - mysql 12 | Commands: 13 | - select 14 | Flags: 15 | - fields 16 | - limit 17 | - order-by 18 | IsTemplate: false 19 | IsTopLevel: true 20 | ShowPerDefault: true 21 | SectionType: Example 22 | --- 23 | You can use sqleton to run a query straight from the CLI. 24 | and use the full set of glazed flags. 25 | 26 | ``` 27 | ❯ sqleton select --table orders \ 28 | --columns order_number,status,totals \ 29 | --limit 10 --order-by "order_number DESC" 30 | +--------------+--------------+--------+ 31 | | status | order_number | totals | 32 | +--------------+--------------+--------+ 33 | | wc-completed | 8002 | 49.45 | 34 | | wc-completed | 7968 | 395.00 | 35 | | wc-completed | 7967 | 128.95 | 36 | | wc-completed | 7966 | 88.95 | 37 | | wc-completed | 7956 | 79.45 | 38 | | wc-completed | 7954 | 136.69 | 39 | | wc-cancelled | 7953 | 136.69 | 40 | | wc-completed | 7944 | 108.95 | 41 | | wc-completed | 7943 | 103.50 | 42 | | wc-cancelled | 7937 | 157.50 | 43 | +--------------+--------------+--------+ 44 | ``` 45 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/run-select-query/05-create-select-query.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Quickly create a query template 3 | Slug: select-create-query 4 | Short: | 5 | ``` 6 | sqleton select --table orders \ 7 | --columns order_number,status,totals \ 8 | --limit 10 --order-by "order_number DESC" \ 9 | --create-query orders 10 | ``` 11 | Topics: 12 | - mysql 13 | - queries 14 | Commands: 15 | - select 16 | - queries 17 | Flags: 18 | - create-query 19 | IsTemplate: false 20 | IsTopLevel: true 21 | ShowPerDefault: true 22 | SectionType: Example 23 | --- 24 | You can use `sqleton select` with the `--create-query ` flag to 25 | quickly scaffold queries that can then be stored in `~/.sqleton/queries`. 26 | 27 | ``` 28 | ❯ sqleton select --table orders --limit 20 \ 29 | --order-by "order_number DESC" \ 30 | --create-query orders 31 | name: orders 32 | short: Select columns from orders 33 | flags: 34 | - name: where 35 | type: string 36 | - name: limit 37 | type: int 38 | help: 'Limit the number of rows (default: 10), set to 0 to disable' 39 | default: 10 40 | - name: offset 41 | type: intG 42 | help: 'Offset the number of rows (default: 0)' 43 | default: 0 44 | - name: distinct 45 | type: bool 46 | help: 'Whether to select distinct rows (default: false)' 47 | default: false 48 | - name: order_by 49 | type: string 50 | help: 'Order by (default: order_number DESC)' 51 | default: order_number DESC 52 | query: |- 53 | SELECT {{ if .distinct }}DISTINCT{{ end }} order_number, status, totals FROM orders 54 | {{ if .where }} WHERE {{.where}} {{ end }} 55 | {{ if .order_by }} ORDER BY {{ .order_by }}{{ end }} 56 | {{ if .limit }} LIMIT {{ .limit }}{{ end }} 57 | OFFSET {{ .offset }} 58 | ``` 59 | 60 | It will prepopulate most flags for the template from the values you pass it. 61 | The flags `--columns` and `--where` however are fixed. 62 | 63 | ``` 64 | ❯ sqleton select --table orders --create-query orders \ 65 | --limit 50 --where "title LIKE '%anthropology%'" 66 | name: orders 67 | short: Select from orders where title LIKE '%anthropology%' 68 | flags: 69 | - name: limit 70 | type: int 71 | help: 'Limit the number of rows (default: 50), set to 0 to disable' 72 | default: 50 73 | - name: offset 74 | type: int 75 | help: 'Offset the number of rows (default: 0)' 76 | default: 0 77 | - name: distinct 78 | type: bool 79 | help: 'Whether to select distinct rows (default: false)' 80 | default: false 81 | - name: order_by 82 | type: string 83 | help: Order by 84 | query: |- 85 | SELECT {{ if .distinct }}DISTINCT{{ end }} * FROM orders WHERE title LIKE '%anthropology%' 86 | {{ if .order_by }} ORDER BY {{ .order_by }}{{ end }} 87 | {{ if .limit }} LIMIT {{ .limit }}{{ end }} 88 | OFFSET {{ .offset }} 89 | ``` 90 | 91 | The `--count` flag also severely restricts the number 92 | of flags in the template: 93 | 94 | ``` 95 | ❯ sqleton select --table orders --create-query orders --count --distinct --columns name 96 | name: orders 97 | short: Count all rows from orders 98 | flags: 99 | - name: where 100 | type: string 101 | query: |- 102 | SELECT COUNT(DISTINCT name) AS count FROM orders 103 | {{ if .where }} WHERE {{.where}} {{ end }} 104 | {{ if .order_by }} ORDER BY {{ .order_by }}{{ end }} 105 | {{ if .limit }} LIMIT {{ .limit }}{{ end }} 106 | OFFSET {{ .offset }} 107 | } 108 | ``` -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/wp/01-wp-ls-posts.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show last 10 posts in a wordpress database 3 | Slug: wp-ls-posts 4 | Short: | 5 | ``` 6 | sqleton wp ls-posts --fields ID,post_title 7 | ``` 8 | Topics: 9 | - wordpress 10 | Commands: 11 | - wp 12 | - ls-posts 13 | IsTemplate: false 14 | IsTopLevel: true 15 | ShowPerDefault: true 16 | SectionType: Example 17 | --- 18 | You can use sqleton to list the posts in a WordPress database. 19 | 20 | Per default, we show only the last 10 posts and pages. 21 | 22 | ``` 23 | ❯ sqleton wp ls-posts --fields ID,post_title 24 | +--------+-------------------------------------------+ 25 | | ID | post_title | 26 | +--------+-------------------------------------------+ 27 | | 703822 | Auto Draft | 28 | | 703808 | Thinking Hydrangeas? Think Native! | 29 | | 703466 | Winter Care of Houseplants | 30 | | 702794 | What is Wintergreen? | 31 | | 702672 | What are Tetraploid Daylilies? | 32 | | 700051 | Growing Clematis – large-flowered hybrids | 33 | | 698918 | Growing Sedges in Your Garden | 34 | | 697647 | Get to Know the Asters | 35 | | 696470 | Why Isn’t My Garden Growing? | 36 | | 695361 | How Big Does That Tree Really Get? | 37 | +--------+-------------------------------------------+ 38 | ``` 39 | 40 | Looking at the query: 41 | 42 | ``` 43 | ❯ sqleton wp ls-posts --print-query 44 | SELECT wp.ID, wp.post_title, wp.post_type, wp.post_status, wp.post_date FROM wp_posts wp 45 | WHERE post_type IN ('post','page') 46 | ORDER BY post_date DESC 47 | LIMIT 10 48 | ``` -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/wp/02-wp-ls-posts-shrubs.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Show all posts about shrubs from 2017 3 | Slug: wp-ls-posts-shrubs 4 | Short: | 5 | ``` 6 | sqleton wp ls-posts 7 | --limit 100 --status publish --order-by post_title \ 8 | --from 2017-01-01 --to 2017-10-01 --title-like Shrubs 9 | ``` 10 | Topics: 11 | - wordpress 12 | Commands: 13 | - wp 14 | - ls-posts 15 | Flags: 16 | - status 17 | - order-by 18 | - limit 19 | - from 20 | - to 21 | - title-like 22 | IsTemplate: false 23 | IsTopLevel: true 24 | ShowPerDefault: true 25 | SectionType: Example 26 | --- 27 | 28 | We can run much more complex queries against the Wordpress DB. 29 | 30 | ``` 31 | ❯ sqleton wp ls-posts --limit 100 --status publish --order-by post_title \ 32 | --from 2017-01-01 --to 2017-10-01 \ 33 | --fields ID,post_title,post_date \ 34 | --title-like Shrubs 35 | +-------+--------------------------------------------------+---------------------+ 36 | | ID | post_title | post_date | 37 | +-------+--------------------------------------------------+---------------------+ 38 | | 15994 | Deer Resistant Trees and Shrubs | 2017-02-26 12:47:57 | 39 | | 15722 | Flowering Trees and Shrubs for Early Spring | 2017-02-07 11:08:15 | 40 | | 23006 | Time to Prune Spring-flowering Shrubs - Part One | 2017-06-26 06:28:41 | 41 | | 23259 | Time to Prune Spring-flowering Shrubs – Part Two | 2017-07-03 06:50:48 | 42 | | 23520 | Tips on Placing Shrubs in Your Garden | 2017-07-10 06:32:17 | 43 | | 17662 | Tips on Pruning Shrubs and Flowering Trees | 2017-04-09 06:35:22 | 44 | | 24391 | Top Drought Resistant Trees and Shrubs | 2017-08-08 03:44:00 | 45 | | 16177 | What Shrubs Should be Pruned in Spring? | 2017-03-05 07:48:52 | 46 | +-------+--------------------------------------------------+---------------------+ 47 | ``` 48 | 49 | ``` 50 | ❯ sqleton wp ls-posts --limit 100 --status publish \ 51 | --order-by post_title \ 52 | --from 2017-01-01 --to 2017-10-01 \ 53 | --title-like Shrubs --print-query 54 | SELECT wp.ID, wp.post_title, wp.post_type, wp.post_status, wp.post_date FROM wp_posts wp 55 | WHERE post_type IN ('post','page') 56 | AND post_status IN ('publish') 57 | AND post_date >= '2017-01-01' 58 | AND post_date <= '2017-10-01' 59 | AND post_title LIKE '%Shrubs%' 60 | ORDER BY post_title 61 | LIMIT 100 62 | ``` 63 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/examples/wp/03-wp-ls-posts-select.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Get the ID of the last 5 draft posts for use in a shell script 3 | Slug: wp-ls-posts-select 4 | Short: | 5 | ``` 6 | sqleton wp ls-posts --type blog --status draft --limit 5 --select ID 7 | ``` 8 | Topics: 9 | - wordpress 10 | Commands: 11 | - ls-posts 12 | - wp 13 | Flags: 14 | - status 15 | - type 16 | - limit 17 | - select 18 | IsTemplate: false 19 | IsTopLevel: true 20 | ShowPerDefault: true 21 | SectionType: Example 22 | --- 23 | 24 | To get only the IDs, to reuse them in another context (for example a shell script loop): 25 | 26 | ``` 27 | ❯ sqleton wp ls-posts --type blog --status draft --limit 5 --select ID 28 | 635239 29 | 76792 30 | 471151 31 | 262931 32 | 79536 33 | ``` 34 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/01-sqleton.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: sqleton 3 | Slug: sqleton 4 | Short: sqleton is a tool to quickly run SQL commands 5 | Topics: 6 | - config 7 | - sources 8 | Commands: 9 | - db 10 | - run 11 | Flags: 12 | - host 13 | - topic 14 | - command 15 | - list 16 | - topics 17 | - examples 18 | - applications 19 | - tutorials 20 | - help 21 | IsTemplate: false 22 | IsTopLevel: true 23 | ShowPerDefault: true 24 | SectionType: GeneralTopic 25 | --- 26 | 27 | ## Overview 28 | 29 | sqleton is a tool to make it easy to run SQL commands against databases. 30 | 31 | sqleton uses the [glazed](https://github.com/go-go-golems/glazed) help system. 32 | 33 | sqleton allows you to use the `dbt` profiles file for quickly referencing databases. -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/02-database-sources.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Configuring database sources 3 | Slug: database-sources 4 | Short: | 5 | There are many ways to configure database sources with sqleton. You can: 6 | - pass host, port, database flags on the command line 7 | - load values from the environment 8 | - specify flags in a config file 9 | - use dbt profiles 10 | Topics: 11 | - config 12 | - dbt 13 | Commands: 14 | - db 15 | Flags: 16 | - host 17 | - user 18 | - database 19 | IsTemplate: false 20 | IsTopLevel: true 21 | ShowPerDefault: true 22 | SectionType: GeneralTopic 23 | --- 24 | 25 | ## Database source 26 | 27 | A database source consists of the following variables: 28 | 29 | - type: postgresql, mysql 30 | - hostname 31 | - port 32 | - username 33 | - password 34 | - database 35 | - schema (optional) 36 | 37 | These values are combined to create a connection string that 38 | is passed to the `sqlx` package for connection. 39 | 40 | > TODO(2022-12-18): support for dsn/driver flags, sqlite connection are planned 41 | See: 42 | https://github.com/wesen/sqleton/issues/19 - add sqlite support 43 | https://github.com/wesen/sqleton/issues/21 - add dsn/driver flags 44 | 45 | To test a connection, you can use the `db ping` command: 46 | 47 | ``` 48 | ❯ export SQLETON_PASSWORD=foobar 49 | ❯ sqleton db test --host localhost --port 3336 --user root 50 | Connection successful 51 | ``` 52 | 53 | ## Command line flags 54 | 55 | You can pass the following flags for configuring a source 56 | 57 | -D, --database string Database name 58 | -H, --host string Database host 59 | -p, --password string Database password 60 | -P, --port int Database port (default 3306) 61 | -s, --schema string Database schema (when applicable) 62 | -t, --type string Database type (mysql, postgres, etc.) (default "mysql") 63 | -u, --user string Database user 64 | 65 | ## dbt support 66 | 67 | [dbt](http://getdbt.com) is a great tool to run analytics against SQL databases. 68 | In order to make it easy to reuse the configurations set by dbt, we allow loading 69 | dbt profile files. 70 | 71 | In order to reuse dbt profiles, you can use the following flags: 72 | 73 | --use-dbt-profiles Use dbt profiles.yml to connect to databases 74 | --dbt-profile string Name of dbt profile to use (default: default) (default "default") 75 | --dbt-profiles-path string Path to dbt profiles.yml (default: ~/.dbt/profiles.yml) 76 | 77 | What we call "profile" is actually a combination of profile name and output name. 78 | You can refer to a specific output `prod` of a profile `production` by using `production.prod`. 79 | 80 | To get an overview of available dbt profiles, you can use the `db ls` command: 81 | 82 | ``` 83 | ❯ sqleton db ls --fields name,hostname,port,database 84 | +---------------------+-----------+-------+-------------------+ 85 | | name | hostname | port | database | 86 | +---------------------+-----------+-------+-------------------+ 87 | | localhost.localhost | localhost | 3336 | ttc_analytics | 88 | | ttc.prod | localhost | 50393 | ttc_analytics | 89 | | prod.prod | localhost | 50393 | ttc_analytics | 90 | | dev.dev | localhost | 50392 | ttc_dev_analytics | 91 | +---------------------+-----------+-------+-------------------+ 92 | ``` 93 | 94 | 95 | ## Environment variables 96 | 97 | All the flags mentioned above can also be set through environment variables, prefaced 98 | with `SQLETON_` and with `-` replaced by `_`. 99 | 100 | For example, to replace `--host localhost --port 1234 --user manuel`, use the following 101 | environment variables: 102 | 103 | ```dotenv 104 | SQLETON_USER=manuel 105 | SQLETON_PORT=1234 106 | SQLETON_HOST=localhost 107 | ``` 108 | 109 | This also applies to the dbt flags. 110 | 111 | ## Configuration 112 | 113 | You can store all these values in a file called `config.yml`. 114 | sqleton will look in the following locations (in that order) 115 | 116 | - . 117 | - $HOME/.sqleton 118 | - /etc/sqleton 119 | 120 | Flags and environment variables will take precedence. 121 | 122 | The config file is a simple yaml file with the variables set: 123 | 124 | ```yaml 125 | type: mysql 126 | host: localhost 127 | port: 3336 128 | user: root 129 | password: somewordpress 130 | schema: wp 131 | database: wp 132 | ``` -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/03-subqueries.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Use subqueries in your SQL 3 | Slug: subqueries 4 | Short: | 5 | ``` 6 | You can add subQueries as a field in your YAML sqleton command. They can then be used 7 | withing your templates to evaluate nested values. 8 | ``` 9 | Topics: 10 | - queries 11 | IsTemplate: false 12 | IsTopLevel: false 13 | ShowPerDefault: false 14 | SectionType: GeneralTopic 15 | --- 16 | If you need to use subqueries in your SQL, you can add them as a field in your YAML sqleton command. 17 | 18 | They can then be used withing your templates to evaluate nested values. 19 | 20 | This makes it easy to create pivoted tables, for example. 21 | 22 | ```yaml 23 | name: posts-counts 24 | short: Count posts by type 25 | flags: 26 | - name: post_type 27 | description: Post type 28 | type: stringList 29 | required: false 30 | subqueries: 31 | post_types: | 32 | SELECT DISTINCT post_type 33 | FROM wp_posts 34 | GROUP BY post_type 35 | LIMIT 4 36 | query: | 37 | {{ $types := sqlColumn (subQuery "post_types") }} 38 | SELECT 39 | {{ range $i, $v := $types }} 40 | {{ $v2 := printf "count%d" $i }} 41 | ( 42 | SELECT count(*) AS count 43 | FROM wp_posts 44 | WHERE post_status = 'publish' 45 | AND post_type = '{{ $v }}' 46 | ) AS `{{ $v }}` {{ if not (eq $i (sub (len $types) 1)) }}, {{ end }} 47 | {{ end }} 48 | ``` 49 | 50 | If you print out the query before running it: 51 | 52 | ``` 53 | ❯ sqleton wp posts-counts --print-query 54 | SELECT 55 | ( 56 | SELECT count(*) AS count 57 | FROM wp_posts 58 | WHERE post_status = 'publish' 59 | AND post_type = 'attachment' 60 | ) AS `attachment` , 61 | ( 62 | SELECT count(*) AS count 63 | FROM wp_posts 64 | WHERE post_status = 'publish' 65 | AND post_type = 'custom_css' 66 | ) AS `custom_css` , 67 | ( 68 | SELECT count(*) AS count 69 | FROM wp_posts 70 | WHERE post_status = 'publish' 71 | AND post_type = 'export_template' 72 | ) AS `export_template` , 73 | ( 74 | SELECT count(*) AS count 75 | FROM wp_posts 76 | WHERE post_status = 'publish' 77 | AND post_type = 'faq' 78 | ) AS `faq` 79 | ``` 80 | 81 | And then run it: 82 | 83 | ``` 84 | ❯ sqleton wp posts-counts 85 | +------------+------------+-----------------+-----+ 86 | | attachment | custom_css | export_template | faq | 87 | +------------+------------+-----------------+-----+ 88 | | 0 | 1 | 2 | 35 | 89 | +------------+------------+-----------------+-----+ 90 | ``` -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/04-aliases.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Creating aliases 3 | Slug: creating-aliases 4 | Short: | 5 | You can add quickly create aliases for existing commands in order 6 | to save flags. These aliases can be stored alongside the query files 7 | and will become accessible as their own full-fledged cobra commands. 8 | Topics: 9 | - queries 10 | - aliases 11 | Commands: 12 | - queries 13 | Flags: 14 | - create-alias 15 | IsTemplate: false 16 | IsTopLevel: true 17 | ShowPerDefault: true 18 | SectionType: GeneralTopic 19 | --- 20 | 21 | ## Creating aliases 22 | 23 | Due to the very flexible glazed output system and the flags available for each command, 24 | it is useful to be able to create aliases to save on typing and having to remember 25 | all variations. 26 | 27 | An alias is just a yaml file stored along the query yaml definition. Aliases can be 28 | located in an `embed.FS` repository, or in a filesystem repository. 29 | 30 | For a query `ttc/wp/posts.yaml`, aliases can be stored in the directory `ttc/wp/posts/`. 31 | 32 | For example, we could have the following alias to show the newest drafts, 33 | in `newest-drafts.yaml`: 34 | 35 | ```yaml 36 | name: newest-drafts 37 | aliasFor: posts 38 | flags: 39 | limit: 10 40 | from: last week 41 | status: draft 42 | filter: post_status,post_type 43 | ``` 44 | 45 | This command can then be executed by running `sqleton ttc wp posts newest-drafts`, 46 | and will be equivalent to running 47 | 48 | ``` 49 | sqleton ttc wp posts --limit 10 --from "last week" \ 50 | --status draft --filter post_status,post_type 51 | ``` 52 | 53 | In order to help with the arduous task of writing YAML file, you can 54 | automatically emit the alias file by using the `--create-alias [name]` flag: 55 | 56 | ``` 57 | ❯ sqleton wp postmeta --key-like 'refund_reason' \ 58 | --db ttc_prod --order-by "post_id DESC" \ 59 | --create-alias refunds 60 | name: refunds 61 | aliasFor: postmeta 62 | flags: 63 | db: ttc_prod 64 | key-like: refund_reason 65 | order-by: post_id DESC 66 | ``` 67 | 68 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/05-print-settings.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Printing connection settings 3 | Slug: print-settings 4 | Short: | 5 | You can print out the currently configured connection settings for quick 6 | reuse in an env file or other configuration file. 7 | Topics: 8 | - settings 9 | Commands: 10 | - print-env 11 | - print-evidence-settings 12 | - print-settings 13 | IsTemplate: false 14 | IsTopLevel: true 15 | ShowPerDefault: false 16 | SectionType: GeneralTopic 17 | --- 18 | 19 | ## Printing settings 20 | 21 | The connection settings for sqleton can be passed in through a variety of ways. 22 | They can be passed as environment flags, as command line flags, as a config file, 23 | and event as a dbt profile. 24 | 25 | Sometimes, these credentials need to be partially or fully exported for use in another 26 | application (say, a docker-compose file, an env file, a JSON for a cloud deploy). 27 | In order to facilitate the often tedious process, sqleton provides the following commands 28 | to make a developer's life easier. 29 | 30 | ### `sqleton db print-env` 31 | 32 | This command prints out the connection settings as environment variables. It is 33 | useful for quickly exporting the settings to an env file. 34 | 35 | ``` 36 | ❯ sqleton db print-env 37 | SQLETON_TYPE=mysql 38 | SQLETON_HOST=localhost 39 | SQLETON_PORT=3306 40 | SQLETON_DATABASE=sqleton 41 | SQLETON_USER=sqleton 42 | SQLETON_PASSWORD=secret 43 | SQLETON_SCHEMA=bones 44 | SQLETON_USE_DBT_PROFILES= 45 | SQLETON_DBT_PROFILES_PATH= 46 | SQLETON_DBT_PROFILE=yolo.bolo 47 | ``` 48 | 49 | Note that this will print out both connection settings and dbt settings. The dbt settings override the 50 | connection settings. It is up to you to clean the output up. 51 | 52 | You can specify a different env variable name prefix. 53 | 54 | ``` 55 | ❯ sqleton db print-env --env-prefix DB_ 56 | DB_TYPE=mysql 57 | DB_HOST=localhost 58 | DB_PORT=3306 59 | DB_DATABASE=sqleton 60 | DB_USER=sqleton 61 | DB_PASSWORD=secret 62 | DB_SCHEMA=bones 63 | DB_USE_DBT_PROFILES= 64 | DB_DBT_PROFILES_PATH= 65 | DB_DBT_PROFILE=yolo.bolo 66 | ``` 67 | 68 | Furthermore, you can add `export ` at the beginning of each line by passing in the `--envrc` flag. 69 | This makes it easier to use in `.envrc` files for the `direnv` tool. 70 | 71 | ``` 72 | ❯ sqleton db print-env --envrc 73 | export SQLETON_TYPE=mysql 74 | export SQLETON_HOST=localhost 75 | export SQLETON_PORT=3306 76 | export SQLETON_DATABASE=sqleton 77 | export SQLETON_USER=sqleton 78 | export SQLETON_PASSWORD=secret 79 | export SQLETON_SCHEMA=bones 80 | export SQLETON_USE_DBT_PROFILES= 81 | export SQLETON_DBT_PROFILES_PATH= 82 | export SQLETON_DBT_PROFILE=yolo.bolo 83 | ``` 84 | 85 | ### `sqleton db print-settings` 86 | 87 | For a more flexible output, you can use the `print-settings` command. This command uses the glazed 88 | library to output the connection settings, which allows you to specify the output format. 89 | 90 | Per default, it will output the connection settings as a single row as a human readable table. 91 | 92 | ``` 93 | ❯ sqleton db print-settings 94 | +---------+----------+-------+------------+-----------------+----------------+--------+----------+-----------+------+ 95 | | user | password | type | dbtProfile | dbtProfilesPath | useDbtProfiles | schema | database | host | port | 96 | +---------+----------+-------+------------+-----------------+----------------+--------+----------+-----------+------+ 97 | | sqleton | secret | mysql | yolo.bolo | | false | bones | sqleton | localhost | 3306 | 98 | +---------+----------+-------+------------+-----------------+----------------+--------+----------+-----------+------+ 99 | ``` 100 | 101 | You can now easily print it as JSON or YAML. 102 | 103 | ``` 104 | ❯ sqleton db print-settings --output yaml 105 | - database: sqleton 106 | dbtProfile: yolo.bolo 107 | dbtProfilesPath: "" 108 | host: localhost 109 | password: secret 110 | port: 3306 111 | schema: bones 112 | type: mysql 113 | useDbtProfiles: false 114 | user: sqleton 115 | ``` 116 | 117 | You can also output each setting as a single row, for example as CSV. 118 | 119 | ``` 120 | ❯ sqleton db print-settings --output csv --individual-rows 121 | value,name 122 | localhost,host 123 | 3306,port 124 | sqleton,database 125 | sqleton,user 126 | secret,password 127 | mysql,type 128 | bones,schema 129 | yolo.bolo,dbtProfile 130 | false,useDbtProfiles 131 | ,dbtProfilesPath 132 | ``` 133 | 134 | Here too, you can specify printing things out as environment variables. 135 | 136 | ``` 137 | ❯ sqleton db print-settings --use-env-names --individual-rows 138 | +---------------------------+-----------+ 139 | | name | value | 140 | +---------------------------+-----------+ 141 | | SQLETON_HOST | localhost | 142 | | SQLETON_PORT | 3306 | 143 | | SQLETON_DATABASE | sqleton | 144 | | SQLETON_USER | sqleton | 145 | | SQLETON_PASSWORD | secret | 146 | | SQLETON_TYPE | mysql | 147 | | SQLETON_SCHEMA | bones | 148 | | SQLETON_DBT_PROFILE | yolo.bolo | 149 | | SQLETON_USE_DBT_PROFILES | false | 150 | | SQLETON_DBT_PROFILES_PATH | | 151 | +---------------------------+-----------+ 152 | ``` 153 | 154 | The environment variables prefix can be configured as well. 155 | 156 | ``` 157 | ❯ sqleton db print-settings --individual-rows --with-env-prefix DB_ 158 | +----------------------+-----------+ 159 | | name | value | 160 | +----------------------+-----------+ 161 | | DB_HOST | localhost | 162 | | DB_PORT | 3306 | 163 | | DB_DATABASE | sqleton | 164 | | DB_USER | sqleton | 165 | | DB_PASSWORD | secret | 166 | | DB_TYPE | mysql | 167 | | DB_SCHEMA | bones | 168 | | DB_DBT_PROFILE | yolo.bolo | 169 | | DB_USE_DBT_PROFILES | false | 170 | | DB_DBT_PROFILES_PATH | | 171 | +----------------------+-----------+ 172 | ``` 173 | 174 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/topics/06-query-commands.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Adding query commands 3 | Slug: query-commands 4 | Short: | 5 | You can add commands to the `sqleton` program in a variety of ways: 6 | - using YAML files 7 | - using SQL files and metadata (TODO) 8 | - using Markdown files 9 | Topics: 10 | - queries 11 | Commands: 12 | - queries 13 | IsTemplate: false 14 | IsTopLevel: true 15 | ShowPerDefault: true 16 | SectionType: GeneralTopic 17 | --- 18 | 19 | ## Using YAML files 20 | 21 | YAML files can be used to add commands to sqleton by using the following layout: 22 | 23 | ```yaml 24 | name: ls-posts-type 25 | short: Show all WP posts, limited, by type 26 | long: Show all posts and their ID 27 | flags: 28 | - name: types 29 | type: stringList 30 | default: 31 | - post 32 | - page 33 | help: Select posts by type 34 | arguments: 35 | - name: limit 36 | shortFlag: l 37 | type: int 38 | default: 10 39 | help: Limit the number of posts 40 | query: | 41 | SELECT wp.ID, wp.post_title, wp.post_type, wp.post_status FROM wp_posts wp 42 | WHERE post_type IN ({{ .types | sqlStringIn }}) 43 | LIMIT {{ .limit }} 44 | ``` 45 | 46 | ## Query repository 47 | 48 | These files can be stored in a repository directory that has the following format: 49 | 50 | ``` 51 | repository/ 52 | subCommand/ 53 | subsubsCommand/ 54 | query.yaml 55 | subCommand2/ 56 | query2.yaml 57 | ``` 58 | 59 | This will result in the following commands being added (including their subcommands): 60 | 61 | ``` 62 | sqleton subCommand subsubsCommand query 63 | sqleton subCommand2 query2 64 | ``` 65 | 66 | A repository can be loaded at compile time as an `embed.FS` by using the 67 | `sqleton.LoadSqlCommandsFromEmbedFS`, and at runtime from a directory by using 68 | `sqleton.LoadSqlCommandsFromDirectory`. 69 | 70 | The configuration flag or variable `repository` can be set to specify a custom 71 | repository, by default, the queries in `$HOME/.sqleton/queries` are loaded. 72 | 73 | You can specify more repositories to be loaded in addition to the default by 74 | specifying a list in `config.yaml`: 75 | 76 | ```yaml 77 | repositories: 78 | - /Users/manuel/code/ttc/ttc-dbt/sqleton-queries 79 | - .sqleton/queries 80 | ``` 81 | 82 | ## Using query parameters 83 | 84 | A query can also provide parameters, which are mapped to command line flags and arguments 85 | 86 | Parameters have the following structure: 87 | 88 | ```yaml 89 | - name: limit 90 | shortFlag: l 91 | type: int 92 | default: 10 93 | help: Limit the number of posts 94 | ``` 95 | 96 | Valid types for a parameter are: 97 | 98 | - `string` 99 | - `int` 100 | - `bool` 101 | - `date` 102 | - `choice` 103 | - `stringList` 104 | - `intList` 105 | 106 | These are then specified in the `flags` and `arguments` section respectively. 107 | 108 | Arguments have to obey a few rules: 109 | - optional arguments can't follow required arguments 110 | - no argument can follow a stringList of intList argument 111 | 112 | 113 | ## Providing help pages for queries 114 | 115 | To add examples, topics, and other help pages for your query, just add a markdown 116 | file inside one of the directories scanned for help pages. 117 | 118 | Look at [wordpress examples](../../../doc/examples/wp) for more examples. 119 | -------------------------------------------------------------------------------- /cmd/sqleton/doc/tutorials/01-placeholder-tutorial.md: -------------------------------------------------------------------------------- 1 | --- 2 | Title: Placeholder tutorial 3 | Slug: placeholder-tutorial 4 | Topics: 5 | - sqleton 6 | IsTemplate: false 7 | IsTopLevel: true 8 | ShowPerDefault: true 9 | SectionType: Tutorial 10 | --- 11 | 12 | XXX fill it out -------------------------------------------------------------------------------- /cmd/sqleton/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "embed" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | 10 | clay "github.com/go-go-golems/clay/pkg" 11 | clay_doc "github.com/go-go-golems/clay/pkg/doc" 12 | "github.com/go-go-golems/clay/pkg/repositories" 13 | "github.com/go-go-golems/clay/pkg/sql" 14 | "github.com/go-go-golems/glazed/pkg/cli" 15 | glazed_cmds "github.com/go-go-golems/glazed/pkg/cmds" 16 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 17 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 18 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 19 | "github.com/go-go-golems/glazed/pkg/help" 20 | "github.com/go-go-golems/glazed/pkg/types" 21 | parka_doc "github.com/go-go-golems/parka/pkg/doc" 22 | "github.com/go-go-golems/sqleton/cmd/sqleton/cmds" 23 | "github.com/go-go-golems/sqleton/cmd/sqleton/cmds/mcp" 24 | sqleton_cmds "github.com/go-go-golems/sqleton/pkg/cmds" 25 | "github.com/go-go-golems/sqleton/pkg/flags" 26 | "github.com/pkg/errors" 27 | "github.com/pkg/profile" 28 | "github.com/rs/zerolog/log" 29 | "github.com/spf13/cobra" 30 | "github.com/spf13/viper" 31 | 32 | // #nosec G108 - pprof is imported for profiling and debugging in development environments only. 33 | // This is gated behind the --mem-profile flag and not enabled by default. 34 | _ "net/http/pprof" 35 | 36 | clay_commandmeta "github.com/go-go-golems/clay/pkg/cmds/commandmeta" 37 | clay_profiles "github.com/go-go-golems/clay/pkg/cmds/profiles" 38 | clay_repositories "github.com/go-go-golems/clay/pkg/cmds/repositories" 39 | ) 40 | 41 | var version = "dev" 42 | var profiler interface { 43 | Stop() 44 | } 45 | 46 | var rootCmd = &cobra.Command{ 47 | Use: "sqleton", 48 | Short: "sqleton runs SQL queries out of template files", 49 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 50 | memProfile, _ := cmd.Flags().GetBool("mem-profile") 51 | if memProfile { 52 | log.Info().Msg("Starting memory profiler") 53 | profiler = profile.Start(profile.MemProfile) 54 | 55 | // on SIGHUP, restart the profiler 56 | sigCh := make(chan os.Signal, 1) 57 | signal.Notify(sigCh, syscall.SIGHUP) 58 | go func() { 59 | for range sigCh { 60 | log.Info().Msg("Restarting memory profiler") 61 | profiler.Stop() 62 | profiler = profile.Start(profile.MemProfile) 63 | } 64 | }() 65 | } 66 | }, 67 | PersistentPostRun: func(cmd *cobra.Command, args []string) { 68 | if profiler != nil { 69 | log.Info().Msg("Stopping memory profiler") 70 | profiler.Stop() 71 | } 72 | }, 73 | Version: version, 74 | } 75 | 76 | func main() { 77 | // first, check if the args are "run-command file.yaml", 78 | // because we need to load the file and then run the command itself. 79 | // we need to do this before cobra, because we don't know which flags to load yet 80 | if len(os.Args) >= 3 && os.Args[1] == "run-command" && os.Args[2] != "--help" { 81 | // load the command 82 | loader := &sqleton_cmds.SqlCommandLoader{ 83 | DBConnectionFactory: sql.OpenDatabaseFromDefaultSqlConnectionLayer, 84 | } 85 | fs_, filePath, err := loaders.FileNameToFsFilePath(os.Args[2]) 86 | if err != nil { 87 | fmt.Printf("Could not get absolute path: %v\n", err) 88 | os.Exit(1) 89 | } 90 | cmds, err := loader.LoadCommands(fs_, filePath, []glazed_cmds.CommandDescriptionOption{}, []alias.Option{}) 91 | if err != nil { 92 | fmt.Printf("Could not load command: %v\n", err) 93 | os.Exit(1) 94 | } 95 | if len(cmds) != 1 { 96 | fmt.Printf("Expected exactly one command, got %d", len(cmds)) 97 | } 98 | 99 | glazeCommand, ok := cmds[0].(glazed_cmds.GlazeCommand) 100 | if !ok { 101 | fmt.Printf("Expected GlazeCommand, got %T", cmds[0]) 102 | os.Exit(1) 103 | } 104 | 105 | cobraCommand, err := sql.BuildCobraCommandWithSqletonMiddlewares(glazeCommand) 106 | if err != nil { 107 | fmt.Printf("Could not build cobra command: %v\n", err) 108 | os.Exit(1) 109 | } 110 | 111 | _, err = initRootCmd() 112 | cobra.CheckErr(err) 113 | 114 | rootCmd.AddCommand(cobraCommand) 115 | restArgs := os.Args[3:] 116 | os.Args = append([]string{os.Args[0], cobraCommand.Use}, restArgs...) 117 | } else { 118 | helpSystem, err := initRootCmd() 119 | cobra.CheckErr(err) 120 | 121 | err = initAllCommands(helpSystem) 122 | cobra.CheckErr(err) 123 | } 124 | 125 | err := rootCmd.Execute() 126 | cobra.CheckErr(err) 127 | } 128 | 129 | var runCommandCmd = &cobra.Command{ 130 | Use: "run-command", 131 | Short: "Run a command from a file", 132 | Args: cobra.ExactArgs(1), 133 | Run: func(cmd *cobra.Command, args []string) { 134 | panic(errors.Errorf("not implemented")) 135 | }, 136 | } 137 | 138 | //go:embed doc/* 139 | var docFS embed.FS 140 | 141 | //go:embed queries/* 142 | var queriesFS embed.FS 143 | 144 | func initRootCmd() (*help.HelpSystem, error) { 145 | helpSystem := help.NewHelpSystem() 146 | err := helpSystem.LoadSectionsFromFS(docFS, ".") 147 | cobra.CheckErr(err) 148 | 149 | err = parka_doc.AddDocToHelpSystem(helpSystem) 150 | cobra.CheckErr(err) 151 | 152 | err = clay_doc.AddDocToHelpSystem(helpSystem) 153 | cobra.CheckErr(err) 154 | 155 | helpSystem.SetupCobraRootCommand(rootCmd) 156 | 157 | err = clay.InitViper("sqleton", rootCmd) 158 | cobra.CheckErr(err) 159 | rootCmd.AddCommand(runCommandCmd) 160 | 161 | rootCmd.AddCommand(cmds.NewCodegenCommand()) 162 | return helpSystem, nil 163 | } 164 | 165 | func initAllCommands(helpSystem *help.HelpSystem) error { 166 | rootCmd.AddCommand(cmds.DbCmd) 167 | 168 | dbtParameterLayer, err := sql.NewDbtParameterLayer() 169 | if err != nil { 170 | return err 171 | } 172 | sqlConnectionParameterLayer, err := sql.NewSqlConnectionParameterLayer() 173 | if err != nil { 174 | return err 175 | } 176 | 177 | runCommand, err := cmds.NewRunCommand(sql.OpenDatabaseFromDefaultSqlConnectionLayer, 178 | glazed_cmds.WithLayersList( 179 | dbtParameterLayer, 180 | sqlConnectionParameterLayer, 181 | )) 182 | if err != nil { 183 | return err 184 | } 185 | 186 | cobraRunCommand, err := sql.BuildCobraCommandWithSqletonMiddlewares(runCommand) 187 | if err != nil { 188 | return err 189 | } 190 | rootCmd.AddCommand(cobraRunCommand) 191 | 192 | selectCommand, err := cmds.NewSelectCommand(sql.OpenDatabaseFromDefaultSqlConnectionLayer, 193 | glazed_cmds.WithLayersList( 194 | dbtParameterLayer, 195 | sqlConnectionParameterLayer, 196 | )) 197 | if err != nil { 198 | return err 199 | } 200 | cobraSelectCommand, err := sql.BuildCobraCommandWithSqletonMiddlewares(selectCommand, 201 | cli.WithCobraShortHelpLayers(cmds.SelectSlug), 202 | ) 203 | if err != nil { 204 | return err 205 | } 206 | rootCmd.AddCommand(cobraSelectCommand) 207 | 208 | queryCommand, err := cmds.NewQueryCommand( 209 | sql.OpenDatabaseFromDefaultSqlConnectionLayer, 210 | glazed_cmds.WithLayersList( 211 | dbtParameterLayer, 212 | sqlConnectionParameterLayer, 213 | )) 214 | if err != nil { 215 | return err 216 | } 217 | cobraQueryCommand, err := sql.BuildCobraCommandWithSqletonMiddlewares(queryCommand) 218 | if err != nil { 219 | return err 220 | } 221 | rootCmd.AddCommand(cobraQueryCommand) 222 | 223 | repositoryPaths := viper.GetStringSlice("repositories") 224 | 225 | defaultDirectory := "$HOME/.sqleton/queries" 226 | _, err = os.Stat(os.ExpandEnv(defaultDirectory)) 227 | if err == nil { 228 | repositoryPaths = append(repositoryPaths, os.ExpandEnv(defaultDirectory)) 229 | } 230 | 231 | loader := &sqleton_cmds.SqlCommandLoader{ 232 | DBConnectionFactory: sql.OpenDatabaseFromDefaultSqlConnectionLayer, 233 | } 234 | directories := []repositories.Directory{ 235 | { 236 | FS: queriesFS, 237 | RootDirectory: "queries", 238 | RootDocDirectory: "queries/doc", 239 | Name: "sqleton", 240 | SourcePrefix: "embed", 241 | }} 242 | 243 | for _, repositoryPath := range repositoryPaths { 244 | dir := os.ExpandEnv(repositoryPath) 245 | // check if dir exists 246 | if fi, err := os.Stat(dir); os.IsNotExist(err) || !fi.IsDir() { 247 | continue 248 | } 249 | directories = append(directories, repositories.Directory{ 250 | FS: os.DirFS(dir), 251 | RootDirectory: ".", 252 | RootDocDirectory: "doc", 253 | Name: dir, 254 | SourcePrefix: "file", 255 | }) 256 | } 257 | 258 | repositories_ := []*repositories.Repository{ 259 | repositories.NewRepository( 260 | repositories.WithDirectories(directories...), 261 | repositories.WithCommandLoader(loader), 262 | ), 263 | } 264 | 265 | allCommands, err := repositories.LoadRepositories( 266 | helpSystem, 267 | rootCmd, 268 | repositories_, 269 | cli.WithCobraMiddlewaresFunc(sql.GetCobraCommandSqletonMiddlewares), 270 | cli.WithCobraShortHelpLayers(layers.DefaultSlug, sql.DbtSlug, sql.SqlConnectionSlug, flags.SqlHelpersSlug), 271 | cli.WithCreateCommandSettingsLayer(), 272 | cli.WithProfileSettingsLayer(), 273 | ) 274 | if err != nil { 275 | return err 276 | } 277 | 278 | // Create and add MCP commands 279 | mcpCommands := mcp.NewMcpCommands(repositories_) 280 | mcpCommands.AddToRootCommand(rootCmd) 281 | 282 | serveCommand, err := cmds.NewServeCommand( 283 | sql.OpenDatabaseFromDefaultSqlConnectionLayer, 284 | repositoryPaths, 285 | ) 286 | if err != nil { 287 | return err 288 | } 289 | cobraServeCommand, err := sql.BuildCobraCommandWithSqletonMiddlewares(serveCommand) 290 | if err != nil { 291 | return err 292 | } 293 | rootCmd.AddCommand(cobraServeCommand) 294 | 295 | // Create and add the unified command management group 296 | commandManagementCmd, err := clay_commandmeta.NewCommandManagementCommandGroup( 297 | allCommands, // Pass the loaded commands 298 | // Pass the existing AddCommandToRowFunc logic as an option 299 | clay_commandmeta.WithListAddCommandToRowFunc(func( 300 | command glazed_cmds.Command, 301 | row types.Row, 302 | parsedLayers *layers.ParsedLayers, 303 | ) ([]types.Row, error) { 304 | // Example: Set 'type' and 'query' based on command type 305 | switch c := command.(type) { 306 | case *sqleton_cmds.SqlCommand: 307 | row.Set("query", c.Query) 308 | row.Set("type", "sql") 309 | case *alias.CommandAlias: // Handle aliases if needed 310 | row.Set("type", "alias") 311 | row.Set("aliasFor", c.AliasFor) 312 | default: 313 | // Default type handling if needed 314 | if _, ok := row.Get("type"); !ok { 315 | row.Set("type", "unknown") 316 | } 317 | } 318 | return []types.Row{row}, nil 319 | }), 320 | ) 321 | if err != nil { 322 | return fmt.Errorf("failed to initialize command management commands: %w", err) 323 | } 324 | rootCmd.AddCommand(commandManagementCmd) // Add the group directly to root 325 | 326 | // Create and add the profiles command 327 | profilesCmd, err := clay_profiles.NewProfilesCommand("sqleton", sqletonInitialProfilesContent) 328 | if err != nil { 329 | return fmt.Errorf("failed to initialize profiles command: %w", err) 330 | } 331 | rootCmd.AddCommand(profilesCmd) 332 | 333 | // Create and add the repositories command group 334 | rootCmd.AddCommand(clay_repositories.NewRepositoriesGroupCommand()) 335 | 336 | rootCmd.PersistentFlags().Bool("mem-profile", false, "Enable memory profiling") 337 | 338 | return nil 339 | } 340 | 341 | // sqletonInitialProfilesContent provides the default YAML content for a new sqleton profiles file. 342 | func sqletonInitialProfilesContent() string { 343 | return `# Sqleton Profiles Configuration 344 | # 345 | # This file allows defining profiles to override default SQL connection 346 | # settings or query parameters for sqleton commands. 347 | # 348 | # Profiles are selected using the --profile flag. 349 | # Settings within a profile override the default values for the specified layer. 350 | # 351 | # Example: 352 | # 353 | # production-db: 354 | # # Override settings for the 'sql-connection' layer 355 | # sql-connection: 356 | # driver: postgres 357 | # dsn: "host=prod.db user=reporter password=secret dbname=reports sslmode=require" 358 | # # You can also specify individual DSN components: 359 | # # host: prod.db 360 | # # port: 5432 361 | # # user: reporter 362 | # # password: secret 363 | # # dbname: reports 364 | # # sslmode: require 365 | # 366 | # # Override settings for the 'dbt' layer (if using dbt) 367 | # dbt: 368 | # dbt-project-path: /path/to/prod/dbt/project 369 | # dbt-profile: production 370 | # 371 | # You can manage this file using the 'sqleton profiles' commands: 372 | # - list: List all profiles 373 | # - get: Get profile settings 374 | # - set: Set a profile setting 375 | # - delete: Delete a profile, layer, or setting 376 | # - edit: Open this file in your editor 377 | # - init: Create this file if it doesn't exist 378 | # - duplicate: Copy an existing profile 379 | ` 380 | } 381 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/examples/01-get-posts.yaml: -------------------------------------------------------------------------------- 1 | name: ls-posts 2 | short: Show all WP posts 3 | long: Show all posts and their ID 4 | query: | 5 | SELECT wp.ID, wp.post_title, wp.post_status FROM wp_posts wp 6 | WHERE post_type = 'post' 7 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/examples/02-get-posts-limit.yaml: -------------------------------------------------------------------------------- 1 | name: ls-posts-limit 2 | short: Show all WP posts, limited 3 | long: Show all posts and their ID 4 | flags: 5 | - name: limit 6 | shortFlag: l 7 | type: int 8 | default: 10 9 | help: Limit the number of posts 10 | - name: status 11 | type: stringList 12 | help: Select posts by status 13 | required: false 14 | query: | 15 | SELECT wp.ID, wp.post_title, wp.post_status FROM wp_posts wp 16 | WHERE post_type = 'post' 17 | {{ if .status -}} 18 | AND post_status IN ({{ .status | sqlStringIn }}) 19 | {{- end }} 20 | LIMIT {{ .limit }} 21 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/examples/03-get-posts-by-type.yaml: -------------------------------------------------------------------------------- 1 | name: ls-posts-type [types...] 2 | short: "Show all WP posts, limited, by type (default: post, page)" 3 | long: Show all posts and their ID 4 | flags: 5 | - name: limit 6 | shortFlag: l 7 | type: int 8 | default: 10 9 | help: Limit the number of posts 10 | - name: status 11 | type: stringList 12 | help: Select posts by status 13 | required: false 14 | - name: order_by 15 | type: string 16 | default: post_date DESC 17 | help: Order by column 18 | arguments: 19 | - name: types 20 | type: stringList 21 | default: 22 | - post 23 | - page 24 | help: Select posts by type 25 | required: false 26 | query: | 27 | SELECT wp.ID, wp.post_title, wp.post_type, wp.post_status, wp.post_date FROM wp_posts wp 28 | WHERE post_type IN ({{ .types | sqlStringIn }}) 29 | {{ if .status -}} 30 | AND post_status IN ({{ .status | sqlStringIn }}) 31 | {{- end }} 32 | ORDER BY {{ .order_by }} 33 | LIMIT {{ .limit }} 34 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/01-show-full-processlist.yaml: -------------------------------------------------------------------------------- 1 | name: full-ps 2 | short: Show full MySQL processlist 3 | long: SHOW FULL PROCESSLIST 4 | query: | 5 | SHOW FULL PROCESSLIST 6 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/02-show-tables.yaml: -------------------------------------------------------------------------------- 1 | name: ls-tables 2 | short: Show all tables in namespace 3 | long: SHOW TABLES 4 | query: | 5 | SHOW TABLES 6 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/index.yaml: -------------------------------------------------------------------------------- 1 | name: index 2 | short: Output the index information of a table in MySQL. 3 | flags: 4 | - name: tables 5 | type: stringList 6 | help: List of table names 7 | - name: tables_like 8 | type: stringList 9 | help: List of table name patterns to match 10 | - name: index_name 11 | type: string 12 | help: Index name 13 | - name: index_name_like 14 | type: stringList 15 | help: List of index name patterns to match 16 | - name: order_by 17 | type: string 18 | default: table_name ASC 19 | help: Order by 20 | query: | 21 | SELECT 22 | TABLE_NAME AS table_name, 23 | INDEX_NAME AS index_name, 24 | NON_UNIQUE AS non_unique, 25 | COLUMN_NAME AS column_name, 26 | SEQ_IN_INDEX AS seq_in_index, 27 | COLLATION AS collation, 28 | CARDINALITY AS cardinality, 29 | SUB_PART AS sub_part, 30 | PACKED AS packed, 31 | NULLABLE AS nullable, 32 | INDEX_TYPE AS index_type, 33 | COMMENT AS comment 34 | FROM 35 | INFORMATION_SCHEMA.STATISTICS 36 | WHERE 1=1 37 | {{ if .tables }} 38 | AND TABLE_NAME IN ({{ .tables | sqlStringIn }}) 39 | {{ end }} 40 | {{ if .tables_like }} 41 | AND ( 42 | {{ range $index, $table := .tables_like }} 43 | {{ if $index }}OR{{end}} 44 | TABLE_NAME LIKE '{{ $table }}' 45 | {{ end }} 46 | ) 47 | {{ end }} 48 | {{ if .index_name }} 49 | AND INDEX_NAME = '{{ .index_name }}' 50 | {{ end }} 51 | {{ if .index_name_like }} 52 | AND ( 53 | {{ range $index, $index_name := .index_name_like }} 54 | {{ if $index }}OR{{end}} 55 | INDEX_NAME LIKE '{{ $index_name }}' 56 | {{ end }} 57 | ) 58 | {{ end }} 59 | ORDER BY {{ .order_by }} -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/ps.yaml: -------------------------------------------------------------------------------- 1 | name: ps 2 | short: Show full MySQL processlist 3 | 4 | flags: 5 | - name: mysql_user 6 | type: stringList 7 | help: Filter by user(s) 8 | - name: user_like 9 | type: string 10 | help: Filter by user(s) using LIKE 11 | - name: db 12 | type: string 13 | help: Database to use 14 | - name: db_like 15 | type: string 16 | help: Database to use using LIKE 17 | - name: state 18 | type: stringList 19 | help: Filter by state(s) 20 | - name: hostname 21 | type: string 22 | help: Filter by host 23 | - name: info_like 24 | type: string 25 | help: Filter by info using LIKE 26 | - name: short_info 27 | type: bool 28 | help: Show only the first 50 characters of info 29 | default: true 30 | - name: medium_info 31 | type: bool 32 | help: Show only the first 80 characters of info 33 | - name: full_info 34 | type: bool 35 | help: Show the full info 36 | - name: foobar 37 | type: intList 38 | help: Filter by foobar 39 | default: [1,2,3] 40 | query: | 41 | SELECT 42 | Id,User,Host,db,Command,Time,State 43 | {{ if .short_info -}} 44 | ,LEFT(info,50) AS info 45 | {{ end -}} 46 | {{ if .medium_info -}} 47 | ,LEFT(info,80) AS info 48 | {{ end -}} 49 | {{ if .full_info -}} 50 | ,info 51 | {{ end -}} 52 | FROM information_schema.processlist 53 | WHERE 1=1 54 | {{ if .user_like -}} 55 | AND User LIKE {{ .user_like | sqlLike }} 56 | {{ end -}} 57 | {{ if .mysql_user -}} 58 | AND User IN ({{ .mysql_user | sqlStringIn }}) 59 | {{ end -}} 60 | {{ if .state -}} 61 | AND State IN ({{ .state | sqlStringIn }}) 62 | {{ end -}} 63 | {{ if .db -}} 64 | AND db = {{ .db | sqlString }} 65 | {{ end -}} 66 | {{ if .db_like -}} 67 | AND db LIKE {{ .db_like | sqlLike }} 68 | {{ end -}} 69 | {{ if .hostname -}} 70 | AND host = {{ .hostname | sqlString }} 71 | {{ end -}} 72 | {{ if .info_like -}} 73 | AND info LIKE {{ .info_like | sqlLike }} 74 | {{ end -}} 75 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/schema.yaml: -------------------------------------------------------------------------------- 1 | name: schema 2 | short: Output the schema of a table in MySQL. 3 | flags: 4 | - name: databases 5 | type: stringList 6 | help: List of database names 7 | - name: databases_like 8 | type: stringList 9 | help: List of database name patterns to match 10 | - name: tables 11 | type: stringList 12 | help: List of table names 13 | - name: tables_like 14 | type: stringList 15 | help: List of table name patterns to match 16 | - name: columns 17 | type: stringList 18 | help: List of column names 19 | - name: columns_like 20 | type: stringList 21 | help: List of column name patterns to match 22 | - name: type 23 | type: string 24 | help: Column data type 25 | query: | 26 | SELECT 27 | TABLE_SCHEMA AS database_name, 28 | TABLE_NAME AS table_name, 29 | COLUMN_NAME AS column_name, 30 | COLUMN_TYPE AS column_type, 31 | IS_NULLABLE AS is_nullable, 32 | COLUMN_KEY AS column_key, 33 | COLUMN_DEFAULT AS column_default, 34 | EXTRA AS extra 35 | FROM 36 | INFORMATION_SCHEMA.COLUMNS 37 | WHERE 1=1 38 | {{ if .databases }} 39 | AND TABLE_SCHEMA IN ({{ .databases | sqlStringIn }}) 40 | {{ end }} 41 | {{ if .databases_like }} 42 | AND ( 43 | {{ range $index, $database := .databases_like }} 44 | {{ if $index }}OR{{end}} 45 | TABLE_SCHEMA LIKE {{ $database | sqlStringLike}} 46 | {{ end }} 47 | ) 48 | {{ end }} 49 | {{ if .tables }} 50 | AND TABLE_NAME IN ({{ .tables | sqlStringIn }}) 51 | {{ end }} 52 | {{ if .tables_like }} 53 | AND ( 54 | {{ range $index, $table := .tables_like }} 55 | {{ if $index }}OR{{end}} 56 | TABLE_NAME LIKE {{ $table | sqlStringLike }} 57 | {{ end }} 58 | ) 59 | {{ end }} 60 | {{ if .columns }} 61 | AND COLUMN_NAME IN ({{ .columns | sqlStringIn }}) 62 | {{ end }} 63 | {{ if .columns_like }} 64 | AND ( 65 | {{ range $index, $column := .columns_like }} 66 | {{ if $index }}OR{{end}} 67 | COLUMN_NAME LIKE {{ $column | sqlStringLike }} 68 | {{ end }} 69 | ) 70 | {{ end }} 71 | {{ if .type }} 72 | AND COLUMN_TYPE = '{{ .type }}' 73 | {{ end }} 74 | ORDER BY table_name 75 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/schema/short.yaml: -------------------------------------------------------------------------------- 1 | name: short 2 | aliasFor: schema 3 | flags: 4 | fields: database_name,table_name,column_name,column_type -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/tables.yaml: -------------------------------------------------------------------------------- 1 | name: tables 2 | short: Get all tables from the database. 3 | flags: 4 | - name: db_schema 5 | type: stringList 6 | help: List of schemas 7 | - name: table 8 | type: stringList 9 | help: List of tables 10 | - name: engine 11 | type: choiceList 12 | choices: ['InnoDB', 'MyISAM', 'MEMORY', 'MERGE', 'ARCHIVE', 'FEDERATED', 'BLACKHOLE', 'CSV', 'NDB', 'PERFORMANCE_SCHEMA', 'TokuDB', 'RocksDB', 'Aria'] 13 | help: Engine type 14 | - name: order_by 15 | type: string 16 | default: TABLE_NAME ASC 17 | help: Order by 18 | - name: limit 19 | help: Limit the number of results 20 | type: int 21 | default: 0 22 | - name: offset 23 | type: int 24 | help: Offset 25 | default: 0 26 | query: | 27 | {{ if .explain }} 28 | EXPLAIN 29 | {{ end }} 30 | SELECT 31 | TABLE_SCHEMA, 32 | TABLE_NAME, 33 | ENGINE 34 | FROM INFORMATION_SCHEMA.TABLES 35 | WHERE 1=1 36 | {{ if .db_schema }} 37 | AND TABLE_SCHEMA IN ({{ .db_schema | sqlStringIn }}) 38 | {{ end }} 39 | {{ if .table }} 40 | AND TABLE_NAME IN ({{ .table | sqlStringIn }}) 41 | {{ end }} 42 | {{ if .engine }} 43 | AND ENGINE IN ({{ .engine | sqlStringIn }}) 44 | {{ end }} 45 | ORDER BY {{ .order_by }} 46 | {{ if .limit }} 47 | LIMIT {{ .limit }} 48 | {{ if .offset }} 49 | OFFSET {{ .offset }} 50 | {{ end }} 51 | {{ end }} 52 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/mysql/users.yaml: -------------------------------------------------------------------------------- 1 | name: users 2 | short: List users from the mysql.user table with various filters. 3 | flags: 4 | - name: hosts 5 | type: stringList 6 | help: Host of the user 7 | - name: users 8 | type: stringList 9 | help: Username 10 | - name: users_like 11 | type: stringList 12 | help: Username pattern for LIKE search 13 | - name: password_expired 14 | type: choice 15 | choices: ['Y', 'N'] 16 | help: Filter users by password expired status 17 | - name: active_privileges 18 | type: stringList 19 | help: List of privileges to check if they are active (Y) 20 | - name: limit 21 | type: int 22 | help: Limit the number of results 23 | default: 10 24 | - name: offset 25 | type: int 26 | help: Offset for the results 27 | default: 0 28 | - name: order_by 29 | type: string 30 | default: User 31 | help: Order by column 32 | query: | 33 | SELECT 34 | User, 35 | Host, 36 | authentication_string, 37 | password_expired, 38 | password_last_changed, 39 | password_lifetime, 40 | max_connections, 41 | max_questions, 42 | max_updates, 43 | max_user_connections 44 | FROM mysql.user 45 | WHERE 1=1 46 | {{ if .hosts }} 47 | AND Host IN ({{ .hosts | sqlStringIn }}) 48 | {{ end }} 49 | {{ if .users }} 50 | AND User IN ({{ .users | sqlStringIn }}) 51 | {{ end }} 52 | {{ if .user_like }} 53 | {{ $first := true }} 54 | {{ range .user_like }} 55 | {{ if $first }} 56 | AND ( 57 | {{ $first = false }} 58 | {{ else }} 59 | OR 60 | {{ end }} 61 | User LIKE '{{ . | sqlStringLike }}' 62 | {{ end }} 63 | ) 64 | {{ end }} 65 | {{ if .password_expired }} 66 | AND password_expired = '{{ .password_expired }}' 67 | {{ end }} 68 | {{ if .active_privileges }} 69 | {{ range .active_privileges }} 70 | AND {{ . }} = 'Y' 71 | {{ end }} 72 | {{ end }} 73 | ORDER BY {{ .order_by | sqlEscape }} 74 | LIMIT {{ .limit }} 75 | OFFSET {{ .offset }} -------------------------------------------------------------------------------- /cmd/sqleton/queries/pg/connections.yaml: -------------------------------------------------------------------------------- 1 | name: connections 2 | short: Show all current PostgreSQL connections 3 | flags: 4 | - name: dbuser 5 | type: string 6 | help: Database user 7 | - name: dbname 8 | type: string 9 | help: Database name 10 | - name: client_addr 11 | type: string 12 | help: Client address 13 | - name: state 14 | type: string 15 | help: State of the connection (e.g., active, idle) 16 | - name: application_name 17 | type: string 18 | help: Application name 19 | - name: limit 20 | type: int 21 | help: Limit the number of results 22 | default: 0 23 | - name: offset 24 | type: int 25 | help: Offset 26 | default: 0 27 | - name: order_by 28 | type: string 29 | default: backend_start DESC 30 | help: Order by 31 | tags: 32 | - pg 33 | - admin 34 | query: | 35 | {{ if .explain }} 36 | EXPLAIN 37 | {{ end }} 38 | SELECT 39 | pid, 40 | usename AS user, 41 | datname AS dbname, 42 | client_addr, 43 | state, 44 | application_name, 45 | backend_start, 46 | query 47 | FROM pg_stat_activity 48 | WHERE 1=1 49 | {{ if .dbuser }} 50 | AND usename = '{{ .dbuser }}' 51 | {{ end }} 52 | {{ if .dbname }} 53 | AND datname = '{{ .dbname }}' 54 | {{ end }} 55 | {{ if .client_addr }} 56 | AND client_addr = '{{ .client_addr }}' 57 | {{ end }} 58 | {{ if .state }} 59 | AND state = '{{ .state }}' 60 | {{ end }} 61 | {{ if .application_name }} 62 | AND application_name = '{{ .application_name }}' 63 | {{ end }} 64 | ORDER BY {{ .order_by }} 65 | {{ if .limit }} 66 | LIMIT {{ .limit }} 67 | {{ if .offset }} 68 | OFFSET {{ .offset }} 69 | {{ end }} 70 | {{ end }} 71 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/pg/kill-connections.yaml: -------------------------------------------------------------------------------- 1 | name: kill-connections 2 | short: Kill a specific PostgreSQL connection or all connections from a specific user or to a specific database 3 | flags: 4 | - name: pid 5 | type: int 6 | help: Process ID of the connection to kill 7 | - name: dbuser 8 | type: string 9 | help: Kill all connections from this database user 10 | - name: dbname 11 | type: string 12 | help: Kill all connections to this database 13 | query: | 14 | {{ if .pid }} 15 | SELECT pg_terminate_backend({{ .pid }}); 16 | {{ else if .dbuser }} 17 | SELECT pg_terminate_backend(pid) 18 | FROM pg_stat_activity 19 | WHERE usename = '{{ .dbuser }}' 20 | AND pid <> pg_backend_pid(); 21 | {{ else if .dbname }} 22 | SELECT pg_terminate_backend(pid) 23 | FROM pg_stat_activity 24 | WHERE datname = '{{ .dbname }}' 25 | AND pid <> pg_backend_pid(); 26 | {{ else }} 27 | -- If no flags are provided, raise an error 28 | RAISE EXCEPTION 'You must provide either a pid, dbuser, or dbname'; 29 | {{ end }} 30 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/pg/locks.yaml: -------------------------------------------------------------------------------- 1 | name: locks 2 | short: Show PostgreSQL locks 3 | flags: 4 | - name: mode 5 | type: string 6 | help: Lock mode (e.g., ExclusiveLock) 7 | - name: state 8 | type: string 9 | help: State of the activity (e.g., idle) 10 | - name: relname 11 | type: string 12 | help: Relation name (table name) 13 | - name: limit 14 | type: int 15 | help: Limit the number of results 16 | default: 0 17 | - name: offset 18 | type: int 19 | help: Offset 20 | default: 0 21 | - name: order_by 22 | type: string 23 | default: query_start DESC 24 | help: Order by 25 | query: | 26 | {{ if .explain }} 27 | EXPLAIN 28 | {{ end }} 29 | SELECT 30 | pg_stat_activity.pid, 31 | pg_class.relname, 32 | pg_locks.transactionid, 33 | pg_locks.granted, 34 | pg_stat_activity.query AS current_query, 35 | pg_stat_activity.state, 36 | pg_stat_activity.query_start, 37 | age(now(), pg_stat_activity.query_start) AS "age" 38 | FROM pg_stat_activity 39 | JOIN pg_locks ON pg_stat_activity.pid = pg_locks.pid 40 | JOIN pg_class ON pg_locks.relation = pg_class.oid 41 | WHERE 1=1 42 | {{ if .mode }} 43 | AND pg_locks.mode = '{{ .mode }}' 44 | {{ end }} 45 | {{ if .state }} 46 | AND pg_stat_activity.state = '{{ .state }}' 47 | {{ end }} 48 | {{ if .relname }} 49 | AND pg_class.relname = '{{ .relname }}' 50 | {{ end }} 51 | ORDER BY {{ .order_by }} 52 | {{ if .limit }} 53 | LIMIT {{ .limit }} 54 | {{ if .offset }} 55 | OFFSET {{ .offset }} 56 | {{ end }} 57 | {{ end }} 58 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/sqlite/hishtory/ls.yaml: -------------------------------------------------------------------------------- 1 | name: ls 2 | short: Get all history entries from the database. 3 | flags: 4 | - name: local_username 5 | type: stringList 6 | help: List of local usernames 7 | - name: hostname 8 | type: stringList 9 | help: List of hostnames 10 | - name: command 11 | type: stringList 12 | help: List of commands 13 | - name: command_like 14 | type: stringList 15 | help: List of commands to match using LIKE 16 | - name: cwd 17 | type: stringList 18 | help: List of current working directories 19 | - name: cwd_like 20 | type: stringList 21 | help: List of current working directories to match using LIKE 22 | - name: home 23 | type: stringList 24 | help: List of home directories 25 | - name: home_like 26 | type: stringList 27 | help: List of home directories to match using LIKE 28 | - name: exit_code 29 | type: intList 30 | help: List of exit codes 31 | - name: from 32 | type: date 33 | help: Start time 34 | - name: to 35 | type: date 36 | help: End time 37 | - name: device_id 38 | type: stringList 39 | help: List of device ids 40 | - name: entry_id 41 | type: stringList 42 | help: List of entry ids 43 | - name: limit 44 | help: Limit the number of results 45 | type: int 46 | default: 10 47 | - name: offset 48 | type: int 49 | help: Offset 50 | default: 0 51 | - name: order_by 52 | type: string 53 | default: start_time DESC 54 | help: Order by 55 | - name: verbose 56 | type: bool 57 | default: false 58 | help: Display all columns 59 | query: | 60 | {{ if .explain }}EXPLAIN{{ end }} 61 | SELECT 62 | local_username AS user, 63 | hostname AS host, 64 | command AS cmd, 65 | current_working_directory AS cwd, 66 | exit_code, 67 | strftime('%Y-%m-%d %H:%M:%S', start_time) AS start_time, 68 | strftime('%Y-%m-%d %H:%M:%S', end_time) AS end_time 69 | {{ if .verbose -}} 70 | , home_directory AS home 71 | , device_id 72 | , entry_id 73 | {{- end }} 74 | FROM history_entries 75 | WHERE 1=1 76 | {{ if .local_username }}AND local_username IN ({{ .local_username | sqlStringIn }}){{ end }} 77 | {{ if .hostname }}AND hostname IN ({{ .hostname | sqlStringIn }}){{ end }} 78 | {{ if .command }}AND command IN ({{ .command | sqlStringIn }}){{ end }} 79 | {{ if .command_like }}AND ({{ range $index, $element := .command_like }}{{ if $index }} OR {{ end }}command LIKE {{ $element | sqlLike }}{{ end }}){{ end }} 80 | {{ if .cwd }}AND current_working_directory IN ({{ .cwd | sqlStringIn }}){{ end }} 81 | {{ if .cwd_like }}AND ({{ range $index, $element := .cwd_like }}{{ if $index }} OR {{ end }}current_working_directory LIKE {{ $element | sqlLike }}{{ end }}){{ end }} 82 | {{ if .home }}AND home_directory IN ({{ .home | sqlStringIn }}){{ end }} 83 | {{ if .home_like }}AND ({{ range $index, $element := .home_like }}{{ if $index }} OR {{ end }}home_directory LIKE {{ $element | sqlLike }}{{ end }}){{ end }} 84 | {{ if .exit_code }}AND exit_code IN ({{ .exit_code | sqlIntIn }}){{ end }} 85 | {{ if .from }}AND start_time >= {{ .from | sqliteDateTime }}{{ end }} 86 | {{ if .to }}AND start_time <= {{ .to | sqliteDateTime }}{{ end }} 87 | {{ if .device_id }}AND device_id IN ({{ .device_id | sqlStringIn }}){{ end }} 88 | {{ if .entry_id }}AND entry_id IN ({{ .entry_id | sqlStringIn }}){{ end }} 89 | ORDER BY {{ .order_by }} 90 | {{ if .limit }}LIMIT {{ .limit }}{{ if .offset }} OFFSET {{ .offset }}{{ end }}{{ end }} 91 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/sqlite/tables.yaml: -------------------------------------------------------------------------------- 1 | name: tables 2 | short: Display information about tables in a sqlite database. 3 | flags: 4 | - name: table_name 5 | type: stringList 6 | help: List of table names 7 | - name: column_name 8 | type: stringList 9 | help: List of column names 10 | - name: column_type 11 | type: stringList 12 | help: List of column types 13 | - name: column_like 14 | type: stringList 15 | help: List of column names to match using LIKE 16 | - name: type_like 17 | type: stringList 18 | help: List of column types to match using LIKE 19 | - name: limit 20 | help: Limit the number of results 21 | type: int 22 | default: 0 23 | - name: offset 24 | type: int 25 | help: Offset 26 | default: 0 27 | - name: order_by 28 | type: string 29 | default: name ASC 30 | help: Order by 31 | query: | 32 | {{ if .explain }} 33 | EXPLAIN 34 | {{ end }} 35 | SELECT 36 | name, 37 | sql 38 | FROM sqlite_master 39 | WHERE type='table' 40 | {{ if .table_name }} 41 | AND name IN ({{ .table_name | sqlStringIn }}) 42 | {{ end }} 43 | {{ if .column_name }} 44 | AND sql LIKE '%{{ .column_name | sqlStringIn }}%' 45 | {{ end }} 46 | {{ if .column_type }} 47 | AND sql LIKE '%{{ .column_type | sqlStringIn }}%' 48 | {{ end }} 49 | {{ if .column_like }} 50 | {{ range $index, $value := .column_like }} 51 | {{ if gt $index 0 }} OR {{ end }} 52 | AND sql LIKE '%{{ $value }}%' 53 | {{ end }} 54 | {{ end }} 55 | {{ if .type_like }} 56 | {{ range $index, $value := .type_like }} 57 | {{ if gt $index 0 }} OR {{ end }} 58 | AND sql LIKE '%{{ $value }}%' 59 | {{ end }} 60 | {{ end }} 61 | ORDER BY {{ .order_by }} 62 | {{ if .limit }} 63 | LIMIT {{ .limit }} 64 | {{ if .offset }} 65 | OFFSET {{ .offset }} 66 | {{ end }} 67 | {{ end }} -------------------------------------------------------------------------------- /cmd/sqleton/queries/wp/ls-posts.yaml: -------------------------------------------------------------------------------- 1 | name: ls-posts 2 | short: "Show all WP posts" 3 | long: Show all posts and their ID 4 | flags: 5 | - name: limit 6 | shortFlag: l 7 | type: int 8 | default: 10 9 | help: Limit the number of posts 10 | - name: offset 11 | type: int 12 | help: Offset 13 | default: 0 14 | - name: status 15 | type: stringList 16 | help: Select posts by status 17 | required: false 18 | - name: order_by 19 | type: string 20 | help: Order by column 21 | - name: ids 22 | type: intList 23 | help: Select posts by id 24 | required: false 25 | - name: types 26 | type: stringList 27 | default: 28 | - post 29 | - page 30 | help: Select posts by type 31 | required: false 32 | - name: body_like 33 | type: stringList 34 | help: Select posts by body 35 | required: false 36 | - name: from 37 | type: date 38 | help: Select posts from date 39 | required: false 40 | - name: to 41 | type: date 42 | help: Select posts to date 43 | required: false 44 | - name: title_like 45 | type: string 46 | help: Select posts by title 47 | required: false 48 | - name: slugs_like 49 | type: stringList 50 | help: Select posts by slug patterns 51 | required: false 52 | - name: templates_like 53 | type: stringList 54 | help: Select posts by template patterns 55 | required: false 56 | - name: slugs 57 | type: stringList 58 | help: Select posts by slug 59 | required: false 60 | - name: templates 61 | type: stringList 62 | help: Select posts by template 63 | required: false 64 | - name: group_by 65 | type: choiceList 66 | choices: 67 | - status 68 | - type 69 | - template 70 | - date 71 | - slug 72 | help: Group and count posts by selected field 73 | required: false 74 | query: | 75 | {{ if not .group_by }} 76 | SELECT 77 | wp.ID, 78 | wp.post_title AS title, 79 | wp.post_name AS slug, 80 | wp.post_type AS type, 81 | wp.post_status AS status, 82 | wpm.meta_value AS template, 83 | wp.post_date AS date 84 | FROM wp_posts wp 85 | LEFT JOIN wp_postmeta wpm ON wp.ID = wpm.post_id AND wpm.meta_key = '_wp_page_template' 86 | {{ else }} 87 | SELECT 88 | {{ if has "status" .group_by }}wp.post_status AS status,{{ end }} 89 | {{ if has "type" .group_by }}wp.post_type AS type,{{ end }} 90 | {{ if has "template" .group_by }}wpm.meta_value AS template,{{ end }} 91 | {{ if has "date" .group_by }}wp.post_date AS date,{{ end }} 92 | {{ if has "slug" .group_by }}wp.post_name AS slug,{{ end }} 93 | COUNT(*) AS count 94 | FROM wp_posts wp 95 | LEFT JOIN wp_postmeta wpm ON wp.ID = wpm.post_id AND wpm.meta_key = '_wp_page_template' 96 | {{ end }} 97 | WHERE 98 | post_type IN ({{ .types | sqlStringIn }}) 99 | 100 | {{ if .status -}} AND post_status IN ({{ .status | sqlStringIn }}) {{- end -}} 101 | 102 | {{ if .from -}} AND post_date >= {{ .from | sqlDate }} {{- end -}} 103 | 104 | {{ if .to -}} AND post_date <= {{ .to | sqlDate }} {{- end -}} 105 | 106 | {{ if .title_like -}} AND post_title LIKE {{ .title_like | sqlLike }} {{- end -}} 107 | 108 | {{- if .slugs_like }} 109 | AND ( 110 | {{- range $index, $slug_pattern := .slugs_like }} 111 | {{- if $index }} OR {{end -}} wp.post_name LIKE {{ $slug_pattern | sqlLike }} 112 | {{- end }} 113 | ) 114 | {{- end }} 115 | 116 | {{- if .templates_like }} 117 | AND ( 118 | {{- range $index, $template_pattern := .templates_like }} 119 | {{- if $index }} OR {{ end -}} 120 | wpm.meta_value LIKE {{ $template_pattern | sqlLike }} 121 | {{- end }} 122 | ) 123 | {{- end }} 124 | 125 | {{- if .body_like }} 126 | AND ( 127 | {{- range $index, $body_pattern := .body_like }} 128 | {{- if $index }} OR {{ end -}} 129 | wp.post_content LIKE {{ $body_pattern | sqlLike }} 130 | {{- end }} 131 | ) 132 | {{- end }} 133 | 134 | {{ if .slugs -}} AND post_name IN ({{ .slugs | sqlStringIn }}) {{- end -}} 135 | 136 | {{ if .templates -}} AND wpm.meta_value IN ({{ .templates | sqlStringIn }}) {{- end -}} 137 | 138 | {{ if .ids -}} AND ID IN ({{ .ids | sqlIntIn }}) {{- end -}} 139 | 140 | {{ if .group_by -}} 141 | GROUP BY {{ .group_by | join ", " }} 142 | {{ end }} 143 | 144 | {{ if not .order_by -}} 145 | {{if .group_by }} 146 | ORDER BY count DESC 147 | {{else -}} 148 | ORDER BY post_date DESC 149 | {{ end }} 150 | {{ else }} 151 | ORDER BY {{ .order_by }} 152 | {{end}} 153 | {{ if .limit }} 154 | LIMIT {{ .limit }} 155 | OFFSET {{ .offset }} 156 | {{ end }} 157 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/wp/post-content.yaml: -------------------------------------------------------------------------------- 1 | name: post-content 2 | short: Get all posts from the WordPress database. 3 | flags: 4 | - name: id 5 | type: intList 6 | help: List of post ids 7 | - name: title 8 | type: string 9 | help: Post title 10 | - name: content 11 | type: string 12 | help: Post content 13 | - name: excerpt 14 | type: string 15 | help: Post excerpt 16 | - name: post_type 17 | type: stringList 18 | help: Post type 19 | default: 20 | - post 21 | - name: status 22 | type: string 23 | help: Post status 24 | - name: author 25 | type: string 26 | help: Post author 27 | - name: template_name 28 | type: string 29 | help: Template name 30 | - name: categories 31 | type: stringList 32 | help: Post categories 33 | - name: tags 34 | type: stringList 35 | help: Post tags 36 | - name: limit 37 | help: Limit the number of results 38 | type: int 39 | default: 5 40 | - name: offset 41 | type: int 42 | help: Offset 43 | default: 0 44 | - name: order_by 45 | type: string 46 | default: post_date DESC 47 | help: Order by 48 | query: | 49 | {{ if .explain }} 50 | EXPLAIN 51 | {{ end }} 52 | WITH CategorySubquery AS ( 53 | SELECT 54 | cat_rel.object_id AS post_id, 55 | GROUP_CONCAT(DISTINCT cat.name ORDER BY cat.name ASC) AS categories 56 | FROM 57 | wp_term_relationships cat_rel 58 | LEFT JOIN wp_term_taxonomy cat_tax ON cat_rel.term_taxonomy_id = cat_tax.term_taxonomy_id AND cat_tax.taxonomy = 'category' 59 | LEFT JOIN wp_terms cat ON cat_tax.term_id = cat.term_id 60 | GROUP BY 61 | cat_rel.object_id 62 | ), 63 | TagSubquery AS ( 64 | SELECT 65 | tag_rel.object_id AS post_id, 66 | GROUP_CONCAT(DISTINCT tag.name ORDER BY tag.name ASC) AS tags 67 | FROM 68 | wp_term_relationships tag_rel 69 | LEFT JOIN wp_term_taxonomy tag_tax ON tag_rel.term_taxonomy_id = tag_tax.term_taxonomy_id AND tag_tax.taxonomy = 'post_tag' 70 | LEFT JOIN wp_terms tag ON tag_tax.term_id = tag.term_id 71 | GROUP BY 72 | tag_rel.object_id 73 | ) 74 | SELECT 75 | p.ID, 76 | p.post_title, 77 | p.post_content, 78 | p.post_excerpt, 79 | p.post_type, 80 | p.post_status, 81 | u.display_name AS author, 82 | pm.meta_value AS template_name, 83 | cat_sub.categories, 84 | tag_sub.tags 85 | FROM 86 | wp_posts p 87 | LEFT JOIN wp_users u ON p.post_author = u.ID 88 | LEFT JOIN wp_postmeta pm ON p.ID = pm.post_id AND pm.meta_key = '_wp_page_template' 89 | LEFT JOIN CategorySubquery cat_sub ON p.ID = cat_sub.post_id 90 | LEFT JOIN TagSubquery tag_sub ON p.ID = tag_sub.post_id 91 | WHERE 92 | p.post_status = 'publish' 93 | {{ if .post_type }} 94 | AND p.post_type IN ({{ .post_type | sqlStringIn }}) 95 | {{ end }} 96 | {{ if .id }} 97 | AND p.ID IN ({{ .id | sqlIntIn }}) 98 | {{ end }} 99 | {{ if .title }} 100 | AND p.post_title LIKE '%{{ .title }}%' 101 | {{ end }} 102 | {{ if .content }} 103 | AND p.post_content LIKE '%{{ .content }}%' 104 | {{ end }} 105 | {{ if .excerpt }} 106 | AND p.post_excerpt LIKE '%{{ .excerpt }}%' 107 | {{ end }} 108 | {{ if .status }} 109 | AND p.post_status = '{{ .status }}' 110 | {{ end }} 111 | {{ if .author }} 112 | AND u.display_name = '{{ .author }}' 113 | {{ end }} 114 | {{ if .template_name }} 115 | AND pm.meta_value = '{{ .template_name }}' 116 | {{ end }} 117 | {{ if .categories }} 118 | AND cat_sub.categories LIKE CONCAT('%', {{ range $index, $element := .categories }}{{ if $index }}, {{ end }}'{{ $element }}', '%'){{ end }} 119 | {{ end }} 120 | {{ if .tags }} 121 | AND tag_sub.tags LIKE CONCAT('%', {{ range $index, $element := .tags }}{{ if $index }}, {{ end }}'{{ $element }}', '%'){{ end }} 122 | {{ end }} 123 | ORDER BY {{ .order_by }} 124 | {{ if .limit }} 125 | LIMIT {{ .limit }} 126 | {{ if .offset }} 127 | OFFSET {{ .offset }} 128 | {{ end }} 129 | {{ end }} 130 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/wp/posts-counts.yaml: -------------------------------------------------------------------------------- 1 | name: posts-counts 2 | short: Count posts by type 3 | flags: 4 | - name: post_type 5 | description: Post type 6 | type: stringList 7 | required: false 8 | subqueries: 9 | post_types: | 10 | SELECT post_type 11 | FROM wp_posts 12 | GROUP BY post_type 13 | LIMIT 4 14 | query: | 15 | {{ $types := sqlColumn (subQuery "post_types") }} 16 | SELECT 17 | {{ range $i, $v := $types }} 18 | {{ $v2 := printf "count%d" $i }} 19 | ( 20 | SELECT count(*) AS count 21 | FROM wp_posts 22 | WHERE post_status = 'publish' 23 | AND post_type = '{{ $v }}' 24 | ) AS `{{ $v }}` {{ if not (eq $i (sub (len $types) 1)) }}, {{ end }} 25 | {{ end }} 26 | -------------------------------------------------------------------------------- /cmd/sqleton/queries/wp/wc/categories.yaml: -------------------------------------------------------------------------------- 1 | name: categories 2 | short: Get product categories from the database. 3 | flags: 4 | - name: slug 5 | type: stringList 6 | help: Category slug 7 | - name: slug_like 8 | type: stringList 9 | help: List of slugs to match with LIKE 10 | - name: name 11 | type: stringList 12 | help: Category name 13 | - name: name_like 14 | type: stringList 15 | help: List of names to match with LIKE 16 | - name: limit 17 | help: Limit the number of results 18 | type: int 19 | default: 0 20 | - name: offset 21 | type: int 22 | help: Offset 23 | default: 0 24 | - name: order_by 25 | type: string 26 | default: tt.description DESC 27 | help: Order by 28 | - name: with_content 29 | type: bool 30 | help: Include content 31 | query: | 32 | {{ if .explain }} 33 | EXPLAIN 34 | {{ end }} 35 | SELECT 36 | tt.term_id AS id 37 | , tt.count AS count 38 | , tt.parent AS parent_id 39 | , tt.taxonomy AS taxonomy 40 | , t.name AS name 41 | {{ if .with_content }} 42 | , tt.description AS description 43 | {{ end }} 44 | FROM wp_terms AS t 45 | INNER JOIN wp_term_taxonomy AS tt ON t.term_id = tt.term_id 46 | WHERE tt.taxonomy = 'product_cat' 47 | {{ if .slug }} 48 | AND t.slug IN ({{ .slug | sqlStringIn }}) 49 | {{ end }} 50 | {{ if .name }} 51 | AND t.name IN ({{ .name | sqlStringIn }}) 52 | {{ end }} 53 | {{ if .name_like }} 54 | AND ( 55 | {{ range $index, $value := .name_like }} 56 | {{ if $index }} 57 | OR 58 | {{ end }} 59 | t.name LIKE '%{{ $value }}%' 60 | {{ end }} 61 | ) 62 | {{ end }} 63 | {{ if .from }} 64 | AND p.post_date >= {{ .from | sqlDate }} 65 | {{ end }} 66 | {{ if .to }} 67 | AND p.post_date <= {{ .to | sqlDate }} 68 | {{ end }} 69 | {{ if .slug_like }} 70 | AND ( 71 | {{ range $index, $value := .slug_like }} 72 | {{ if $index }} 73 | OR 74 | {{ end }} 75 | t.slug LIKE '%{{ $value }}%' 76 | {{ end }} 77 | ) 78 | {{ end }} 79 | ORDER BY {{ .order_by }} 80 | {{ if .limit }} 81 | LIMIT {{ .limit }} 82 | {{ if .offset }} 83 | OFFSET {{ .offset }} 84 | {{ end }} 85 | {{ end }} -------------------------------------------------------------------------------- /cmd/sqleton/queries/wp/wc/tax-rates.yaml: -------------------------------------------------------------------------------- 1 | name: tax-rates 2 | short: Get all tax rates from the WooCommerce database. 3 | flags: 4 | - name: id 5 | type: intList 6 | help: List of tax rate ids 7 | - name: name 8 | type: string 9 | help: Tax rate name 10 | - name: name_like 11 | type: stringList 12 | help: Tax rate name patterns 13 | - name: class 14 | type: stringList 15 | help: Tax rate classes 16 | - name: country 17 | type: stringList 18 | help: Tax rate countries 19 | - name: state 20 | type: stringList 21 | help: Tax rate states 22 | - name: limit 23 | help: Limit the number of results 24 | type: int 25 | default: 0 26 | - name: offset 27 | type: int 28 | help: Offset 29 | default: 0 30 | - name: order_by 31 | type: string 32 | default: tax_rate_id DESC 33 | help: Order by 34 | query: | 35 | {{ if .explain }} 36 | EXPLAIN 37 | {{ end }} 38 | SELECT 39 | tax_rate_id, 40 | tax_rate, 41 | tax_rate_class, 42 | tax_rate_compound, 43 | tax_rate_country, 44 | tax_rate_name, 45 | tax_rate_order, 46 | tax_rate_priority, 47 | tax_rate_shipping, 48 | tax_rate_state 49 | FROM wp_woocommerce_tax_rates 50 | WHERE 1=1 51 | {{ if .id }} 52 | AND tax_rate_id IN ({{ .id | sqlIntIn }}) 53 | {{ end }} 54 | {{ if .name }} 55 | AND tax_rate_name = '{{ .name }}' 56 | {{ end }} 57 | {{ if .name_like }} 58 | AND ({{ range $index, $element := .name_like }}{{ if gt $index 0 }} OR {{ end }}tax_rate_name LIKE '{{ $element }}'{{ end }}) 59 | {{ end }} 60 | {{ if .class }} 61 | AND tax_rate_class IN ({{ .class | sqlStringIn }}) 62 | {{ end }} 63 | {{ if .country }} 64 | AND tax_rate_country IN ({{ .country | sqlStringIn }}) 65 | {{ end }} 66 | {{ if .state }} 67 | AND tax_rate_state IN ({{ .state | sqlStringIn }}) 68 | {{ end }} 69 | ORDER BY {{ .order_by }} 70 | {{ if .limit }} 71 | LIMIT {{ .limit }} 72 | {{ if .offset }} 73 | OFFSET {{ .offset }} 74 | {{ end }} 75 | {{ end }} -------------------------------------------------------------------------------- /doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-go-golems/sqleton/7c4bc3869206a2c125ce96faf01909ebcff16b59/doc/logo.png -------------------------------------------------------------------------------- /doc/vhs/demo.tape: -------------------------------------------------------------------------------- 1 | Output doc/gifs/demo.gif 2 | 3 | Set FontSize 14 4 | Set Width 1040 5 | Set Height 1040 6 | 7 | Set TypingSpeed 0.01 8 | 9 | Type "sqleton db ls --use-dbt-profiles --fields name,type,hostname" Enter 10 | Sleep 1200ms 11 | 12 | Type "sqleton run examples/show-processlist.sql --output yaml" Enter 13 | Sleep 2000ms 14 | 15 | Type "sqleton mysql ps --select-template '{{.Id}} -- {{.User}}'" Enter 16 | Sleep 1200ms 17 | 18 | Type "sqleton wp ls-posts --fields ID,post_title" Enter 19 | Sleep 1200ms 20 | 21 | Type "sqleton help wp --examples" Enter 22 | Sleep 3000ms 23 | 24 | Type "sqleton wp ls-posts --limit 100 --status publish --order-by post_title \" Enter 25 | Type " --from 2017-01-01 --to 2017-10-01 \" Enter 26 | Type " --fields ID,post_title,post_date \" Enter 27 | Type " --title-like Shrubs" Enter 28 | Sleep 2000ms 29 | 30 | Type "cat ~/.sqleton/queries/ttc/01-orders.yaml" Enter 31 | Sleep 1200ms 32 | 33 | Type "sqleton ttc orders" Enter 34 | Sleep 1200ms 35 | 36 | Type "sqleton ttc ls-orders --limit 10 --from 'last year' --to 'today' --print-query" Enter 37 | Sleep 1200ms 38 | 39 | Type "sqleton ttc ls-orders --limit 10 --from 'last year' --to 'today' --fields ID,post_date,post_status " Enter 40 | Sleep 1200ms 41 | 42 | Sleep 2000ms 43 | 44 | -------------------------------------------------------------------------------- /docker-compose.override.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | sqleton: 4 | # env_file: 5 | # - .env.docker 6 | volumes: 7 | - ./docker/sqleton:/root/.sqleton 8 | - /Users/manuel/code/ttc/ttc/sql/sqleton:/queries 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | sqleton: 4 | build: 5 | context: . 6 | args: 7 | GOOS: linux 8 | GOARCH: arm64 9 | image: go-go-golems/sqleton:arm64v8 10 | ports: 11 | - "8080:8080" 12 | command: ["serve", "--serve-host", "0.0.0.0", "--serve-port", "8080"] 13 | -------------------------------------------------------------------------------- /examples/config.yml: -------------------------------------------------------------------------------- 1 | type: mysql 2 | host: localhost 3 | port: 3336 4 | user: root 5 | password: somewordpress 6 | schema: wp 7 | database: wp 8 | -------------------------------------------------------------------------------- /examples/show-processlist.sql: -------------------------------------------------------------------------------- 1 | SHOW PROCESSLIST 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-go-golems/sqleton 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/dave/jennifer v1.7.0 7 | github.com/go-go-golems/clay v0.1.39 8 | github.com/go-go-golems/glazed v0.5.48 9 | github.com/go-go-golems/parka v0.5.26 10 | github.com/go-sql-driver/mysql v1.9.1 11 | github.com/huandu/go-sqlbuilder v1.35.0 12 | github.com/iancoleman/strcase v0.3.0 13 | github.com/jmoiron/sqlx v1.4.0 14 | github.com/mattn/go-sqlite3 v1.14.24 15 | github.com/pkg/errors v0.9.1 16 | github.com/pkg/profile v1.7.0 17 | github.com/rs/zerolog v1.34.0 18 | github.com/spf13/cobra v1.9.1 19 | github.com/spf13/viper v1.20.1 20 | github.com/stretchr/testify v1.10.0 21 | golang.org/x/sync v0.13.0 22 | gopkg.in/yaml.v3 v3.0.1 23 | ) 24 | 25 | require ( 26 | filippo.io/edwards25519 v1.1.0 // indirect 27 | github.com/BurntSushi/toml v1.2.1 // indirect 28 | github.com/Masterminds/goutils v1.1.1 // indirect 29 | github.com/Masterminds/semver v1.5.0 // indirect 30 | github.com/Masterminds/sprig v2.22.0+incompatible // indirect 31 | github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect 32 | github.com/adrg/frontmatter v0.2.0 // indirect 33 | github.com/alecthomas/chroma/v2 v2.16.0 // indirect 34 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect 35 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 36 | github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect 37 | github.com/aws/aws-sdk-go-v2/config v1.29.6 // indirect 38 | github.com/aws/aws-sdk-go-v2/credentials v1.17.59 // indirect 39 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect 40 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect 41 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect 42 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect 43 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/ssm v1.58.1 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect 47 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect 48 | github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect 49 | github.com/aws/smithy-go v1.22.2 // indirect 50 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 51 | github.com/aymerick/douceur v0.2.0 // indirect 52 | github.com/bahlo/generic-list-go v0.2.0 // indirect 53 | github.com/bits-and-blooms/bitset v1.22.0 // indirect 54 | github.com/blevesearch/bleve/v2 v2.5.0 // indirect 55 | github.com/blevesearch/bleve_index_api v1.2.7 // indirect 56 | github.com/blevesearch/geo v0.1.20 // indirect 57 | github.com/blevesearch/go-faiss v1.0.25 // indirect 58 | github.com/blevesearch/go-porterstemmer v1.0.3 // indirect 59 | github.com/blevesearch/gtreap v0.1.1 // indirect 60 | github.com/blevesearch/mmap-go v1.0.4 // indirect 61 | github.com/blevesearch/scorch_segment_api/v2 v2.3.9 // indirect 62 | github.com/blevesearch/segment v0.9.1 // indirect 63 | github.com/blevesearch/snowballstem v0.9.0 // indirect 64 | github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect 65 | github.com/blevesearch/vellum v1.1.0 // indirect 66 | github.com/blevesearch/zapx/v11 v11.4.1 // indirect 67 | github.com/blevesearch/zapx/v12 v12.4.1 // indirect 68 | github.com/blevesearch/zapx/v13 v13.4.1 // indirect 69 | github.com/blevesearch/zapx/v14 v14.4.1 // indirect 70 | github.com/blevesearch/zapx/v15 v15.4.1 // indirect 71 | github.com/blevesearch/zapx/v16 v16.2.2 // indirect 72 | github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect 73 | github.com/buger/jsonparser v1.1.1 // indirect 74 | github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect 75 | github.com/charmbracelet/glamour v0.9.1 // indirect 76 | github.com/charmbracelet/lipgloss v1.1.0 // indirect 77 | github.com/charmbracelet/x/ansi v0.8.0 // indirect 78 | github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect 79 | github.com/charmbracelet/x/term v0.2.1 // indirect 80 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 81 | github.com/dlclark/regexp2 v1.11.5 // indirect 82 | github.com/felixge/fgprof v0.9.3 // indirect 83 | github.com/fsnotify/fsnotify v1.9.0 // indirect 84 | github.com/go-openapi/errors v0.22.0 // indirect 85 | github.com/go-openapi/strfmt v0.23.0 // indirect 86 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 87 | github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect 88 | github.com/golang/protobuf v1.5.4 // indirect 89 | github.com/golang/snappy v0.0.4 // indirect 90 | github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect 91 | github.com/google/uuid v1.6.0 // indirect 92 | github.com/gorilla/css v1.0.1 // indirect 93 | github.com/huandu/xstrings v1.5.0 // indirect 94 | github.com/imdario/mergo v0.3.16 // indirect 95 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 96 | github.com/itchyny/gojq v0.12.12 // indirect 97 | github.com/itchyny/timefmt-go v0.1.5 // indirect 98 | github.com/jedib0t/go-pretty v4.3.0+incompatible // indirect 99 | github.com/json-iterator/go v1.1.12 // indirect 100 | github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54 // indirect 101 | github.com/kucherenkovova/safegroup v1.0.2 // indirect 102 | github.com/labstack/echo/v4 v4.13.3 // indirect 103 | github.com/labstack/gommon v0.4.2 // indirect 104 | github.com/lib/pq v1.10.9 // indirect 105 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 106 | github.com/mailru/easyjson v0.7.7 // indirect 107 | github.com/mattn/go-colorable v0.1.13 // indirect 108 | github.com/mattn/go-isatty v0.0.20 // indirect 109 | github.com/mattn/go-runewidth v0.0.16 // indirect 110 | github.com/microcosm-cc/bluemonday v1.0.27 // indirect 111 | github.com/mitchellh/copystructure v1.2.0 // indirect 112 | github.com/mitchellh/mapstructure v1.5.0 // indirect 113 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 114 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 115 | github.com/modern-go/reflect2 v1.0.2 // indirect 116 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 117 | github.com/mschoch/smat v0.2.0 // indirect 118 | github.com/muesli/reflow v0.3.0 // indirect 119 | github.com/muesli/termenv v0.16.0 // indirect 120 | github.com/oklog/ulid v1.3.1 // indirect 121 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 122 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 123 | github.com/richardlehane/mscfb v1.0.4 // indirect 124 | github.com/richardlehane/msoleps v1.0.4 // indirect 125 | github.com/rivo/uniseg v0.4.7 // indirect 126 | github.com/sagikazarmark/locafero v0.7.0 // indirect 127 | github.com/sourcegraph/conc v0.3.0 // indirect 128 | github.com/spf13/afero v1.12.0 // indirect 129 | github.com/spf13/cast v1.7.1 // indirect 130 | github.com/spf13/pflag v1.0.6 // indirect 131 | github.com/subosito/gotenv v1.6.0 // indirect 132 | github.com/tj/go-naturaldate v1.3.0 // indirect 133 | github.com/ugorji/go/codec v1.2.11 // indirect 134 | github.com/valyala/bytebufferpool v1.0.0 // indirect 135 | github.com/valyala/fasttemplate v1.2.2 // indirect 136 | github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect 137 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect 138 | github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect 139 | github.com/xuri/excelize/v2 v2.9.0 // indirect 140 | github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect 141 | github.com/yuin/goldmark v1.7.8 // indirect 142 | github.com/yuin/goldmark-emoji v1.0.5 // indirect 143 | github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87 // indirect 144 | github.com/ziflex/lecho/v3 v3.7.0 // indirect 145 | go.etcd.io/bbolt v1.4.0 // indirect 146 | go.mongodb.org/mongo-driver v1.14.0 // indirect 147 | go.uber.org/multierr v1.11.0 // indirect 148 | golang.org/x/crypto v0.36.0 // indirect 149 | golang.org/x/net v0.38.0 // indirect 150 | golang.org/x/sys v0.31.0 // indirect 151 | golang.org/x/term v0.30.0 // indirect 152 | golang.org/x/text v0.23.0 // indirect 153 | golang.org/x/time v0.8.0 // indirect 154 | google.golang.org/protobuf v1.36.1 // indirect 155 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 156 | gopkg.in/yaml.v2 v2.4.0 // indirect 157 | ) 158 | -------------------------------------------------------------------------------- /lefthook.yml: -------------------------------------------------------------------------------- 1 | pre-commit: 2 | commands: 3 | lint: 4 | glob: '*.go' 5 | run: make lintmax gosec govulncheck 6 | test: 7 | glob: '*.go' 8 | run: make test 9 | parallel: true 10 | 11 | pre-push: 12 | commands: 13 | # release: 14 | # run: make goreleaser 15 | lint: 16 | run: make lintmax gosec govulncheck 17 | test: 18 | run: make test 19 | parallel: true 20 | -------------------------------------------------------------------------------- /misc/parca.yaml: -------------------------------------------------------------------------------- 1 | object_storage: 2 | bucket: 3 | type: "FILESYSTEM" 4 | config: 5 | directory: "./data" 6 | 7 | scrape_configs: 8 | - job_name: "default" 9 | scrape_interval: "3s" 10 | static_configs: 11 | - targets: [ '127.0.0.1:8080' ] 12 | 13 | # Custom scrape endpoints can be added like just like the example below. 14 | # The profile name will be `fgprof`, and it will be scraped from the given 15 | # path and since it is a delta profile, a query parameter 16 | # ?seconds= will be added. 17 | # 18 | # profiling_config: 19 | # pprof_config: 20 | # fgprof: 21 | # enabled: true 22 | # path: /debug/pprof/fgprof 23 | # delta: true 24 | -------------------------------------------------------------------------------- /pinocchio/sqleton/create-command.yaml: -------------------------------------------------------------------------------- 1 | name: create-command 2 | short: Generate a Sqleton query 3 | flags: 4 | - name: additional_system 5 | type: stringList 6 | help: Additional system prompt 7 | - name: additional 8 | type: stringList 9 | help: Additional prompt 10 | - name: context 11 | type: fileList 12 | help: Additional context 13 | - name: ddl 14 | type: stringFromFiles 15 | help: DDL for the table 16 | required: false 17 | - name: types 18 | type: stringList 19 | help: List of types 20 | default: 21 | - int 22 | - file 23 | - fileList 24 | - string 25 | - stringList 26 | - stringFromFile 27 | - objectFromFile 28 | - objectListFromFile 29 | - stringListFromFile 30 | - intList 31 | - float 32 | - bool 33 | - floatList 34 | - choice 35 | - choiceList 36 | - name: instructions 37 | type: string 38 | help: Additional language specific instructions 39 | required: false 40 | - name: topic 41 | type: string 42 | help: Topic of the query 43 | required: false 44 | - name: instructions_file 45 | type: stringFromFiles 46 | help: Additional language specific instructions 47 | required: false 48 | - name: topic_file 49 | type: stringFromFiles 50 | help: Topic of the query 51 | required: false 52 | - name: example_name 53 | type: string 54 | help: Name of the example 55 | default: get all items from WooCommerce orders 56 | - name: example 57 | type: stringFromFiles 58 | help: Example of the table 59 | default: | 60 | name: animals 61 | short: Get all animals from the database. 62 | flags: 63 | - name: id 64 | type: intList 65 | help: List of ids 66 | - name: name 67 | type: string 68 | help: Animal name 69 | - name: weight 70 | type: float 71 | help: Animal weight 72 | - name: from 73 | type: date 74 | help: From date 75 | - name: to 76 | type: date 77 | help: To date 78 | - name: height 79 | type: float 80 | help: Animal height 81 | - name: color 82 | type: string 83 | help: Animal color 84 | - name: species 85 | type: stringList 86 | help: Animal species 87 | - name: limit 88 | help: Limit the number of results 89 | type: int 90 | default: 0 91 | - name: offset 92 | type: int 93 | help: Offset 94 | default: 0 95 | - name: order_by 96 | type: string 97 | default: birthdate DESC 98 | help: Order by 99 | query: | 100 | {{ if .explain }} 101 | EXPLAIN 102 | {{ end }} 103 | SELECT 104 | id, 105 | name, 106 | birthdate, 107 | weight, 108 | height, 109 | color, 110 | species 111 | FROM animals 112 | WHERE 1=1 113 | {{ if .id }} 114 | AND id IN ({{ .id | sqlIntIn }}) 115 | {{ end }} 116 | {{ if .name }} 117 | AND name = '{{ .name }}' 118 | {{ end }} 119 | {{ if .from }} 120 | AND birthdate >= {{ .from | sqlDate }} 121 | {{ end }} 122 | {{ if .to }} 123 | AND birthdate <= {{ .to | sqlDate }} 124 | {{ end }} 125 | {{ if .weight }} 126 | AND weight = {{ .weight }} 127 | {{ end }} 128 | {{ if .height }} 129 | AND height = {{ .height }} 130 | {{ end }} 131 | {{ if .color }} 132 | AND color = '{{ .color }}' 133 | {{ end }} 134 | {{ if .species }} 135 | AND species IN ({{ .species | sqlStringIn }}) 136 | {{ end }} 137 | ORDER BY {{ .order_by }} 138 | {{ if .limit }} 139 | LIMIT {{ .limit }} 140 | {{ if .offset }} 141 | OFFSET {{ .offset }} 142 | {{ end }} 143 | {{ end }} 144 | 145 | system-prompt: | 146 | You are an experienced SQL developer. You know how to write SQL queries. You write clearly and concisely. 147 | {{ .additional_system | join "\n" }} 148 | prompt: | 149 | I want to generate templates for SQL queries, stored in YAML and with the `query` field using go template syntax. 150 | The templates expose command line parameters that the user can use to control the query, 151 | and generate useful WHERE and GROUP BY statements. 152 | 153 | The `flags` stored in the YAML can be of different types: {{ .types | join ", " }}. These are then passed to the go 154 | template. 155 | 156 | Instead of "x > 10", the template language uses "gt x 10". 157 | 158 | Here are the sql-specific go template functions that are registered: 159 | 160 | ``` 161 | // sqlEscape escapes single quotes in a string for SQL queries. 162 | // It doubles any single quote characters to prevent SQL injection. 163 | func sqlEscape(value string) string 164 | // sqlString wraps a string value in single quotes for SQL queries. 165 | func sqlString(value string) string 166 | // sqlStringLike formats a string for use in SQL LIKE queries, wrapping the value with '%' and escaping it. 167 | func sqlStringLike(value string) string 168 | // sqlStringIn converts a slice of values into a SQL IN clause string, properly escaping and quoting each value. 169 | // Returns an error if the input cannot be cast to a slice of strings. 170 | func sqlStringIn(values interface{}) (string, error) 171 | // sqlIn converts a slice of interface{} values into a comma-separated string for SQL queries. 172 | // Each value is formatted using fmt.Sprintf with the %v verb. 173 | func sqlIn(values []interface{}) string 174 | // sqlIntIn converts a slice of integer values into a comma-separated string for SQL queries. 175 | // Returns an empty string if the input cannot be cast to a slice of int64. 176 | func sqlIntIn(values interface{}) string 177 | // sqlDate_ formats a date value for SQL queries, using different formats based on the date's timezone. 178 | // Returns an error if the date cannot be parsed or formatted. 179 | // This is a helper function used by other date formatting functions. 180 | func sqlDate_(date interface{}, fullFormat string, defaultFormat string) (string, error) 181 | // sqlDate formats a date value for SQL queries as YYYY-MM-DD or RFC3339, based on the date's timezone. 182 | // Returns an error if the date cannot be parsed or formatted. 183 | func sqlDate(date interface{}) (string, error) 184 | // sqlDateTime formats a datetime value for SQL queries as YYYY-MM-DDTHH:MM:SS or RFC3339, based on the datetime's timezone. 185 | // Returns an error if the datetime cannot be parsed or formatted. 186 | func sqlDateTime(date interface{}) (string, error) 187 | // sqliteDate formats a date value specifically for SQLite queries as YYYY-MM-DD. 188 | // Returns an error if the date cannot be parsed or formatted. 189 | func sqliteDate(date interface{}) (string, error) 190 | // sqliteDateTime formats a datetime value specifically for SQLite queries as YYYY-MM-DD HH:MM:SS. 191 | // Returns an error if the datetime cannot be parsed or formatted. 192 | func sqliteDateTime(date interface{}) (string, error) 193 | // sqlLike formats a string for use in SQL LIKE queries by wrapping the value with '%'. 194 | func sqlLike(value string) string 195 | ``` 196 | 197 | {{ if .example }} 198 | Here is an example that queries the {{ .example_name }}. 199 | 200 | {{ .example }} 201 | 202 | {{ end }} 203 | 204 | Based on these examples and the provided table structure (if any), create a YAML command that: 205 | 1. Follows the structure and naming conventions shown 206 | 2. Includes appropriate flags for filtering and control 207 | 3. Uses Go template syntax for dynamic SQL generation 208 | 4. Handles NULL values and edge cases appropriately 209 | 5. Includes proper WHERE clause construction 210 | 6. Uses appropriate SQL functions and operations 211 | 212 | {{ if .ddl }} 213 | Here is the DDL for the table structure: 214 | {{ .ddl }} 215 | ``` 216 | {{ end }} 217 | 218 | Use order_by instead of sort_by. 219 | For %_like flags, take a stringList, and iterate over it to create a filter statement of LIKE queries joined by OR. 220 | 221 | IMPORTANT GUIDELINES: 222 | - Never use the flag name "database" as it is already used. 223 | 224 | Before generating the command, make a short bullet list of the flags you want to use and why, their type, and make sure they are valid. 225 | 226 | 227 | {{ if .instructions }} 228 | INSTRUCTIONS: 229 | --- 230 | {{ .instructions }} 231 | --- 232 | {{ end }} 233 | {{ if .instructions_file }} 234 | INSTRUCTIONS FILE: 235 | --- 236 | {{ .instructions_file }} 237 | --- 238 | {{ end }} 239 | 240 | {{- .additional | join "\n" }} 241 | 242 | {{ if .context}}Additional Context: 243 | {{ range .context }} 244 | Path: {{ .Path }} 245 | --- 246 | {{ .Content }} 247 | --- 248 | {{- end }} 249 | {{ end }} 250 | 251 | 252 | {{ if .instructions }} 253 | INSTRUCTIONS: 254 | --- 255 | {{ .instructions }} 256 | --- 257 | {{ end }} 258 | {{ if .instructions_file }} 259 | INSTRUCTIONS FILE: 260 | --- 261 | {{ .instructions_file }} 262 | --- 263 | {{ end }} 264 | -------------------------------------------------------------------------------- /pkg/cmds/factory.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "github.com/go-go-golems/clay/pkg/sql" 5 | "github.com/go-go-golems/parka/pkg/handlers" 6 | ) 7 | 8 | func NewRepositoryFactory() handlers.RepositoryFactory { 9 | loader := &SqlCommandLoader{ 10 | DBConnectionFactory: sql.OpenDatabaseFromDefaultSqlConnectionLayer, 11 | } 12 | 13 | return handlers.NewRepositoryFactoryFromReaderLoaders(loader) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/cmds/loaders.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/fs" 7 | 8 | "github.com/go-go-golems/clay/pkg/sql" 9 | "github.com/go-go-golems/glazed/pkg/cmds" 10 | "github.com/go-go-golems/glazed/pkg/cmds/alias" 11 | "github.com/go-go-golems/glazed/pkg/cmds/layout" 12 | "github.com/go-go-golems/glazed/pkg/cmds/loaders" 13 | "github.com/pkg/errors" 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | type SqlCommandLoader struct { 18 | DBConnectionFactory sql.DBConnectionFactory 19 | } 20 | 21 | var _ loaders.CommandLoader = (*SqlCommandLoader)(nil) 22 | 23 | func (scl *SqlCommandLoader) LoadCommands( 24 | f fs.FS, entryName string, 25 | options []cmds.CommandDescriptionOption, 26 | aliasOptions []alias.Option, 27 | ) ([]cmds.Command, error) { 28 | r, err := f.Open(entryName) 29 | if err != nil { 30 | return nil, err 31 | } 32 | defer func(r fs.File) { 33 | _ = r.Close() 34 | }(r) 35 | 36 | return loaders.LoadCommandOrAliasFromReader( 37 | r, 38 | scl.loadSqlCommandFromReader, 39 | options, 40 | aliasOptions) 41 | } 42 | 43 | func (scl *SqlCommandLoader) IsFileSupported(f fs.FS, fileName string) bool { 44 | return loaders.CheckYamlFileType(f, fileName, "sqleton") 45 | } 46 | 47 | func (scl *SqlCommandLoader) loadSqlCommandFromReader( 48 | s io.Reader, 49 | options []cmds.CommandDescriptionOption, 50 | _ []alias.Option, 51 | ) ([]cmds.Command, error) { 52 | scd := &SqlCommandDescription{} 53 | err := yaml.NewDecoder(s).Decode(scd) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | if scd.Type == "" { 59 | scd.Type = "sqleton" 60 | } else if scd.Type != "sqleton" { 61 | return nil, fmt.Errorf("invalid type: %s", scd.Type) 62 | } 63 | 64 | options_ := []cmds.CommandDescriptionOption{ 65 | cmds.WithShort(scd.Short), 66 | cmds.WithLong(scd.Long), 67 | cmds.WithFlags(scd.Flags...), 68 | cmds.WithArguments(scd.Arguments...), 69 | cmds.WithLayersList(scd.Layers...), 70 | cmds.WithType(scd.Type), 71 | cmds.WithTags(scd.Tags...), 72 | cmds.WithMetadata(scd.Metadata), 73 | cmds.WithLayout(&layout.Layout{ 74 | Sections: scd.Layout, 75 | }), 76 | } 77 | options_ = append(options_, options...) 78 | 79 | sq, err := NewSqlCommand( 80 | cmds.NewCommandDescription( 81 | scd.Name, 82 | ), 83 | WithDbConnectionFactory(scl.DBConnectionFactory), 84 | WithQuery(scd.Query), 85 | WithSubQueries(scd.SubQueries), 86 | ) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | for _, option := range options_ { 92 | option(sq.Description()) 93 | } 94 | 95 | if !sq.IsValid() { 96 | return nil, errors.New("Invalid command") 97 | } 98 | 99 | return []cmds.Command{sq}, nil 100 | } 101 | -------------------------------------------------------------------------------- /pkg/cmds/sql.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | clay_sql "github.com/go-go-golems/clay/pkg/sql" 7 | "github.com/go-go-golems/glazed/pkg/cmds" 8 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 9 | "github.com/go-go-golems/glazed/pkg/cmds/layout" 10 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 11 | "github.com/go-go-golems/glazed/pkg/middlewares" 12 | "github.com/go-go-golems/glazed/pkg/settings" 13 | "github.com/go-go-golems/sqleton/pkg/flags" 14 | "github.com/jmoiron/sqlx" 15 | "github.com/pkg/errors" 16 | "gopkg.in/yaml.v3" 17 | "io" 18 | "strings" 19 | ) 20 | 21 | type SqletonCommand interface { 22 | RunQueryIntoGlaze( 23 | ctx context.Context, 24 | db *sqlx.DB, 25 | parameters map[string]interface{}, 26 | gp middlewares.TableProcessor, 27 | ) error 28 | RenderQuery(parameters map[string]interface{}) (string, error) 29 | } 30 | 31 | var _ cmds.GlazeCommand = (*SqlCommand)(nil) 32 | var _ cmds.CommandWithMetadata = (*SqlCommand)(nil) 33 | 34 | type SqlCommandDescription struct { 35 | Name string `yaml:"name"` 36 | Short string `yaml:"short"` 37 | Long string `yaml:"long,omitempty"` 38 | Layout []*layout.Section `yaml:"layout,omitempty"` 39 | Flags []*parameters.ParameterDefinition `yaml:"flags,omitempty"` 40 | Arguments []*parameters.ParameterDefinition `yaml:"arguments,omitempty"` 41 | Layers []layers.ParameterLayer `yaml:"layers,omitempty"` 42 | Type string `yaml:"type,omitempty"` 43 | Tags []string `yaml:"tags,omitempty"` 44 | Metadata map[string]interface{} `yaml:"metadata,omitempty"` 45 | 46 | SubQueries map[string]string `yaml:"subqueries,omitempty"` 47 | Query string `yaml:"query"` 48 | } 49 | 50 | // SqlCommand describes a command line command that runs a query 51 | type SqlCommand struct { 52 | *cmds.CommandDescription `yaml:",inline"` 53 | Query string `yaml:"query"` 54 | SubQueries map[string]string `yaml:"subqueries,omitempty"` 55 | dbConnectionFactory clay_sql.DBConnectionFactory `yaml:"-"` 56 | renderedQuery string 57 | } 58 | 59 | func (s *SqlCommand) Metadata( 60 | ctx context.Context, 61 | parsedLayers *layers.ParsedLayers) (map[string]interface{}, error) { 62 | db, err := s.dbConnectionFactory(parsedLayers) 63 | if err != nil { 64 | return nil, err 65 | } 66 | defer func(db *sqlx.DB) { 67 | _ = db.Close() 68 | }(db) 69 | 70 | err = db.PingContext(ctx) 71 | if err != nil { 72 | return nil, errors.Wrapf(err, "Could not ping database") 73 | } 74 | 75 | query, err := s.RenderQuery(ctx, db, parsedLayers.GetDataMap()) 76 | if err != nil { 77 | return nil, errors.Wrapf(err, "Could not generate query") 78 | } 79 | 80 | return map[string]interface{}{ 81 | "query": query, 82 | }, nil 83 | } 84 | 85 | func (s *SqlCommand) String() string { 86 | return fmt.Sprintf("SqlCommand{Name: %s, Parents: %s}", s.Name, strings.Join(s.Parents, " ")) 87 | } 88 | 89 | func (s *SqlCommand) ToYAML(w io.Writer) error { 90 | enc := yaml.NewEncoder(w) 91 | defer func(enc *yaml.Encoder) { 92 | _ = enc.Close() 93 | }(enc) 94 | 95 | return enc.Encode(s) 96 | } 97 | 98 | type SqlCommandOption func(*SqlCommand) 99 | 100 | func WithDbConnectionFactory(factory clay_sql.DBConnectionFactory) SqlCommandOption { 101 | return func(s *SqlCommand) { 102 | s.dbConnectionFactory = factory 103 | } 104 | } 105 | 106 | func WithQuery(query string) SqlCommandOption { 107 | return func(s *SqlCommand) { 108 | s.Query = query 109 | } 110 | } 111 | 112 | func WithSubQueries(subQueries map[string]string) SqlCommandOption { 113 | return func(s *SqlCommand) { 114 | s.SubQueries = subQueries 115 | } 116 | } 117 | 118 | func NewSqlCommand( 119 | description *cmds.CommandDescription, 120 | options ...SqlCommandOption, 121 | ) (*SqlCommand, error) { 122 | glazedParameterLayer, err := settings.NewGlazedParameterLayers() 123 | if err != nil { 124 | return nil, errors.Wrap(err, "could not create Glazed parameter layer") 125 | } 126 | sqlConnectionParameterLayer, err := clay_sql.NewSqlConnectionParameterLayer() 127 | if err != nil { 128 | return nil, errors.Wrap(err, "could not create SQL connection parameter layer") 129 | } 130 | dbtParameterLayer, err := clay_sql.NewDbtParameterLayer() 131 | if err != nil { 132 | return nil, errors.Wrap(err, "could not create dbt parameter layer") 133 | } 134 | sqlHelpersParameterLayer, err := flags.NewSqlHelpersParameterLayer() 135 | if err != nil { 136 | return nil, errors.Wrap(err, "could not create SQL helpers parameter layer") 137 | } 138 | description.Layers.AppendLayers( 139 | sqlHelpersParameterLayer, 140 | sqlConnectionParameterLayer, 141 | dbtParameterLayer, 142 | glazedParameterLayer, 143 | ) 144 | 145 | ret := &SqlCommand{ 146 | CommandDescription: description, 147 | SubQueries: make(map[string]string), 148 | } 149 | 150 | for _, option := range options { 151 | option(ret) 152 | } 153 | 154 | return ret, nil 155 | } 156 | 157 | func (s *SqlCommand) RunIntoGlazeProcessor( 158 | ctx context.Context, 159 | parsedLayers *layers.ParsedLayers, 160 | gp middlewares.Processor, 161 | ) error { 162 | if s.dbConnectionFactory == nil { 163 | return errors.New("dbConnectionFactory is not set") 164 | } 165 | 166 | // at this point, the factory can probably be passed the sql-connection parsed layer 167 | db, err := s.dbConnectionFactory(parsedLayers) 168 | if err != nil { 169 | return err 170 | } 171 | defer func(db *sqlx.DB) { 172 | _ = db.Close() 173 | }(db) 174 | 175 | err = db.PingContext(ctx) 176 | if err != nil { 177 | return errors.Wrapf(err, "Could not ping database") 178 | } 179 | 180 | dataMap := parsedLayers.GetDataMap() 181 | 182 | printQuery := false 183 | if printQuery_, ok := parsedLayers.GetParameter("sql-helpers", "print-query"); ok { 184 | printQuery = printQuery_.Value.(bool) 185 | } 186 | 187 | if printQuery { 188 | return s.PrintQuery(ctx, db, dataMap) 189 | } 190 | 191 | return s.RunIntoGlazeProcessorWithDB(ctx, db, dataMap, gp) 192 | } 193 | 194 | func (s *SqlCommand) PrintQuery( 195 | ctx context.Context, 196 | db *sqlx.DB, 197 | dataMap map[string]interface{}, 198 | ) error { 199 | var err error 200 | s.renderedQuery, err = s.RenderQuery(ctx, db, dataMap) 201 | if err != nil { 202 | return errors.Wrapf(err, "Could not generate query") 203 | } 204 | 205 | fmt.Println(s.renderedQuery) 206 | return &cmds.ExitWithoutGlazeError{} 207 | } 208 | 209 | func (s *SqlCommand) RunIntoGlazeProcessorWithDB( 210 | ctx context.Context, 211 | db *sqlx.DB, 212 | dataMap map[string]interface{}, 213 | gp middlewares.Processor, 214 | ) error { 215 | var err error 216 | s.renderedQuery, err = s.RenderQuery(ctx, db, dataMap) 217 | if err != nil { 218 | return errors.Wrapf(err, "Could not generate query") 219 | } 220 | 221 | err = s.RunQueryIntoGlaze(ctx, db, gp) 222 | if err != nil { 223 | return errors.Wrapf(err, "Could not run query") 224 | } 225 | 226 | return nil 227 | } 228 | 229 | func (s *SqlCommand) RenderQueryFull( 230 | ctx context.Context, 231 | parsedLayers *layers.ParsedLayers, 232 | ) (string, error) { 233 | if s.dbConnectionFactory == nil { 234 | return "", errors.Errorf("dbConnectionFactory is not set") 235 | } 236 | 237 | // at this point, the factory can probably be passed the sql-connection parsed layer 238 | db, err := s.dbConnectionFactory(parsedLayers) 239 | if err != nil { 240 | return "", err 241 | } 242 | defer func(db *sqlx.DB) { 243 | _ = db.Close() 244 | }(db) 245 | 246 | err = db.PingContext(ctx) 247 | if err != nil { 248 | return "", errors.Wrapf(err, "Could not ping database") 249 | } 250 | 251 | query, err := s.RenderQuery(ctx, db, parsedLayers.GetDataMap()) 252 | if err != nil { 253 | return "", errors.Wrapf(err, "Could not generate query") 254 | } 255 | return query, nil 256 | } 257 | 258 | func (s *SqlCommand) Description() *cmds.CommandDescription { 259 | return s.CommandDescription 260 | } 261 | 262 | func (s *SqlCommand) IsValid() bool { 263 | return s.Name != "" && s.Query != "" && s.Short != "" 264 | } 265 | 266 | func (s *SqlCommand) RenderQuery( 267 | ctx context.Context, 268 | db *sqlx.DB, 269 | ps map[string]interface{}, 270 | ) (string, error) { 271 | ret, err := clay_sql.RenderQuery(ctx, db, s.Query, s.SubQueries, ps) 272 | if err != nil { 273 | return "", errors.Wrap(err, "Could not render query") 274 | } 275 | 276 | return ret, nil 277 | } 278 | 279 | // RunQueryIntoGlaze runs the query and processes the results into Glaze. 280 | // This requires RenderQuery to be invoked first in order to have a s.renderedQuery. 281 | // NOTE(manuel, 2024-04-11) This really could benefit of a further cleanup, what with codegen now 282 | func (s *SqlCommand) RunQueryIntoGlaze( 283 | ctx context.Context, 284 | db *sqlx.DB, 285 | gp middlewares.Processor) error { 286 | return clay_sql.RunQueryIntoGlaze(ctx, db, s.renderedQuery, []interface{}{}, gp) 287 | } 288 | -------------------------------------------------------------------------------- /pkg/cmds/sql_test.go: -------------------------------------------------------------------------------- 1 | package cmds 2 | 3 | import ( 4 | "context" 5 | "github.com/go-go-golems/clay/pkg/sql" 6 | "github.com/go-go-golems/glazed/pkg/cmds" 7 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 8 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 9 | assert2 "github.com/go-go-golems/glazed/pkg/helpers/assert" 10 | "github.com/go-go-golems/glazed/pkg/middlewares" 11 | "github.com/go-go-golems/glazed/pkg/middlewares/table" 12 | "github.com/go-go-golems/glazed/pkg/types" 13 | "github.com/jmoiron/sqlx" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | "testing" 17 | 18 | // sqlite 19 | _ "github.com/mattn/go-sqlite3" 20 | ) 21 | 22 | // Here we do a bunch of unit tests in a pretty end to end style by using an in memory SQLite database. 23 | 24 | func createDB(_ *layers.ParsedLayers) (*sqlx.DB, error) { 25 | db, err := sqlx.Connect("sqlite3", ":memory:") 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | // create test table 31 | _, err = db.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)") 32 | if err != nil { 33 | return nil, err 34 | } 35 | // insert test data 36 | testData := []struct { 37 | id int 38 | name string 39 | }{ 40 | {1, "test1"}, 41 | {2, "test2"}, 42 | {3, "test3"}, 43 | } 44 | for _, d := range testData { 45 | _, err = db.Exec("INSERT INTO test (id, name) VALUES (?, ?)", d.id, d.name) 46 | if err != nil { 47 | return nil, err 48 | } 49 | } 50 | 51 | // second table for a join 52 | _, err = db.Exec("CREATE TABLE test2 (id INTEGER PRIMARY KEY, test_id INTEGER, name TEXT)") 53 | if err != nil { 54 | return nil, err 55 | } 56 | // insert test data 57 | testData2 := []struct { 58 | id int 59 | test_id int 60 | name string 61 | }{ 62 | {1, 1, "test1_1"}, 63 | {2, 1, "test1_2"}, 64 | {3, 2, "test2_3"}, 65 | } 66 | for _, d := range testData2 { 67 | _, err = db.Exec("INSERT INTO test2 (id, test_id, name) VALUES (?, ?, ?)", d.id, d.test_id, d.name) 68 | if err != nil { 69 | return nil, err 70 | } 71 | } 72 | 73 | return db, nil 74 | } 75 | 76 | func TestSimpleRender(t *testing.T) { 77 | s, err := NewSqlCommand( 78 | cmds.NewCommandDescription("test"), 79 | WithDbConnectionFactory(createDB), 80 | WithQuery("SELECT * FROM test"), 81 | ) 82 | require.NoError(t, err) 83 | 84 | query, err := s.RenderQuery(context.Background(), nil, nil) 85 | require.NoError(t, err) 86 | assert.Equal(t, "SELECT * FROM test", query) 87 | } 88 | 89 | func TestSimpleTemplateRender(t *testing.T) { 90 | s, err := NewSqlCommand( 91 | cmds.NewCommandDescription("test"), 92 | WithDbConnectionFactory(createDB), 93 | WithQuery("SELECT * FROM {{.table}}"), 94 | ) 95 | require.NoError(t, err) 96 | 97 | query, err := s.RenderQuery(context.Background(), nil, map[string]interface{}{ 98 | "table": "test", 99 | }) 100 | require.NoError(t, err) 101 | assert.Equal(t, "SELECT * FROM test", query) 102 | } 103 | 104 | func TestSimpleRun(t *testing.T) { 105 | s, err := NewSqlCommand( 106 | cmds.NewCommandDescription("test"), 107 | WithDbConnectionFactory(createDB), 108 | WithQuery("SELECT * FROM test"), 109 | ) 110 | require.NoError(t, err) 111 | 112 | gp := middlewares.NewTableProcessor() 113 | require.NoError(t, err) 114 | gp.AddTableMiddleware(&table.NullTableMiddleware{}) 115 | ctx := context.Background() 116 | err = s.RunIntoGlazeProcessor(ctx, layers.NewParsedLayers(), gp) 117 | require.NoError(t, err) 118 | 119 | err = gp.Close(ctx) 120 | require.NoError(t, err) 121 | table_ := gp.GetTable() 122 | require.NoError(t, err) 123 | 124 | expected := []types.Row{ 125 | types.NewRow( 126 | types.MRP("id", int64(1)), 127 | types.MRP("name", "test1"), 128 | ), 129 | types.NewRow( 130 | types.MRP("id", int64(2)), 131 | types.MRP("name", "test2"), 132 | ), 133 | types.NewRow( 134 | types.MRP("id", int64(3)), 135 | types.MRP("name", "test3"), 136 | ), 137 | } 138 | 139 | assert2.EqualRows(t, expected, table_.Rows) 140 | } 141 | 142 | func makeSimpleDefaultLayer(options ...layers.ParsedLayerOption) (*layers.ParsedLayers, error) { 143 | defaultLayer, err := layers.NewParameterLayer("default", "Default", 144 | layers.WithParameterDefinitions( 145 | parameters.NewParameterDefinition( 146 | "name", 147 | parameters.ParameterTypeString, 148 | ), 149 | parameters.NewParameterDefinition( 150 | "test", 151 | parameters.ParameterTypeString, 152 | )), 153 | ) 154 | if err != nil { 155 | return nil, err 156 | } 157 | parsedDefaultLayer, err := layers.NewParsedLayer(defaultLayer, options...) 158 | if err != nil { 159 | return nil, err 160 | } 161 | parsedLayers := layers.NewParsedLayers(layers.WithParsedLayer(layers.DefaultSlug, parsedDefaultLayer)) 162 | return parsedLayers, nil 163 | 164 | } 165 | func TestSimpleSubQuery(t *testing.T) { 166 | s, err := NewSqlCommand( 167 | cmds.NewCommandDescription("test"), 168 | WithDbConnectionFactory(createDB), 169 | WithQuery(` 170 | SELECT * FROM test 171 | WHERE id IN ( 172 | {{ sqlColumn "SELECT test_id FROM test2 WHERE name = {{.name | sqlString }}" | sqlIntIn }} 173 | ) 174 | `, 175 | ), 176 | ) 177 | require.NoError(t, err) 178 | db, err := createDB(nil) 179 | require.NoError(t, err) 180 | defer func(db *sqlx.DB) { 181 | _ = db.Close() 182 | }(db) 183 | 184 | ps := map[string]interface{}{ 185 | "name": "test2_3", 186 | } 187 | 188 | ctx := context.Background() 189 | s_, err := s.RenderQuery(ctx, db, ps) 190 | require.NoError(t, err) 191 | assert.Equal(t, sql.CleanQuery(` 192 | SELECT * FROM test 193 | WHERE id IN ( 194 | 2 195 | ) 196 | `), s_) 197 | 198 | parsedLayers, err := makeSimpleDefaultLayer( 199 | layers.WithParsedParameterValue("name", "test2_3"), 200 | ) 201 | require.NoError(t, err) 202 | 203 | gp := middlewares.NewTableProcessor() 204 | gp.AddTableMiddleware(&table.NullTableMiddleware{}) 205 | err = s.RunIntoGlazeProcessor(ctx, parsedLayers, gp) 206 | require.NoError(t, err) 207 | 208 | err = gp.Close(ctx) 209 | require.NoError(t, err) 210 | table_ := gp.GetTable() 211 | require.NoError(t, err) 212 | 213 | assert2.EqualRows(t, []types.Row{ 214 | types.NewRow(types.MRP("id", int64(2)), types.MRP("name", "test2")), 215 | }, table_.Rows) 216 | } 217 | 218 | func TestSimpleSubQuerySingle(t *testing.T) { 219 | s, err := NewSqlCommand( 220 | cmds.NewCommandDescription("test"), 221 | WithDbConnectionFactory(createDB), 222 | WithQuery(` 223 | SELECT * FROM test 224 | WHERE id = {{ sqlSingle "SELECT test_id FROM test2 WHERE name = {{.name | sqlString }} LIMIT 1" }} 225 | `, 226 | ), 227 | ) 228 | require.NoError(t, err) 229 | db, err := createDB(nil) 230 | require.NoError(t, err) 231 | defer func(db *sqlx.DB) { 232 | _ = db.Close() 233 | }(db) 234 | 235 | parsedLayers, err := makeSimpleDefaultLayer( 236 | layers.WithParsedParameterValue("name", "test1_1"), 237 | ) 238 | require.NoError(t, err) 239 | 240 | s_, err := s.RenderQuery(context.Background(), db, parsedLayers.GetDataMap()) 241 | require.NoError(t, err) 242 | assert.Equal(t, sql.CleanQuery(` 243 | SELECT * FROM test 244 | WHERE id = 1 245 | `), s_) 246 | 247 | // fail if we return more than 1 248 | s, err = NewSqlCommand( 249 | cmds.NewCommandDescription("test"), 250 | WithDbConnectionFactory(createDB), 251 | WithQuery(` 252 | SELECT * FROM test 253 | WHERE id = {{ sqlSingle "SELECT test_id FROM test2" | sqlIntIn }} 254 | `, 255 | ), 256 | ) 257 | require.NoError(t, err) 258 | 259 | _, err = s.RenderQuery(context.Background(), db, parsedLayers.GetDataMap()) 260 | assert.Error(t, err) 261 | 262 | // fail if there are more than 2 fields 263 | s, err = NewSqlCommand( 264 | cmds.NewCommandDescription("test"), 265 | WithDbConnectionFactory(createDB), 266 | WithQuery(` 267 | SELECT * FROM test 268 | WHERE id = {{ sqlSingle "SELECT test_id, name FROM test2 WHERE name = {{.name | sqlString }} LIMIT 1" }} 269 | `, 270 | ), 271 | ) 272 | require.NoError(t, err) 273 | 274 | _, err = s.RenderQuery(context.Background(), db, parsedLayers.GetDataMap()) 275 | assert.Error(t, err) 276 | } 277 | 278 | func TestSimpleSubQueryWithArguments(t *testing.T) { 279 | s, err := NewSqlCommand( 280 | cmds.NewCommandDescription("test"), 281 | WithDbConnectionFactory(createDB), 282 | WithQuery(` 283 | SELECT * FROM test 284 | WHERE id IN ( 285 | {{ sqlColumn "SELECT test_id FROM test2 WHERE name = {{.name | sqlString }} AND id = {{.test2_id}}" "test2_id" 2 | sqlIntIn }} 286 | ) 287 | `, 288 | ), 289 | ) 290 | require.NoError(t, err) 291 | db, err := createDB(nil) 292 | require.NoError(t, err) 293 | defer func(db *sqlx.DB) { 294 | _ = db.Close() 295 | }(db) 296 | 297 | parsedLayers, err := makeSimpleDefaultLayer( 298 | layers.WithParsedParameterValue("name", "test1_2"), 299 | ) 300 | require.NoError(t, err) 301 | 302 | ctx := context.Background() 303 | s_, err := s.RenderQuery(ctx, db, parsedLayers.GetDataMap()) 304 | require.NoError(t, err) 305 | assert.Equal(t, sql.CleanQuery(` 306 | SELECT * FROM test 307 | WHERE id IN ( 308 | 1 309 | ) 310 | `), s_) 311 | 312 | gp := middlewares.NewTableProcessor() 313 | gp.AddTableMiddleware(&table.NullTableMiddleware{}) 314 | err = s.RunIntoGlazeProcessor(ctx, parsedLayers, gp) 315 | require.NoError(t, err) 316 | 317 | err = gp.Close(ctx) 318 | require.NoError(t, err) 319 | table_ := gp.GetTable() 320 | require.NoError(t, err) 321 | 322 | assert2.EqualRows(t, []types.Row{ 323 | types.NewRow( 324 | types.MRP("id", int64(1)), 325 | types.MRP("name", "test1"), 326 | ), 327 | }, table_.Rows) 328 | 329 | _, err = NewSqlCommand( 330 | cmds.NewCommandDescription("test"), 331 | WithDbConnectionFactory(createDB), 332 | WithQuery(` 333 | SELECT * FROM test 334 | WHERE id IN ( 335 | {{ sqlColumn "SELECT test_id, id FROM test2 WHERE name = {{.name | sqlString }} AND id = {{.test2_id}}" "test2_id" 2 | sqlIntIn }} 336 | ) 337 | `, 338 | ), 339 | ) 340 | require.NoError(t, err) 341 | } 342 | 343 | func TestSliceSubQueryWithArguments(t *testing.T) { 344 | s, err := NewSqlCommand( 345 | cmds.NewCommandDescription("test"), 346 | WithDbConnectionFactory(createDB), 347 | WithQuery(` 348 | SELECT * FROM test 349 | WHERE id IN ( 350 | {{ range sqlSlice "SELECT id, test_id FROM test2 ORDER BY id" }}{{- index . 1 -}} +{{ end }}0 351 | ) 352 | `, 353 | ), 354 | ) 355 | require.NoError(t, err) 356 | db, err := createDB(nil) 357 | require.NoError(t, err) 358 | defer func(db *sqlx.DB) { 359 | _ = db.Close() 360 | }(db) 361 | 362 | parsedLayers, err := makeSimpleDefaultLayer( 363 | layers.WithParsedParameterValue("test", "test1_2"), 364 | ) 365 | require.NoError(t, err) 366 | 367 | s_, err := s.RenderQuery(context.Background(), db, parsedLayers.GetDataMap()) 368 | require.NoError(t, err) 369 | assert.Equal(t, sql.CleanQuery(` 370 | SELECT * FROM test 371 | WHERE id IN ( 372 | 1+1+2+0 373 | ) 374 | `), s_) 375 | 376 | } 377 | func TestMapSubQueryWithArguments(t *testing.T) { 378 | s, err := NewSqlCommand( 379 | cmds.NewCommandDescription("test"), 380 | WithDbConnectionFactory(createDB), 381 | WithQuery(` 382 | SELECT * FROM test 383 | WHERE id IN ( 384 | {{ range sqlMap "SELECT id, test_id FROM test2 ORDER BY id" }}{{- index . "id" -}} +{{ end }}0 385 | ) 386 | `, 387 | ), 388 | ) 389 | require.NoError(t, err) 390 | db, err := createDB(nil) 391 | require.NoError(t, err) 392 | defer func(db *sqlx.DB) { 393 | _ = db.Close() 394 | }(db) 395 | 396 | parsedLayers, err := makeSimpleDefaultLayer( 397 | layers.WithParsedParameterValue("name", "test1_2"), 398 | ) 399 | require.NoError(t, err) 400 | 401 | s_, err := s.RenderQuery(context.Background(), db, parsedLayers.GetDataMap()) 402 | require.NoError(t, err) 403 | assert.Equal(t, sql.CleanQuery(` 404 | SELECT * FROM test 405 | WHERE id IN ( 406 | 1+2+3+0 407 | ) 408 | `), s_) 409 | 410 | } 411 | 412 | func TestMapSubQuery(t *testing.T) { 413 | s, err := NewSqlCommand( 414 | cmds.NewCommandDescription("test"), 415 | WithDbConnectionFactory(createDB), 416 | WithQuery(` 417 | SELECT * FROM test 418 | WHERE id IN ( 419 | {{ sqlColumn (subQuery "test2_id") "test2_id" 2 | sqlIntIn }} 420 | ) 421 | `, 422 | ), 423 | WithSubQueries(map[string]string{ 424 | "test2_id": "SELECT test_id FROM test2 WHERE name = {{.name | sqlString }} AND id = {{.test2_id}}", 425 | }), 426 | ) 427 | require.NoError(t, err) 428 | db, err := createDB(nil) 429 | require.NoError(t, err) 430 | defer func(db *sqlx.DB) { 431 | _ = db.Close() 432 | }(db) 433 | 434 | parsedLayers, err := makeSimpleDefaultLayer( 435 | layers.WithParsedParameterValue("name", "test1_2"), 436 | ) 437 | require.NoError(t, err) 438 | 439 | ctx := context.Background() 440 | s_, err := s.RenderQuery(ctx, db, parsedLayers.GetDataMap()) 441 | require.NoError(t, err) 442 | assert.Equal(t, sql.CleanQuery(` 443 | SELECT * FROM test 444 | WHERE id IN ( 445 | 1 446 | ) 447 | `), s_) 448 | 449 | gp := middlewares.NewTableProcessor() 450 | gp.AddTableMiddleware(&table.NullTableMiddleware{}) 451 | err = s.RunIntoGlazeProcessor(ctx, parsedLayers, gp) 452 | require.NoError(t, err) 453 | 454 | err = gp.Close(ctx) 455 | require.NoError(t, err) 456 | table_ := gp.GetTable() 457 | require.NoError(t, err) 458 | rows := table_.Rows 459 | assert.Equal(t, 1, len(rows)) 460 | row := rows[0] 461 | id, ok := row.Get("id") 462 | assert.True(t, ok) 463 | assert.Equal(t, int64(1), id) 464 | name, ok := row.Get("name") 465 | assert.True(t, ok) 466 | assert.Equal(t, "test1", name) 467 | 468 | } 469 | -------------------------------------------------------------------------------- /pkg/codegen/codegen.go: -------------------------------------------------------------------------------- 1 | package codegen 2 | 3 | import ( 4 | "github.com/dave/jennifer/jen" 5 | cmds2 "github.com/go-go-golems/glazed/pkg/cmds" 6 | "github.com/go-go-golems/glazed/pkg/cmds/parameters" 7 | "github.com/go-go-golems/glazed/pkg/codegen" 8 | "github.com/go-go-golems/sqleton/pkg/cmds" 9 | "github.com/iancoleman/strcase" 10 | ) 11 | 12 | type SqlCommandCodeGenerator struct { 13 | PackageName string 14 | } 15 | 16 | const SqletonCmdsPath = "github.com/go-go-golems/sqleton/pkg/cmds" 17 | 18 | func (s *SqlCommandCodeGenerator) defineConstants(f *jen.File, cmdName string, cmd *cmds.SqlCommand) { 19 | // Define the constant for the main query. 20 | queryConstName := strcase.ToLowerCamel(cmdName) + "CommandQuery" 21 | f.Const().Id(queryConstName).Op("=").Lit(cmd.Query) 22 | 23 | if len(cmd.SubQueries) > 0 { 24 | for name, subQuery := range cmd.SubQueries { 25 | subQueryConstName := strcase.ToLowerCamel(cmdName) + "CommandSubQuery" + name 26 | f.Const().Id(subQueryConstName).Op("=").Lit(subQuery) 27 | } 28 | } 29 | } 30 | 31 | func (s *SqlCommandCodeGenerator) defineStruct(f *jen.File, cmdName string) { 32 | structName := strcase.ToCamel(cmdName) + "Command" 33 | f.Type().Id(structName).Struct( 34 | jen.Op("*").Qual(codegen.GlazedCommandsPath, "CommandDescription"), 35 | jen.Id("Query").String().Tag(map[string]string{"yaml": "query"}), 36 | jen.Id("SubQueries").Map(jen.String()).String().Tag(map[string]string{"yaml": "subqueries,omitempty"}), 37 | ) 38 | } 39 | 40 | func (s *SqlCommandCodeGenerator) defineParametersStruct( 41 | f *jen.File, 42 | cmdName string, 43 | cmd *cmds2.CommandDescription, 44 | ) { 45 | structName := strcase.ToCamel(cmdName) + "CommandParameters" 46 | f.Type().Id(structName).StructFunc(func(g *jen.Group) { 47 | cmd.GetDefaultFlags().ForEach(func(flag *parameters.ParameterDefinition) { 48 | s := g.Id(strcase.ToCamel(flag.Name)) 49 | s = codegen.FlagTypeToGoType(s, flag.Type) 50 | s.Tag(map[string]string{"glazed.parameter": strcase.ToSnake(flag.Name)}) 51 | }) 52 | cmd.GetDefaultArguments().ForEach(func(arg *parameters.ParameterDefinition) { 53 | s := g.Id(strcase.ToCamel(arg.Name)) 54 | s = codegen.FlagTypeToGoType(s, arg.Type) 55 | s.Tag(map[string]string{"glazed.parameter": strcase.ToSnake(arg.Name)}) 56 | }) 57 | }) 58 | } 59 | 60 | func (s *SqlCommandCodeGenerator) renderQuery() []jen.Code { 61 | return []jen.Code{ 62 | jen.Id("ps").Op(":=").Qual(codegen.MapsHelpersPath, "StructToMap").Call(jen.Id("params"), jen.Lit(false)), 63 | jen.List(jen.Id("renderedQuery"), jen.Err()).Op(":=").Qual(codegen.ClaySqlPath, "RenderQuery").Call( 64 | jen.Id("ctx"), jen.Id("db"), jen.Id("p").Dot("Query"), jen.Id("p").Dot("SubQueries"), jen.Id("ps"), 65 | ), 66 | jen.If(jen.Err().Op("!=").Nil()).Block(jen.Return(jen.Err())), 67 | jen.Line(), 68 | } 69 | } 70 | 71 | func (s *SqlCommandCodeGenerator) defineRunIntoGlazedMethod(f *jen.File, cmdName string) { 72 | methodName := "RunIntoGlazed" 73 | receiver := strcase.ToCamel(cmdName) + "Command" 74 | parametersStruct := strcase.ToCamel(cmdName) + "CommandParameters" 75 | 76 | f.Func(). 77 | Params(jen.Id("p").Op("*").Id(receiver)).Id(methodName). 78 | Params( 79 | jen.Id("ctx").Qual("context", "Context"), 80 | jen.Id("db").Op("*").Qual("github.com/jmoiron/sqlx", "DB"), 81 | jen.Id("params").Op("*").Id(parametersStruct), 82 | jen.Id("gp").Qual(codegen.GlazedMiddlewaresPath, "Processor"), 83 | ).Error(). 84 | BlockFunc(func(g *jen.Group) { 85 | for _, c := range s.renderQuery() { 86 | g.Add(c) 87 | } 88 | g.Err().Op("=").Qual(codegen.ClaySqlPath, "RunQueryIntoGlaze").Call( 89 | jen.Id("ctx"), jen.Id("db"), jen.Id("renderedQuery"), jen.Index().Interface().Values(), jen.Id("gp"), 90 | ) 91 | g.If(jen.Err().Op("!=").Nil()).Block(jen.Return(jen.Err())) 92 | g.Return(jen.Nil()) 93 | }) 94 | } 95 | 96 | func (s *SqlCommandCodeGenerator) defineNewFunction(f *jen.File, cmdName string, cmd *cmds.SqlCommand) error { 97 | funcName := "New" + strcase.ToCamel(cmdName) + "Command" 98 | commandStruct := strcase.ToCamel(cmdName) + "Command" 99 | queryConstName := strcase.ToLowerCamel(cmdName) + "CommandQuery" 100 | 101 | description := cmd.Description() 102 | 103 | var err_ error 104 | f.Func().Id(funcName).Params(). 105 | Params(jen.Op("*").Id(commandStruct), jen.Error()). 106 | Block( 107 | // TODO(manuel, 2023-12-07) Can be refactored since this is duplicated in geppetto/codegen.go 108 | jen.Var().Id("flagDefs").Op("="). 109 | Index().Op("*"). 110 | Qual(codegen.GlazedParametersPath, "ParameterDefinition"). 111 | ValuesFunc(func(g *jen.Group) { 112 | err_ = cmd.GetDefaultFlags().ForEachE(func(flag *parameters.ParameterDefinition) error { 113 | dict, err := codegen.ParameterDefinitionToDict(flag) 114 | if err != nil { 115 | return err 116 | } 117 | g.Values(dict) 118 | return nil 119 | }) 120 | }), 121 | jen.Line(), 122 | jen.Var().Id("argDefs").Op("="). 123 | Index().Op("*"). 124 | Qual(codegen.GlazedParametersPath, "ParameterDefinition"). 125 | ValuesFunc(func(g *jen.Group) { 126 | err_ = cmd.GetDefaultArguments().ForEachE(func(arg *parameters.ParameterDefinition) error { 127 | dict, err := codegen.ParameterDefinitionToDict(arg) 128 | if err != nil { 129 | return err 130 | } 131 | g.Values(dict) 132 | return nil 133 | }) 134 | }), 135 | jen.Line(), 136 | jen.Id("cmdDescription").Op(":=").Qual(codegen.GlazedCommandsPath, "NewCommandDescription"). 137 | Call( 138 | jen.Lit(description.Name), 139 | jen.Line().Qual(codegen.GlazedCommandsPath, "WithShort").Call(jen.Lit(description.Short)), 140 | jen.Line().Qual(codegen.GlazedCommandsPath, "WithLong").Call(jen.Lit(description.Long)), 141 | jen.Line().Qual(codegen.GlazedCommandsPath, "WithFlags").Call(jen.Id("flagDefs").Op("...")), 142 | jen.Line().Qual(codegen.GlazedCommandsPath, "WithArguments").Call(jen.Id("argDefs").Op("...")), 143 | ), 144 | jen.Line(), 145 | 146 | jen.Return(jen.Op("&").Id(commandStruct).Values(jen.Dict{ 147 | jen.Id("CommandDescription"): jen.Id("cmdDescription"), 148 | jen.Id("Query"): jen.Id(queryConstName), 149 | jen.Id("SubQueries"): jen.Map(jen.String()).String().Values(jen.DictFunc(func(d jen.Dict) { 150 | if len(cmd.SubQueries) > 0 { 151 | for name := range cmd.SubQueries { 152 | subQueryConstName := strcase.ToLowerCamel(cmdName) + "CommandSubQuery" + name 153 | d[jen.Lit(name)] = jen.Id(subQueryConstName) 154 | } 155 | } 156 | })), 157 | }), jen.Nil()), 158 | ) 159 | 160 | return err_ 161 | } 162 | 163 | func (s *SqlCommandCodeGenerator) GenerateCommandCode(cmd *cmds.SqlCommand) (*jen.File, error) { 164 | f := jen.NewFile(s.PackageName) 165 | cmdName := strcase.ToLowerCamel(cmd.Name) 166 | 167 | // Define constants, struct, and methods using helper functions. 168 | s.defineConstants(f, cmdName, cmd) 169 | s.defineStruct(f, cmdName) 170 | f.Line() 171 | s.defineParametersStruct(f, cmdName, cmd.Description()) 172 | s.defineRunIntoGlazedMethod(f, cmdName) 173 | f.Line() 174 | err := s.defineNewFunction(f, cmdName, cmd) 175 | if err != nil { 176 | return nil, err 177 | } 178 | 179 | return f, nil 180 | } 181 | -------------------------------------------------------------------------------- /pkg/flags/helpers.yaml: -------------------------------------------------------------------------------- 1 | slug: sql-helpers 2 | name: SQL helpers 3 | Description: | 4 | Helpers flags to print queries and explain 5 | flags: 6 | - name: explain 7 | type: bool 8 | help: Explain the query 9 | default: false 10 | # - name: explain-format 11 | # type: string 12 | # help: Explain format (json, tree, dot) 13 | # default: tree 14 | # - name: explain-type 15 | # type: string 16 | # help: Explain type (extended, partitions, format, triggers, analyze, costs, buffers, timing, summary) 17 | # default: summary 18 | - name: print-query 19 | type: bool 20 | help: Print the query 21 | default: false -------------------------------------------------------------------------------- /pkg/flags/settings.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | _ "embed" 5 | "github.com/go-go-golems/glazed/pkg/cmds/layers" 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | //go:embed "helpers.yaml" 10 | var helpersFlagsYaml []byte 11 | 12 | const SqlHelpersSlug = "sql-helpers" 13 | 14 | type SqlHelpersSettings struct { 15 | Explain bool `glazed.parameter:"explain"` 16 | PrintQuery bool `glazed.parameter:"print-query"` 17 | } 18 | 19 | func NewSqlHelpersParameterLayer( 20 | options ...layers.ParameterLayerOptions, 21 | ) (*layers.ParameterLayerImpl, error) { 22 | ret, err := layers.NewParameterLayerFromYAML(helpersFlagsYaml, options...) 23 | if err != nil { 24 | return nil, errors.Wrap(err, "Failed to initialize helpers parameter layer") 25 | } 26 | return ret, nil 27 | } 28 | --------------------------------------------------------------------------------