├── .gitignore ├── screenshot.png ├── package.json ├── scripts ├── fail_on_diff.sh └── .github │ └── workflows │ └── pull_requests.yml ├── yarn.lock ├── examples ├── basic │ ├── main.go │ ├── go.mod │ └── go.sum └── connecthook │ ├── main.go │ ├── go.mod │ └── go.sum ├── go.mod ├── stdlib_test.go ├── LICENSE.md ├── net_test.go ├── date_test.go ├── regexp_test.go ├── stdlib.go ├── regexp.go ├── net.go ├── .github └── workflows │ └── pull_requests.yml ├── encoding_test.go ├── date.go ├── encoding.go ├── math_test.go ├── aggregate_test.go ├── go.sum ├── math.go ├── string.go ├── string_test.go ├── aggregate.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | cover.out 2 | node_modules -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/go-sqlite3-stdlib/HEAD/screenshot.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "prettier": "^2.6.2" 4 | }, 5 | "scripts": { 6 | "fmt": "gofmt -w -s . && yarn prettier -w '*.md'" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /scripts/fail_on_diff.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | if [[ "$(git diff)" != "" ]]; then 6 | printf "\033[0;31mFAILURE: Unexpected diff (did you run 'yarn format'?)\n\n\033[0m" 7 | git diff --color=never 8 | exit 1 9 | fi 10 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | prettier@^2.6.2: 6 | version "2.6.2" 7 | resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.6.2.tgz#e26d71a18a74c3d0f0597f55f01fb6c06c206032" 8 | integrity sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew== 9 | -------------------------------------------------------------------------------- /examples/basic/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | 7 | _ "github.com/mattn/go-sqlite3" 8 | stdlib "github.com/multiprocessio/go-sqlite3-stdlib" 9 | ) 10 | 11 | func main() { 12 | stdlib.Register("sqlite3_ext") 13 | db, err := sql.Open("sqlite3_ext", ":memory:") 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | var s string 19 | err = db.QueryRow("SELECT repeat('x', 2)").Scan(&s) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | fmt.Println(s) 25 | } 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/multiprocessio/go-sqlite3-stdlib 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 7 | github.com/mattn/go-sqlite3 v1.14.14 8 | github.com/stretchr/testify v1.8.0 9 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa 10 | gonum.org/v1/gonum v0.11.0 11 | ) 12 | 13 | require ( 14 | github.com/davecgh/go-spew v1.1.1 // indirect 15 | github.com/pmezard/go-difflib v1.0.0 // indirect 16 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect 17 | gopkg.in/yaml.v3 v3.0.1 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /stdlib_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | // SOURCE: https://stackoverflow.com/a/50123125/1507139 10 | func TestMain(m *testing.M) { 11 | rc := m.Run() 12 | 13 | // rc 0 means we've passed, 14 | // and CoverMode will be non empty if run with -cover 15 | if rc == 0 && testing.CoverMode() != "" { 16 | c := testing.Coverage() 17 | // For some reason this value is lower than what -cover reports 18 | if c < 0.95 { 19 | log.Println("Tests passed but coverage failed at", c) 20 | rc = -1 21 | } 22 | } 23 | os.Exit(rc) 24 | } 25 | -------------------------------------------------------------------------------- /examples/basic/go.mod: -------------------------------------------------------------------------------- 1 | module basic 2 | 3 | go 1.18 4 | 5 | replace github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220526202821-009799d2879d => ../../ 6 | 7 | require ( 8 | github.com/mattn/go-sqlite3 v1.14.14 9 | github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220526202821-009799d2879d 10 | ) 11 | 12 | require ( 13 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect 14 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect 15 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect 16 | gonum.org/v1/gonum v0.11.0 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /examples/connecthook/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | 7 | sqlite3 "github.com/mattn/go-sqlite3" 8 | stdlib "github.com/multiprocessio/go-sqlite3-stdlib" 9 | ) 10 | 11 | func main() { 12 | sql.Register("sqlite3_ext", 13 | &sqlite3.SQLiteDriver{ 14 | ConnectHook: stdlib.ConnectHook, 15 | }) 16 | db, err := sql.Open("sqlite3_ext", ":memory:") 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | var s string 22 | err = db.QueryRow("SELECT repeat('x', 2)").Scan(&s) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | fmt.Println(s) 28 | } 29 | -------------------------------------------------------------------------------- /examples/connecthook/go.mod: -------------------------------------------------------------------------------- 1 | module connecthook 2 | 3 | go 1.18 4 | 5 | replace github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220526202821-009799d2879d => ../../ 6 | 7 | require ( 8 | github.com/mattn/go-sqlite3 v1.14.14 9 | github.com/multiprocessio/go-sqlite3-stdlib v0.0.0-20220526202821-009799d2879d 10 | ) 11 | 12 | require ( 13 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect 14 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect 15 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect 16 | gonum.org/v1/gonum v0.11.0 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2022 Multiprocess Labs LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /net_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_url(t *testing.T) { 9 | url := "https://x.com:90/some/path.html?p=123&z=%5B1%2C2%5D#section-1" 10 | tests := []struct { 11 | fn string 12 | out string 13 | }{ 14 | {"url_scheme", "https"}, 15 | {"url_host", "x.com:90"}, 16 | {"url_port", "90"}, 17 | {"url_path", "/some/path.html"}, 18 | {"url_fragment", "section-1"}, 19 | } 20 | 21 | for _, test := range tests { 22 | assertQuery(t, fmt.Sprintf("SELECT %s('%s')", test.fn, url), test.out) 23 | assertQuery(t, fmt.Sprintf("SELECT %s('z kljsdfabsdf ://')", test.fn), "") 24 | } 25 | 26 | assertQuery(t, fmt.Sprintf("SELECT url_param('%s', 'z')", url), "[1,2]") 27 | assertQuery(t, fmt.Sprintf("SELECT url_param(' kljz sdf ://', 'z')"), "") 28 | } 29 | -------------------------------------------------------------------------------- /date_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_date(t *testing.T) { 9 | tests := []struct { 10 | in string 11 | fn string 12 | out int 13 | }{ 14 | {"2021-04-05", "date_year", 2021}, 15 | {"May 6, 2020", "date_month", 5}, 16 | {"May 6, 2020", "date_day", 6}, 17 | {"May 6, 2020", "date_yearday", 127}, 18 | {"May 6, 2020 4:50 PM", "date_hour", 16}, 19 | {"May 6, 2020 4:50", "date_minute", 50}, 20 | {"May 6, 2020 4:50:20", "date_second", 20}, 21 | {"May 6, 2020 4:50:20", "date_unix", 1588740620}, 22 | } 23 | 24 | for _, test := range tests { 25 | assertQuery(t, fmt.Sprintf("SELECT %s('%s')", test.fn, test.in), fmt.Sprintf("%d", test.out)) 26 | assertQuery(t, fmt.Sprintf("SELECT %s(' total garbage')", test.fn), "-1") 27 | } 28 | 29 | assertQuery(t, "SELECT date_rfc3339('May 6, 2020 4:50:20')", "2020-05-06T04:50:20Z") 30 | assertQuery(t, "SELECT date_rfc3339(' total garbage ')", "") 31 | } 32 | -------------------------------------------------------------------------------- /regexp_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import "testing" 4 | 5 | func Test_regexp(t *testing.T) { 6 | assertQuery(t, "SELECT 'ZaB' REGEXP '[a-zA-Z]+'", "1") 7 | assertQuery(t, "SELECT 'ZaB0' REGEXP '[a-zA-Z]+$'", "0") 8 | 9 | // bad regexp 10 | assertQuery(t, "SELECT 'ZaB0' REGEXP ']['", "0") 11 | } 12 | 13 | func Test_regexp_split_part(t *testing.T) { 14 | assertQuery(t, "SELECT regexp_split_part('ab12', '[a-zA-Z]1', 1)", "2") 15 | assertQuery(t, "SELECT regexp_split_part('ab12', '[a-zA-Z]1', -1)", "2") 16 | assertQuery(t, "SELECT regexp_split_part('ab12', '[a-zA-Z]1', 100)", "") 17 | 18 | // bad regexp 19 | assertQuery(t, "SELECT regexp_split_part('ab12', '][', 1)", "") 20 | } 21 | 22 | func Test_regexp_count(t *testing.T) { 23 | assertQuery(t, "SELECT regexp_count('ab12', '[a-zA-Z]1')", "1") 24 | assertQuery(t, "SELECT regexp_count('ac22', '[a-zA-Z]1')", "0") 25 | 26 | // bad regexp 27 | assertQuery(t, "SELECT regexp_count('ab12', '][')", "0") 28 | } 29 | -------------------------------------------------------------------------------- /stdlib.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/mattn/go-sqlite3" 7 | ) 8 | 9 | var extensions = []map[string]any{ 10 | mathFunctions, 11 | stringFunctions, 12 | regexpFunctions, 13 | dateFunctions, 14 | netFunctions, 15 | encodingFunctions, 16 | } 17 | 18 | func ConnectHook(conn *sqlite3.SQLiteConn) error { 19 | var err error 20 | for _, functions := range extensions { 21 | for name, impl := range functions { 22 | err = conn.RegisterFunc(name, impl, true) 23 | // Yes it's weird to not break on return/error 24 | // but this way we get 100% test coverage on 25 | // this function. Errors shouldn't happen 26 | // outside of development anyway. 27 | } 28 | } 29 | 30 | for name, impl := range aggregateFunctions { 31 | err = conn.RegisterAggregator(name, impl, true) 32 | } 33 | 34 | return err 35 | } 36 | 37 | func Register(driverName string) { 38 | sql.Register(driverName, 39 | &sqlite3.SQLiteDriver{ 40 | ConnectHook: ConnectHook, 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /regexp.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | func _regexp(re, s string) bool { 8 | c, err := regexp.Compile(re) 9 | if err != nil { 10 | return false 11 | } 12 | 13 | return c.MatchString(s) 14 | } 15 | 16 | func regexpSplitPart(_s, _split, _part any) string { 17 | s := stringy(_s) 18 | split := stringy(_split) 19 | part := int(floaty(_part)) 20 | 21 | c, err := regexp.Compile(split) 22 | if err != nil { 23 | return "" 24 | } 25 | 26 | pieces := part + 1 27 | if pieces == 0 { 28 | pieces = -1 29 | } 30 | 31 | parts := c.Split(s, pieces) 32 | 33 | if len(parts) == 0 || part >= len(parts) { 34 | return "" 35 | } 36 | 37 | if part < 0 { 38 | part = (len(parts) + part) % len(parts) 39 | } 40 | 41 | return parts[part] 42 | } 43 | 44 | func regexpCount(_s, _re any) int64 { 45 | s := stringy(_s) 46 | re := stringy(_re) 47 | 48 | c, err := regexp.Compile(re) 49 | if err != nil { 50 | return 0 51 | } 52 | 53 | return int64(len(c.FindAllStringIndex(s, -1))) 54 | } 55 | 56 | var regexpFunctions = map[string]any{ 57 | "regexp": _regexp, 58 | "regexp_split_part": regexpSplitPart, 59 | "regexp_count": regexpCount, 60 | } 61 | -------------------------------------------------------------------------------- /net.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import "net/url" 4 | 5 | func urlHost(u string) string { 6 | p, err := url.Parse(u) 7 | if err != nil { 8 | return "" 9 | } 10 | 11 | return p.Host 12 | } 13 | 14 | func urlPort(u string) string { 15 | p, err := url.Parse(u) 16 | if err != nil { 17 | return "" 18 | } 19 | 20 | return p.Port() 21 | } 22 | 23 | func urlScheme(u string) string { 24 | p, err := url.Parse(u) 25 | if err != nil { 26 | return "" 27 | } 28 | 29 | return p.Scheme 30 | } 31 | 32 | func urlPath(u string) string { 33 | p, err := url.Parse(u) 34 | if err != nil { 35 | return "" 36 | } 37 | 38 | return p.Path 39 | } 40 | 41 | func urlParam(u, k string) string { 42 | p, err := url.Parse(u) 43 | if err != nil { 44 | return "" 45 | } 46 | 47 | return p.Query().Get(k) 48 | } 49 | 50 | func urlFragment(u string) string { 51 | p, err := url.Parse(u) 52 | if err != nil { 53 | return "" 54 | } 55 | 56 | return p.Fragment 57 | } 58 | 59 | var netFunctions = map[string]any{ 60 | "url_host": urlHost, 61 | "url_port": urlPort, 62 | "url_scheme": urlScheme, 63 | "url_path": urlPath, 64 | "url_param": urlParam, 65 | "url_fragment": urlFragment, 66 | } 67 | -------------------------------------------------------------------------------- /.github/workflows/pull_requests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | jobs: 8 | validate: 9 | name: 'Code is clean' 10 | runs-on: ubuntu-latest 11 | 12 | if: github.event_name == 'pull_request' 13 | steps: 14 | - uses: actions/checkout@master 15 | 16 | - uses: actions/setup-go@v3 17 | with: 18 | go-version: '>=1.18.0' 19 | check-latest: true 20 | - run: yarn 21 | - run: yarn fmt 22 | - run: ./scripts/fail_on_diff.sh 23 | 24 | tests: 25 | name: 'Run tests' 26 | 27 | strategy: 28 | matrix: 29 | os: [ubuntu-latest, windows-latest, macos-latest] 30 | runs-on: ${{ matrix.os }} 31 | 32 | steps: 33 | - uses: actions/checkout@master 34 | 35 | - uses: actions/setup-go@v3 36 | with: 37 | go-version: '>=1.18.0' 38 | check-latest: true 39 | 40 | - name: Downgrade MinGW on Windows (see https://github.com/golang/go/issues/46099) 41 | run: | 42 | if [ "$RUNNER_OS" == "Windows" ]; then 43 | choco install mingw --version 10.2.0 --allow-downgrade 44 | fi 45 | shell: bash 46 | 47 | - run: go build 48 | - run: go test -race -cover 49 | # Test examples 50 | - run: go run main.go 51 | working-directory: ./examples/connecthook 52 | - run: go run main.go 53 | working-directory: ./examples/basic 54 | -------------------------------------------------------------------------------- /scripts/.github/workflows/pull_requests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | 11 | if: github.event_name == 'pull_request' 12 | steps: 13 | - uses: actions/checkout@master 14 | with: 15 | ref: ${{ github.ref }} 16 | 17 | - run: ./scripts/ci/prepare_linux.sh 18 | - run: go test -race -cover 19 | - run: gofmt -w -s . 20 | - run: ./scripts/fail_on_diff.sh 21 | 22 | dsq-tests-ubuntu: 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - uses: actions/checkout@master 27 | with: 28 | ref: ${{ github.ref }} 29 | 30 | - run: ./scripts/ci/prepare_linux.sh 31 | - run: go build 32 | - run: ./scripts/test.py 33 | 34 | dsq-tests-windows: 35 | runs-on: windows-latest 36 | 37 | steps: 38 | - uses: actions/checkout@master 39 | with: 40 | ref: ${{ github.ref }} 41 | 42 | - run: ./scripts/ci/prepare_windows.ps1 43 | shell: pwsh 44 | - run: go build 45 | - run: ./scripts/test.py 46 | shell: bash 47 | 48 | # Subtle behavioral differences between powershell/cmd and bash. For example nested double quotes must be escaped. 49 | - run: python3 ./scripts/test.py 50 | shell: powershell 51 | - run: python3 ./scripts/test.py 52 | shell: cmd 53 | 54 | dsq-tests-macos: 55 | runs-on: macos-latest 56 | 57 | steps: 58 | - uses: actions/checkout@master 59 | with: 60 | ref: ${{ github.ref }} 61 | 62 | - run: ./scripts/ci/prepare_macos.sh 63 | - run: go build 64 | - run: ./scripts/test.py 65 | -------------------------------------------------------------------------------- /encoding_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_hash(t *testing.T) { 9 | in := "galloping cattle" 10 | tests := []struct { 11 | fn string 12 | out string 13 | }{ 14 | {"base32", "M5QWY3DPOBUW4ZZAMNQXI5DMMU======"}, 15 | {"base64", "Z2FsbG9waW5nIGNhdHRsZQ=="}, 16 | {"md5", "aee95ca37cfc14f4e623bb7effabbe1d"}, 17 | {"sha1", "45f159ccf56cf30d2dacfe4ce02ce0de88838344"}, 18 | {"sha256", "b618391489bd9629f7338f01015f9145b929eb845fd72f2145e83561741963e9"}, 19 | {"sha512", "ec875557fe4cd2c9f8370f31c97f9c3956286efa18b4146c46a99a11c13f3aac947e8cb6a9acce29901f08151340ba2bcbe52c32e6acd8f0ca7475f3472baac3"}, 20 | {"sha3_256", "04ec6e35949de98dfa5f56b93a02fadf15e188f1421d45a1d082c9e803b7a05d"}, 21 | {"sha3_512", "f22e4b213c152cb1081585d9408847b016b05b352a9442faa66a55cff02ac20cec181e2376a641f54ce77b922cd14a0a8d11a4b1e153b329a1d8b5eb80be2300"}, 22 | {"blake2b_256", "d76d404e37d5db7f0ddfe62a99ac5d8744d77b58a0a6e7ed5d37de7a372f8f28"}, 23 | {"blake2b_512", "ce3df894669848b1a513454b0e6254e61694d42f9996f20fdacde0daea61444b961f33cbcc594f9290752aa3724bdbdc7be2932f14765d53b9f20baec91274cc"}, 24 | } 25 | 26 | for _, test := range tests { 27 | assertQuery(t, fmt.Sprintf("SELECT %s('%s')", test.fn, in), test.out) 28 | } 29 | } 30 | 31 | func Test_unhash(t *testing.T) { 32 | out := "galloping cattle" 33 | tests := []struct { 34 | fn string 35 | in string 36 | }{ 37 | {"from_base32", "M5QWY3DPOBUW4ZZAMNQXI5DMMU======"}, 38 | {"from_base64", "Z2FsbG9waW5nIGNhdHRsZQ=="}, 39 | } 40 | 41 | for _, test := range tests { 42 | assertQuery(t, fmt.Sprintf("SELECT %s('%s')", test.fn, test.in), out) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /date.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/araddon/dateparse" 7 | ) 8 | 9 | func dateYear(s string) int64 { 10 | t, err := dateparse.ParseAny(s) 11 | if err != nil { 12 | return -1 13 | } 14 | 15 | return int64(t.Year()) 16 | } 17 | 18 | func dateMonth(s string) int64 { 19 | t, err := dateparse.ParseAny(s) 20 | if err != nil { 21 | return -1 22 | } 23 | 24 | return int64(t.Month()) 25 | } 26 | 27 | func dateDay(s string) int64 { 28 | t, err := dateparse.ParseAny(s) 29 | if err != nil { 30 | return -1 31 | } 32 | 33 | return int64(t.Day()) 34 | } 35 | 36 | func dateYearDay(s string) int64 { 37 | t, err := dateparse.ParseAny(s) 38 | if err != nil { 39 | return -1 40 | } 41 | 42 | return int64(t.YearDay()) 43 | } 44 | 45 | func dateHour(s string) int64 { 46 | t, err := dateparse.ParseAny(s) 47 | if err != nil { 48 | return -1 49 | } 50 | 51 | return int64(t.Hour()) 52 | } 53 | 54 | func dateMinute(s string) int64 { 55 | t, err := dateparse.ParseAny(s) 56 | if err != nil { 57 | return -1 58 | } 59 | 60 | return int64(t.Minute()) 61 | } 62 | 63 | func dateSecond(s string) int64 { 64 | t, err := dateparse.ParseAny(s) 65 | if err != nil { 66 | return -1 67 | } 68 | 69 | return int64(t.Second()) 70 | } 71 | 72 | func dateUnix(s string) int64 { 73 | t, err := dateparse.ParseAny(s) 74 | if err != nil { 75 | return -1 76 | } 77 | 78 | return t.Unix() 79 | } 80 | 81 | func dateRfc3339(s string) string { 82 | t, err := dateparse.ParseAny(s) 83 | if err != nil { 84 | return "" 85 | } 86 | 87 | return t.Format(time.RFC3339) 88 | } 89 | 90 | var dateFunctions = map[string]any{ 91 | "date_year": stringy1int64(dateYear), 92 | "date_month": stringy1int64(dateMonth), 93 | "date_day": stringy1int64(dateDay), 94 | "date_yearday": stringy1int64(dateYearDay), 95 | "date_hour": stringy1int64(dateHour), 96 | "date_minute": stringy1int64(dateMinute), 97 | "date_second": stringy1int64(dateSecond), 98 | "date_unix": stringy1int64(dateUnix), 99 | "date_rfc3339": stringy1string(dateRfc3339), 100 | } 101 | -------------------------------------------------------------------------------- /encoding.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha1" 6 | "crypto/sha256" 7 | "crypto/sha512" 8 | "encoding/base32" 9 | "encoding/base64" 10 | "fmt" 11 | 12 | "golang.org/x/crypto/blake2b" 13 | "golang.org/x/crypto/sha3" 14 | ) 15 | 16 | func ext_md5(s string) string { 17 | return fmt.Sprintf("%x", md5.Sum([]byte(s))) 18 | } 19 | 20 | func ext_sha1(s string) string { 21 | return fmt.Sprintf("%x", sha1.Sum([]byte(s))) 22 | } 23 | 24 | func ext_sha256(s string) string { 25 | return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) 26 | } 27 | 28 | func ext_sha512(s string) string { 29 | return fmt.Sprintf("%x", sha512.Sum512([]byte(s))) 30 | } 31 | 32 | func sha3_256(s string) string { 33 | return fmt.Sprintf("%x", sha3.Sum256([]byte(s))) 34 | } 35 | 36 | func sha3_512(s string) string { 37 | return fmt.Sprintf("%x", sha3.Sum512([]byte(s))) 38 | } 39 | 40 | func blake2b_256(s string) string { 41 | return fmt.Sprintf("%x", blake2b.Sum256([]byte(s))) 42 | } 43 | 44 | func blake2b_512(s string) string { 45 | return fmt.Sprintf("%x", blake2b.Sum512([]byte(s))) 46 | } 47 | 48 | func toBase64(s string) string { 49 | return base64.StdEncoding.EncodeToString([]byte(s)) 50 | } 51 | 52 | func fromBase64(s string) string { 53 | d, _ := base64.StdEncoding.DecodeString(s) 54 | return string(d) 55 | } 56 | 57 | func toBase32(s string) string { 58 | return base32.StdEncoding.EncodeToString([]byte(s)) 59 | } 60 | 61 | func fromBase32(s string) string { 62 | d, _ := base32.StdEncoding.DecodeString(s) 63 | return string(d) 64 | } 65 | 66 | var encodingFunctions = map[string]any{ 67 | "base64": stringy1string(toBase64), 68 | "from_base64": stringy1string(fromBase64), 69 | "base32": stringy1string(toBase32), 70 | "from_base32": stringy1string(fromBase32), 71 | 72 | "md5": stringy1string(ext_md5), 73 | "sha1": stringy1string(ext_sha1), 74 | "sha256": stringy1string(ext_sha256), 75 | "sha512": stringy1string(ext_sha512), 76 | "sha3_256": stringy1string(sha3_256), 77 | "sha3_512": stringy1string(sha3_512), 78 | "blake2b_256": stringy1string(blake2b_256), 79 | "blake2b_512": stringy1string(blake2b_512), 80 | } 81 | -------------------------------------------------------------------------------- /examples/basic/go.sum: -------------------------------------------------------------------------------- 1 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= 2 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 6 | github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= 7 | github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 11 | github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 15 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= 16 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 17 | golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs= 18 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= 19 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 20 | gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= 21 | gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= 22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 24 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 25 | -------------------------------------------------------------------------------- /examples/connecthook/go.sum: -------------------------------------------------------------------------------- 1 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= 2 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 6 | github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= 7 | github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 11 | github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 15 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= 16 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 17 | golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs= 18 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= 19 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 20 | gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= 21 | gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= 22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 24 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 25 | -------------------------------------------------------------------------------- /math_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "database/sql" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var initialized = false 11 | 12 | func assertQueryPrepare(t *testing.T, queries []string, res string) { 13 | if !initialized { 14 | Register("sqlite3_ext") 15 | initialized = true 16 | } 17 | 18 | db, err := sql.Open("sqlite3_ext", ":memory:") 19 | assert.Nil(t, err) 20 | 21 | for _, query := range queries[:len(queries)-1] { 22 | _, err = db.Exec(query) 23 | assert.Nil(t, err) 24 | } 25 | 26 | query := queries[len(queries)-1] 27 | t.Logf("Query: %s, expecting: %s", query, res) 28 | var s string 29 | err = db.QueryRow(query).Scan(&s) 30 | assert.Nil(t, err) 31 | assert.Equal(t, res, s) 32 | } 33 | 34 | func assertQuery(t *testing.T, query string, res string) { 35 | assertQueryPrepare(t, []string{query}, res) 36 | } 37 | 38 | func Test_floaty(t *testing.T) { 39 | tests := []struct { 40 | in any 41 | out float64 42 | }{ 43 | {"0.1", 0.1}, 44 | {"sdflkj", 0}, 45 | {12, 12.0}, 46 | {nil, 0}, 47 | {int8(1), 1}, 48 | {int16(1), 1}, 49 | {int32(1), 1}, 50 | {int64(1), 1}, 51 | {uint(1), 1}, 52 | {uint16(1), 1}, 53 | {uint32(1), 1}, 54 | {uint64(1), 1}, 55 | {float32(1), 1}, 56 | } 57 | 58 | for _, test := range tests { 59 | assert.Equal(t, test.out, floaty(test.in)) 60 | } 61 | } 62 | 63 | func Test_acos(t *testing.T) { 64 | assertQuery(t, "SELECT acos(0.1)", "1.4706289056333368") 65 | } 66 | 67 | func Test_radians(t *testing.T) { 68 | assertQuery(t, "SELECT radians(180)", "3.141592653589793") 69 | } 70 | 71 | func Test_degrees(t *testing.T) { 72 | assertQuery(t, "SELECT degrees(3.141592653589793)", "180") 73 | } 74 | 75 | func Test_mod(t *testing.T) { 76 | assertQuery(t, "SELECT mod('10', '2')", "0") 77 | } 78 | 79 | func Test_pi(t *testing.T) { 80 | assertQuery(t, "SELECT pi()", "3.141592653589793") 81 | } 82 | 83 | func Test_acosh(t *testing.T) { 84 | assertQuery(t, "SELECT acosh(2.0)", "1.3169578969248166") 85 | } 86 | 87 | func Test_trunc(t *testing.T) { 88 | assertQuery(t, "SELECT trunc(10.4)", "10") 89 | assertQuery(t, "SELECT trunc(-10.9)", "-10") 90 | assertQuery(t, "SELECT trunc(0.49999)", "0") 91 | assertQuery(t, "SELECT trunc(-0.49999)", "0") 92 | } 93 | 94 | func Test_floor(t *testing.T) { 95 | assertQueryPrepare(t, []string{ 96 | "CREATE TABLE x (n INT)", 97 | "INSERT INTO x VALUES (2)", 98 | "SELECT floor(n) FROM x", 99 | }, "2") 100 | } 101 | -------------------------------------------------------------------------------- /aggregate_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import "testing" 4 | 5 | func Test_stddev(t *testing.T) { 6 | assertQueryPrepare(t, []string{ 7 | "CREATE TABLE x (n INT)", 8 | "INSERT INTO x VALUES (1), (2)", 9 | "SELECT stddev_pop(n) FROM x", 10 | }, "0.5") 11 | } 12 | 13 | func Test_mode(t *testing.T) { 14 | assertQueryPrepare(t, []string{ 15 | "CREATE TABLE x (n INT)", 16 | "INSERT INTO x VALUES (1), (2), (2)", 17 | "SELECT mode(n) FROM x", 18 | }, "2") 19 | 20 | assertQueryPrepare(t, []string{ 21 | "CREATE TABLE x (n TEXT)", 22 | "INSERT INTO x VALUES ('a'), ('b'), ('a')", 23 | "SELECT mode(n) FROM x", 24 | }, "a") 25 | } 26 | 27 | func Test_median(t *testing.T) { 28 | assertQueryPrepare(t, []string{ 29 | "CREATE TABLE x (n INT)", 30 | "INSERT INTO x VALUES (1), (2), (2)", 31 | "SELECT median(n) FROM x", 32 | }, "2") 33 | 34 | assertQueryPrepare(t, []string{ 35 | "CREATE TABLE x (n INT)", 36 | "INSERT INTO x VALUES (1), (2), (3)", 37 | "SELECT median(n) FROM x", 38 | }, "2") 39 | 40 | assertQueryPrepare(t, []string{ 41 | "CREATE TABLE x (n INT)", 42 | "INSERT INTO x VALUES (1)", 43 | "SELECT median(n) FROM x", 44 | }, "1") 45 | 46 | assertQueryPrepare(t, []string{ 47 | "CREATE TABLE x (n INT)", 48 | "SELECT coalesce(median(n), 'null') FROM x", 49 | }, "null") 50 | 51 | assertQueryPrepare(t, []string{ 52 | "CREATE TABLE x (n TEXT)", 53 | "INSERT INTO x VALUES ('a'), ('b'), ('a'), ('c'), ('d')", 54 | "SELECT median(n) FROM x", 55 | }, "b") 56 | 57 | assertQueryPrepare(t, []string{ 58 | "CREATE TABLE x (n TEXT)", 59 | "INSERT INTO x VALUES (null), (null)", 60 | "SELECT coalesce(median(n), 'null') FROM x", 61 | }, "null") 62 | 63 | assertQueryPrepare(t, []string{ 64 | "CREATE TABLE x (n REAL)", 65 | "INSERT INTO x VALUES (1.2), (3.4), (4.4)", 66 | "SELECT median(n) FROM x", 67 | }, "3.4") 68 | } 69 | 70 | func Test_percentile(t *testing.T) { 71 | assertQueryPrepare(t, []string{ 72 | "CREATE TABLE x (n INT)", 73 | "INSERT INTO x VALUES (1), (3), (4)", 74 | "SELECT perc_50(n) FROM x", 75 | }, "3") 76 | 77 | assertQueryPrepare(t, []string{ 78 | "CREATE TABLE x (n INT)", 79 | "INSERT INTO x VALUES (5), (2), (4)", 80 | "SELECT perc(n, 75) FROM x", 81 | }, "5") 82 | 83 | assertQueryPrepare(t, []string{ 84 | "CREATE TABLE x (n INT)", 85 | "INSERT INTO x VALUES (1), (3), (4)", 86 | "SELECT perc_cont_50(n) FROM x", 87 | }, "2") 88 | 89 | assertQueryPrepare(t, []string{ 90 | "CREATE TABLE x (n INT)", 91 | "INSERT INTO x VALUES (5), (2), (4)", 92 | "SELECT perc_cont(n, 75) FROM x", 93 | }, "4.25") 94 | } 95 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= 2 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 7 | github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= 8 | github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 9 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 10 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 11 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 12 | github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= 13 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 14 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 15 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 16 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 17 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 18 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 19 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= 20 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 21 | golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs= 22 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= 23 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 24 | gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= 25 | gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= 26 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 27 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 28 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 29 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 30 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 31 | -------------------------------------------------------------------------------- /math.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "math" 5 | "strconv" 6 | ) 7 | 8 | func floaty(a any) float64 { 9 | switch t := a.(type) { 10 | case float64: 11 | return t 12 | case float32: 13 | return float64(t) 14 | case int: 15 | return float64(t) 16 | case int8: 17 | return float64(t) 18 | case int16: 19 | return float64(t) 20 | case int32: 21 | return float64(t) 22 | case int64: 23 | return float64(t) 24 | case uint: 25 | return float64(t) 26 | case uint16: 27 | return float64(t) 28 | case uint32: 29 | return float64(t) 30 | case uint64: 31 | return float64(t) 32 | case string: 33 | s, _ := strconv.ParseFloat(t, 64) 34 | return s 35 | default: 36 | return 0.0 37 | } 38 | } 39 | 40 | func degrees(rad float64) float64 { 41 | return floaty(rad) * (180 / math.Pi) 42 | } 43 | 44 | func pi() float64 { 45 | return math.Pi 46 | } 47 | 48 | func radians(deg float64) float64 { 49 | return floaty(deg) * (math.Pi / 180) 50 | } 51 | 52 | // Rounds toward zero 53 | func trunc(f float64) float64 { 54 | if f >= 0 { 55 | return math.Floor(f) 56 | } 57 | 58 | r := math.Floor(math.Abs(f)) 59 | if r == 0 { 60 | return 0 61 | } 62 | 63 | return -1 * r 64 | } 65 | 66 | func floaty1Float64(f func(float64) float64) func(a any) float64 { 67 | return func(a any) float64 { 68 | return f(floaty(a)) 69 | } 70 | } 71 | 72 | func floaty2Float64(f func(float64, float64) float64) func(a any, b any) float64 { 73 | return func(a any, b any) float64 { 74 | return f(floaty(a), floaty(b)) 75 | } 76 | } 77 | 78 | var mathFunctions = map[string]any{ 79 | "acos": floaty1Float64(math.Acos), 80 | "acosh": floaty1Float64(math.Acosh), 81 | "asin": floaty1Float64(math.Asin), 82 | "asinh": floaty1Float64(math.Asinh), 83 | "atan": floaty1Float64(math.Atan), 84 | // TODO: atan2 85 | "atanh": floaty1Float64(math.Atanh), 86 | "ceil": floaty1Float64(math.Ceil), 87 | "ceiling": floaty1Float64(math.Ceil), 88 | "cos": floaty1Float64(math.Cos), 89 | "cosh": floaty1Float64(math.Cosh), 90 | "degrees": floaty1Float64(degrees), 91 | "exp": floaty1Float64(math.Exp), 92 | "floor": floaty1Float64(math.Floor), 93 | "ln": floaty1Float64(math.Log), 94 | "log": floaty1Float64(math.Log), 95 | "log10": floaty1Float64(math.Log10), 96 | // TODO: support log(B, X) 97 | "log2": floaty1Float64(math.Log2), 98 | "mod": floaty2Float64(math.Mod), 99 | "pi": pi, 100 | "pow": floaty2Float64(math.Pow), 101 | "power": floaty2Float64(math.Pow), 102 | "radians": floaty1Float64(radians), 103 | "sin": floaty1Float64(math.Sin), 104 | "sinh": floaty1Float64(math.Sinh), 105 | "sqrt": floaty1Float64(math.Sqrt), 106 | "tan": floaty1Float64(math.Tan), 107 | "tanh": floaty1Float64(math.Tanh), 108 | "trunc": floaty1Float64(trunc), 109 | "truncate": floaty1Float64(trunc), 110 | } 111 | -------------------------------------------------------------------------------- /string.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "unicode/utf8" 7 | ) 8 | 9 | func stringy(a any) string { 10 | if a == nil { 11 | return "null" 12 | } 13 | 14 | switch t := a.(type) { 15 | case string: 16 | return t 17 | case []byte: 18 | if len(t) == 0 { 19 | return "null" 20 | } 21 | 22 | return string(t) 23 | } 24 | 25 | return fmt.Sprintf("%v", a) 26 | } 27 | 28 | func stringy1int64(f func(a string) int64) func(a any) int64 { 29 | return func(a any) int64 { 30 | return f(stringy(a)) 31 | } 32 | } 33 | 34 | func stringy2int64(f func(a, b string) int64) func(a, b any) int64 { 35 | return func(a, b any) int64 { 36 | return f(stringy(a), stringy(b)) 37 | } 38 | } 39 | 40 | func stringy1string(f func(a string) string) func(a any) string { 41 | return func(a any) string { 42 | return f(stringy(a)) 43 | } 44 | } 45 | 46 | func repeat(s any, n int) string { 47 | var sb strings.Builder 48 | for i := 0; i < n; i++ { 49 | sb.WriteString(stringy(s)) 50 | } 51 | 52 | return sb.String() 53 | } 54 | 55 | func charindex(s, sub string) int64 { 56 | return int64(strings.Index(s, sub)) 57 | } 58 | 59 | // SOURCE: https://stackoverflow.com/a/20225618/1507139 60 | func reverse(s string) string { 61 | size := len(s) 62 | buf := make([]byte, size) 63 | for start := 0; start < size; { 64 | r, n := utf8.DecodeRuneInString(s[start:]) 65 | start += n 66 | utf8.EncodeRune(buf[size-start:], r) 67 | } 68 | return string(buf) 69 | } 70 | 71 | func lpad(_s any, length int, _padWith ...any) string { 72 | s := stringy(_s) 73 | 74 | if len(s) > length { 75 | return s[:length] 76 | } 77 | 78 | padWith := []rune(" ") 79 | if len(_padWith) > 0 { 80 | padWith = []rune(stringy(_padWith[0])) 81 | } 82 | 83 | var sb strings.Builder 84 | for i := 0; i < length-len(s); i++ { 85 | sb.WriteRune(padWith[i%len(padWith)]) 86 | } 87 | 88 | sb.WriteString(s) 89 | return sb.String()[:length] 90 | } 91 | 92 | func rpad(_s any, length int, _padWith ...any) string { 93 | s := stringy(_s) 94 | 95 | if len(s) > length { 96 | return s[:length] 97 | } 98 | 99 | padWith := []rune(" ") 100 | if len(_padWith) > 0 { 101 | padWith = []rune(stringy(_padWith[0])) 102 | } 103 | 104 | var sb strings.Builder 105 | sb.WriteString(s) 106 | for i := 0; i < length-len(s); i++ { 107 | sb.WriteRune(padWith[i%len(padWith)]) 108 | } 109 | 110 | return sb.String() 111 | } 112 | 113 | func length(s string) int64 { 114 | return int64(len(s)) 115 | } 116 | 117 | func splitPart(s, sub, _part any) string { 118 | part := int(floaty(_part)) 119 | parts := strings.Split(stringy(s), stringy(sub)) 120 | if len(parts) == 0 || part >= len(parts) { 121 | return "" 122 | } 123 | 124 | if part < 0 { 125 | part = (len(parts) + part) % len(parts) 126 | } 127 | 128 | return parts[part] 129 | } 130 | 131 | var stringFunctions = map[string]any{ 132 | "len": stringy1int64(length), 133 | "split_part": splitPart, 134 | "repeat": repeat, 135 | "replicate": repeat, 136 | "strpos": stringy2int64(charindex), 137 | "charindex": stringy2int64(charindex), 138 | "reverse": stringy1string(reverse), 139 | "lpad": lpad, 140 | "rpad": rpad, 141 | } 142 | -------------------------------------------------------------------------------- /string_test.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_string(t *testing.T) { 11 | assert.Equal(t, "abc", stringy([]byte("abc"))) 12 | assert.Equal(t, "null", stringy(nil)) 13 | } 14 | 15 | func Test_len(t *testing.T) { 16 | assertQuery(t, "SELECT len('a')", "1") 17 | } 18 | 19 | func Test_split_part(t *testing.T) { 20 | assertQuery(t, "SELECT split_part('a', '', 0)", "a") 21 | assertQuery(t, "SELECT split_part('a', 'blub', 2)", "") 22 | assertQuery(t, "SELECT split_part('ablubablubb', 'blub', 2)", "b") 23 | assertQuery(t, "SELECT split_part('1,2,3', ',', 2)", "3") 24 | assertQuery(t, "SELECT split_part('1,2,3', ',', -1)", "3") 25 | } 26 | 27 | func Test_repeat(t *testing.T) { 28 | assertQuery(t, "SELECT repeat('a', 3)", "aaa") 29 | assertQuery(t, "SELECT replicate('a', 3)", "aaa") 30 | assertQuery(t, "SELECT repeat(3, 3)", "333") 31 | assertQuery(t, "SELECT repeat(null, 3)", "nullnullnull") 32 | } 33 | 34 | func Test_strpos(t *testing.T) { 35 | tests := []struct { 36 | haystack string 37 | needle string 38 | expectedIndex string 39 | }{ 40 | { 41 | "abcde", 42 | "f", 43 | "-1", 44 | }, 45 | { 46 | "abcde", 47 | "e", 48 | "4", 49 | }, 50 | { 51 | "abcde", 52 | "c", 53 | "2", 54 | }, 55 | { 56 | "abcde", 57 | "a", 58 | "0", 59 | }, 60 | } 61 | 62 | for _, test := range tests { 63 | assertQuery(t, fmt.Sprintf("SELECT strpos('%s', '%s')", test.haystack, test.needle), test.expectedIndex) 64 | assertQuery(t, fmt.Sprintf("SELECT charindex('%s', '%s')", test.haystack, test.needle), test.expectedIndex) 65 | } 66 | 67 | assertQuery(t, "SELECT strpos(1234, 3)", "2") 68 | } 69 | 70 | func Test_replace(t *testing.T) { 71 | assertQuery(t, "SELECT replace(' whatever ', 'whatever', 'blah')", " blah ") 72 | assertQuery(t, "SELECT replace(3443, 44, 55)", "3553") 73 | } 74 | 75 | func Test_reverse(t *testing.T) { 76 | assertQuery(t, "SELECT reverse('234')", "432") 77 | } 78 | 79 | func Test_lpad(t *testing.T) { 80 | tests := []struct { 81 | in string 82 | length int 83 | pad string 84 | out string 85 | }{ 86 | { 87 | "abcde", 88 | 3, 89 | "0", 90 | "abc", 91 | }, 92 | { 93 | "aa", 94 | 3, 95 | "0", 96 | "0aa", 97 | }, 98 | { 99 | "aa", 100 | 2, 101 | "0", 102 | "aa", 103 | }, 104 | { 105 | "a", 106 | 0, 107 | "0", 108 | "", 109 | }, 110 | } 111 | 112 | for _, test := range tests { 113 | assertQuery(t, fmt.Sprintf("SELECT lpad('%s', %d, '%s');", test.in, test.length, test.pad), test.out) 114 | } 115 | 116 | // Test no third argument variation 117 | assertQuery(t, "SELECT lpad('22', 3);", " 22") 118 | 119 | // int variation 120 | assertQuery(t, "SELECT lpad(22, 3, 0);", "022") 121 | } 122 | 123 | func Test_rpad(t *testing.T) { 124 | tests := []struct { 125 | in string 126 | length int 127 | pad string 128 | out string 129 | }{ 130 | { 131 | "abcde", 132 | 3, 133 | "0", 134 | "abc", 135 | }, 136 | { 137 | "aa", 138 | 3, 139 | "0", 140 | "aa0", 141 | }, 142 | { 143 | "aa", 144 | 2, 145 | "0", 146 | "aa", 147 | }, 148 | { 149 | "a", 150 | 0, 151 | "0", 152 | "", 153 | }, 154 | } 155 | 156 | for _, test := range tests { 157 | assertQuery(t, fmt.Sprintf("SELECT rpad('%s', %d, '%s');", test.in, test.length, test.pad), test.out) 158 | } 159 | 160 | // Test no 3rd argument variant 161 | assertQuery(t, "SELECT rpad('22', 3);", "22 ") 162 | } 163 | -------------------------------------------------------------------------------- /aggregate.go: -------------------------------------------------------------------------------- 1 | package stdlib 2 | 3 | import ( 4 | "bytes" 5 | "math" 6 | "sort" 7 | 8 | "gonum.org/v1/gonum/stat" 9 | ) 10 | 11 | type mode struct { 12 | counts map[any]int 13 | top any 14 | topCount int 15 | } 16 | 17 | func newMode() *mode { 18 | return &mode{ 19 | counts: map[any]int{}, 20 | } 21 | } 22 | 23 | func (m *mode) Step(x any) { 24 | m.counts[x]++ 25 | c := m.counts[x] 26 | if c > m.topCount { 27 | m.top = x 28 | m.topCount = c 29 | } 30 | } 31 | 32 | func (m *mode) Done() any { 33 | return m.top 34 | } 35 | 36 | type stddev struct { 37 | mean float64 38 | count int 39 | meanSquared float64 40 | } 41 | 42 | func newStddev() *stddev { return &stddev{} } 43 | 44 | func (s *stddev) Step(x any) { 45 | // Welford's method 46 | // https://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/ 47 | xf := floaty(x) 48 | s.count++ 49 | oldMean := s.mean 50 | s.mean += (xf - s.mean) / float64(s.count) 51 | s.meanSquared += (xf - s.mean) * (xf - oldMean) 52 | } 53 | 54 | func (s *stddev) Done() float64 { 55 | if s.count < 2 { 56 | return 0 57 | } 58 | 59 | return s.meanSquared / float64(s.count-1) 60 | } 61 | 62 | type percentile struct { 63 | xs []float64 64 | percentile float64 65 | kind stat.CumulantKind 66 | } 67 | 68 | func newPercentile() *percentile { 69 | return &percentile{ 70 | kind: stat.Empirical, 71 | } 72 | } 73 | 74 | func newPercentileN(n int) func() *percentile { 75 | return func() *percentile { 76 | p := newPercentile() 77 | p.percentile = float64(n) 78 | return p 79 | } 80 | } 81 | 82 | func newPercentileCont() *percentile { 83 | return &percentile{ 84 | kind: stat.LinInterp, 85 | } 86 | } 87 | 88 | func newPercentileContN(n int) func() *percentile { 89 | return func() *percentile { 90 | p := newPercentileCont() 91 | p.percentile = float64(n) 92 | return p 93 | } 94 | } 95 | 96 | func (s *percentile) Step(x any, perc ...any) { 97 | if len(perc) > 0 { 98 | s.percentile = floaty(perc[0]) 99 | } 100 | 101 | s.xs = append(s.xs, floaty(x)) 102 | } 103 | 104 | func (s *percentile) Done() float64 { 105 | if s.percentile == 0 || len(s.xs) == 0 { 106 | return 0 107 | } 108 | 109 | sort.Float64s(s.xs) 110 | r := stat.Quantile(s.percentile/100, s.kind, s.xs, nil) 111 | return r 112 | } 113 | 114 | type sqliteValueKind uint 115 | 116 | const ( 117 | sqliteNull sqliteValueKind = iota 118 | sqliteInt 119 | sqliteString 120 | sqliteReal 121 | sqliteBlob 122 | ) 123 | 124 | type sqliteValue struct { 125 | kind sqliteValueKind 126 | i int64 127 | s string 128 | r float64 129 | b []byte 130 | } 131 | 132 | type sqliteValues []sqliteValue 133 | 134 | func (svs *sqliteValues) Len() int { 135 | return len(*svs) 136 | } 137 | 138 | func (svs *sqliteValues) Less(i, j int) bool { 139 | ie := (*svs)[i] 140 | je := (*svs)[j] 141 | if ie.kind != je.kind { 142 | // TODO: support mixed value types? 143 | return false 144 | } 145 | 146 | switch ie.kind { 147 | case sqliteInt: 148 | return ie.i < je.i 149 | case sqliteString: 150 | return ie.s < je.s 151 | case sqliteReal: 152 | return ie.r < je.r 153 | case sqliteBlob: 154 | return bytes.Compare(ie.b, je.b) < 0 155 | } 156 | 157 | return false 158 | } 159 | 160 | func (svs *sqliteValues) Swap(i, j int) { 161 | (*svs)[i], (*svs)[j] = (*svs)[j], (*svs)[i] 162 | } 163 | 164 | type median struct { 165 | xs sqliteValues 166 | } 167 | 168 | func newMedian() *median { 169 | return &median{} 170 | } 171 | 172 | func (m *median) Step(x any) { 173 | v := sqliteValue{kind: sqliteNull} 174 | switch t := x.(type) { 175 | case int64: 176 | v.kind = sqliteInt 177 | v.i = t 178 | case int: 179 | v.kind = sqliteInt 180 | v.i = int64(t) 181 | case string: 182 | v.kind = sqliteString 183 | v.s = t 184 | case float64: 185 | v.kind = sqliteReal 186 | v.r = t 187 | case []byte: 188 | v.kind = sqliteBlob 189 | v.b = t 190 | } 191 | m.xs = append(m.xs, v) 192 | } 193 | 194 | func (m *median) Done() any { 195 | if len(m.xs) == 0 { 196 | return nil 197 | } 198 | 199 | sort.Sort(&m.xs) 200 | e := m.xs[int(math.Floor(float64(len(m.xs))/2))] 201 | switch e.kind { 202 | case sqliteInt: 203 | return e.i 204 | case sqliteString: 205 | return e.s 206 | case sqliteReal: 207 | return e.r 208 | case sqliteBlob: 209 | return e.b 210 | } 211 | 212 | return nil 213 | } 214 | 215 | var aggregateFunctions = map[string]any{ 216 | "stddev": newStddev, 217 | "stdev": newStddev, 218 | "stddev_pop": newStddev, 219 | "mode": newMode, 220 | "median": newMedian, 221 | "percentile_25": newPercentileN(25), 222 | "perc_25": newPercentileN(25), 223 | "percentile_50": newPercentileN(50), 224 | "perc_50": newPercentileN(50), 225 | "percentile_75": newPercentileN(75), 226 | "perc_75": newPercentileN(75), 227 | "percentile_90": newPercentileN(90), 228 | "perc_90": newPercentileN(90), 229 | "percentile_95": newPercentileN(95), 230 | "perc_95": newPercentileN(95), 231 | "percentile_99": newPercentileN(99), 232 | "perc_99": newPercentileN(99), 233 | "percentile": newPercentile, 234 | "perc": newPercentile, 235 | "percentile_cont_25": newPercentileContN(25), 236 | "perc_cont_25": newPercentileContN(25), 237 | "percentile_cont_50": newPercentileContN(50), 238 | "perc_cont_50": newPercentileContN(50), 239 | "percentile_cont_75": newPercentileContN(75), 240 | "perc_cont_75": newPercentileContN(75), 241 | "percentile_cont_90": newPercentileContN(90), 242 | "perc_cont_90": newPercentileContN(90), 243 | "percentile_cont_95": newPercentileContN(95), 244 | "perc_cont_95": newPercentileContN(95), 245 | "percentile_cont_99": newPercentileContN(99), 246 | "perc_cont_99": newPercentileContN(99), 247 | "percentile_cont": newPercentileCont, 248 | "perc_cont": newPercentileCont, 249 | } 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A standard library for mattn/go-sqlite3 2 | 3 | As an alternative to compiling C extensions like 4 | [extension-functions.c](https://www.sqlite.org/contrib) and 5 | [sqlean](https://github.com/nalgeon/sqlean) into 6 | [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), this package 7 | implements many of these functions (and more from PostgreSQL) in Go. 8 | 9 | These are in addition to [all builtin 10 | functions](https://www.sqlite.org/lang_corefunc.html) provided by 11 | SQLite. 12 | 13 | Continue reading for all functions, notes and examples. 14 | 15 | # Why would I use this? 16 | 17 | This library is used in 18 | [DataStation](https://github.com/multiprocessio/datastation) and 19 | [dsq](https://github.com/multiprocessio/dsq) to simplify and power 20 | data analysis in SQL. 21 | 22 | ![Analyzing logs with SQL in DataStation](./screenshot.png) 23 | 24 | Read the [DataStation blog 25 | post](https://datastation.multiprocess.io/docs/0.11.0-release-notes.html) 26 | to better understand the background. 27 | 28 | # Example 29 | 30 | ```go 31 | package main 32 | 33 | import ( 34 | "fmt" 35 | "database/sql" 36 | 37 | _ "github.com/mattn/go-sqlite3" 38 | stdlib "github.com/multiprocessio/go-sqlite3-stdlib" 39 | ) 40 | 41 | func main() { 42 | stdlib.Register("sqlite3_ext") 43 | db, err := sql.Open("sqlite3_ext", ":memory:") 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | var s string 49 | err = db.QueryRow("SELECT repeat('x', 2)").Scan(&s) 50 | if err != nil { 51 | panic(err) 52 | } 53 | 54 | fmt.Println(s) 55 | } 56 | ``` 57 | 58 | Alternatively if you want to be able to add your own additional 59 | extensions you can just use the `ConnectHook`: 60 | 61 | ```go 62 | package main 63 | 64 | import ( 65 | "database/sql" 66 | "fmt" 67 | 68 | sqlite3 "github.com/mattn/go-sqlite3" 69 | stdlib "github.com/multiprocessio/go-sqlite3-stdlib" 70 | ) 71 | 72 | func main() { 73 | sql.Register("sqlite3_ext", 74 | &sqlite3.SQLiteDriver{ 75 | ConnectHook: stdlib.ConnectHook, 76 | }) 77 | db, err := sql.Open("sqlite3_ext", ":memory:") 78 | if err != nil { 79 | panic(err) 80 | } 81 | 82 | var s string 83 | err = db.QueryRow("SELECT repeat('x', 2)").Scan(&s) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | fmt.Println(s) 89 | } 90 | ``` 91 | 92 | # Functions 93 | 94 | ## Strings 95 | 96 | | Name(s) | Notes | Example | 97 | | ----------------- | --------------------------------------------------------- | -------------------------------------------------------------- | 98 | | repeat, replicate | | `repeat('f', 5) = 'fffff'` | 99 | | strpos, charindex | 0-indexed position of substring in string | `strpos('abc', 'b') = 1` | 100 | | reverse | | `reverse('abc') = 'cba'` | 101 | | lpad | Omit the third argument to default to padding with spaces | `lpad('22', 3, '0') = '022'` | 102 | | rpad | Omit the third argument to default to padding with spaces | `rpad('22', 3, '0') = '220'` | 103 | | len | Shorthand for `length` | `len('my string') = '9'` | 104 | | split_part | Split string an take nth split piece | `split('1,2,3', ',', 0) = '1'`, `split('1,2,3', ',' -1) = '3'` | 105 | | regexp | Go's regexp package, not PCRE | `x REGEXP '[a-z]+$'`, `REGEXP('[a-z]+$', x)` | 106 | | regexp_count | Number of times the regexp matches in string | `regexp_count('abc1', '[a-z]1') = '1'` | 107 | | regexp_split_part | Regexp equivalent of `split_part` | `regexp_split_part('ab12', '[a-z]1', 0) = 'a'` | 108 | 109 | ## Aggregation 110 | 111 | Most of these are implemented as bindings to 112 | [gonum](https://gonum.org/v1/gonum). 113 | 114 | | Name(s) | Notes | Example | 115 | | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ------------------------------ | 116 | | stddev, stdev, stddev_pop | | `stddev(n)` | 117 | | mode | | `mode(n)` | 118 | | median | | `median(n)` | 119 | | percentile, perc | Discrete | `perc(response_time, 95)` | 120 | | percentile_25, perc_25, percentile_50, perc_50, percentile_75, perc_75, percentile_90, perc_90, percentile_95, perc_95, percentile_99, perc_99 | Discrete | `perc_99(response_time)` | 121 | | percentile_cont, perc_cont | Continuous | `perc_cont(response_time, 95)` | 122 | | percentile_cont_25, perc_cont_25, percentile_cont_50, perc_cont_50, percentile_cont_75, perc_cont_75, percentile_cont_90, perc_cont_90, percentile_cont_95, perc_cont_95, percentile_cont_99, perc_cont_99 | Continuous | `perc_cont_99(response_time)` | 123 | 124 | ## Net 125 | 126 | | Name(s) | Notes | Example | 127 | | ------------ | ----- | ------------------------------------------------------------------------------------- | 128 | | url_scheme | | `url_scheme('https://x.com:90/home.html') = 'https'` | 129 | | url_host | | `url_host('https://x.com:90/home.html') = 'x.com:90'` | 130 | | url_port | | `url_port('https://x.com:90/home.html') = '90'` | 131 | | url_path | | `url_path('https://x.com/some/path.html?p=123') = '/some/path.html'` | 132 | | url_param | | `url_param('https://x.com/home.html?p=123&z=%5B1%2C2%5D#section-1', 'z') = '[1,2]'` | 133 | | url_fragment | | `url_fragment('https://x.com/home.html?p=123&z=%5B1%2C2%5D#section-1') = 'section-1'` | 134 | 135 | ## Date 136 | 137 | Best effort family of date parsing (uses 138 | [dateparse](https://github.com/araddon/dateparse)) and date part 139 | retrieval. Results will differ depending on your computer's timezone. 140 | 141 | | Name(s) | Notes | Example | 142 | | ------------ | ------------------- | ------------------------------------------------------------ | 143 | | date_year | | `date_year('2021-04-05') = 2021` | 144 | | date_month | January is 1, not 0 | `date_month('May 6, 2021') = 5` | 145 | | date_day | | `date_day('May 6, 2021') = 6` | 146 | | date_yearday | Day offset in year | `date_yearday('May 6, 2021') = 127` | 147 | | date_hour | 24-hour | `date_hour('May 6, 2021 4:50 PM') = 16` | 148 | | date_minute | | `date_minute('May 6, 2021 4:50') = 50` | 149 | | date_second | | `date_second('May 6, 2021 4:50:20') = 20` | 150 | | date_unix | | `date_unix('May 6, 2021 4:50:20') = 1588740620` | 151 | | date_rfc3339 | | `date_rfc3339('May 6, 2021 4:50:20') = 2020-05-06T04:50:20Z` | 152 | 153 | ## Math 154 | 155 | | Name(s) | Notes | Example | 156 | | --------------- | -------------------------------------------------------- | ------------------------------------------ | 157 | | acos | | `acos(n)` | 158 | | acosh | | `acosh(n)` | 159 | | asin | | `asin(n)` | 160 | | asinh | | `asinh(n)` | 161 | | atan | | `atan(n)` | 162 | | atanh | | `atanh(n)` | 163 | | ceil, ceiling | | `ceil(n)` | 164 | | cos | | `ceil(n)` | 165 | | cosh | | `cosh(n)` | 166 | | degrees | | `degrees(radians)` | 167 | | exp | e^n | `exp(n)` | 168 | | floor | | `floor(n)` | 169 | | ln, log | | `log(x)` | 170 | | log10 | | `log10(x)` | 171 | | log2 | | `log2(x)` | 172 | | mod | | `mod(num, denom)` | 173 | | pi | | `pi()` | 174 | | pow, power | | `pow(base, exp)` | 175 | | radians | | `radians(degrees)` | 176 | | sin | | `sin(n)` | 177 | | sinh | | `sinh(n)` | 178 | | sqrt | | `sqrt(n)` | 179 | | tan | | `tan(n)` | 180 | | tanh | | `tanh(n)` | 181 | | trunc, truncate | Rounds up to zero if negative, down to zero if positive. | `trunc(-10.9) = -10`, `trunc(10.4) = 10.0` | 182 | 183 | ## Encoding 184 | 185 | | Name(s) | Notes | Example | 186 | | ----------- | ----------------------------- | ---------------- | 187 | | base64 | Convert string to base64 | `base64(s)` | 188 | | from_base64 | Convert string from base64 | `from_base64(s)` | 189 | | base32 | Convert string to base32 | `base32(s)` | 190 | | from_base32 | Convert string from base32 | `from_base32(s)` | 191 | | md5 | Hex md5 sum of string | `md5(s)` | 192 | | sha1 | Hex sha1 sum of string | `sha1(s)` | 193 | | sha256 | Hex sha256 sum of string | `sha256(s)` | 194 | | sha512 | Hex sha512 sum of string | `sha512(s)` | 195 | | sha3_256 | Hex sha3_256 sum of string | `sha3_256(s)` | 196 | | sha3_512 | Hex sha3_512 sum of string | `sha3_512(s)` | 197 | | blake2b_256 | Hex blake2b_256 sum of string | `blake2b_256(s)` | 198 | | blake2b_512 | Hex blake2b_512 sum of string | `blake2b_512(s)` | 199 | 200 | # How is this tested? 201 | 202 | There is 95% test coverage and automated tests on Windows, macOS and 203 | Linux. 204 | 205 | # I just want to use it as a CLI or GUI 206 | 207 | See [dsq](https://github.com/multiprocessio/dsq) (a command-line tool 208 | for executing SQL on data files) and 209 | [DataStation](https://github.com/multiprocessio/datastation), a GUI 210 | application for querying and building reports with data from 211 | databases, servers, and files. 212 | 213 | # Contribute 214 | 215 | Join the [#dev channel on the Multiprocess Labs 216 | Discord](https://discord.gg/22ZCpaS9qm). 217 | 218 | If you have an idea for a new function, say so on the Discord channel 219 | or open an issue here. 220 | 221 | Make sure the function doesn't already exist in dsq (or the sqlite3 222 | CLI). 223 | 224 | # License 225 | 226 | This software is licensed under an Apache 2.0 license. 227 | --------------------------------------------------------------------------------